@ -4,26 +4,28 @@ import common.ecommerce.shared.support.Page;
@@ -4,26 +4,28 @@ import common.ecommerce.shared.support.Page;
import common.ecommerce.shared.sync.SyncResult ;
import common.ecommerce.shared.sync.SyncSpec ;
import common.ecommerce.channel.model.order.SalesOrder ;
import common.ecommerce.channel.model.order.SalesOrderQuery ;
import common.ecommerce.channel.model.product.ProductListing ;
import common.ecommerce.channel.spi.Sink ;
import common.ecommerce.shared.spi.SyncCursorStore ;
import org.slf4j.Logger ;
import org.slf4j.LoggerFactory ;
import java.time.Instant ;
import java.util.function.BiFunction ;
/ * *
* 增量同步引擎 。
*
* < pre >
* from = lastWatermark - overlapWindow
* to = now
* loop 拉一页 → 经 Sink upsert → cursor = page . nextCursor
* to = 本次批次终点 ( 宿主传入 , 不取系统时间 )
* loop 拉一页 ( [ from , to ) ) → 经 Sink upsert → cursor = page . nextCursor
* 直到 cursor = = null
* 全部成功才推进水位线 , 失败不推进
* < / pre >
*
* < p > < b > 查询窗口与水位线一致 < / b > : 拉取用 { @code [ from , to ) } , 推进也到 { @code to } 。
* 若查询不带上界 、 水位线却推到 { @code to } , 窗口内尚未入账的数据会被永久跳过 。 < / p >
*
* < p > 引擎不持有线程或定时器 , 由宿主调度触发 。 < / p >
* /
public final class SyncEngine {
@ -46,33 +48,49 @@ public final class SyncEngine {
@@ -46,33 +48,49 @@ public final class SyncEngine {
}
/ * *
* 同步销售订单 。
* 拉一页的函数 。
*
* < p > { @code from } 与 { @code to } 构成左闭右开的时间窗 ; 首页 { @code cursor } 为 null ,
* 后续页原样传回上一页的 { @code nextCursor } 。 < / p >
* /
public interface PageFetcher < T > {
Page < T > fetch ( Instant from , Instant to , String cursor ) ;
}
/ * *
* 增量同步销售订单 。
*
* < p > 整批成功才推进水位线 ; 任何一页抛异常都会直接冒出 ,
* { @code SyncResult } 不会返回 , 水位线保持原值 。 引擎不私起线程 / 定时器 。 < / p >
*
* @param spec 同步维度 ( provider + account + stream + overlap )
* @param to 本次增量终点 ; 会作为新水位线写入 , 需非 null
* @param fetch 拉一页的函数 : ( from , cursor ) → page ; 首页 { @code cursor } 为 null
* @param to 本次增量终点 ( 不含 ) ; 查询上界 , 同时作为新水位线
* @param fetch 拉一页 : ( from , to , cursor ) → page
* @return 本次同步统计 ; 成功路径恒为 { @code watermarkAdvanced = true }
* @throws IllegalStateException 账户 / 流还没有水位线 ( 必须先跑一次全量建底 )
* @throws RuntimeException { @code fetch } / { @code sink } 抛出的异常原样冒出
* @throws IllegalArgumentException { @code to } 为 null 或不晚于起点
* @throws IllegalStateException 账户 / 流还没有水位线 ( 必须先跑 { @link # bootstrapSalesOrders } )
* @throws RuntimeException { @code fetch } / { @code sink } 抛出的异常原样冒出
* /
public SyncResult syncSalesOrders ( SyncSpec spec , Instant to ,
BiFunction < Instant , String , Page < SalesOrder > > fetch ) {
public SyncResult syncSalesOrders ( SyncSpec spec , Instant to , PageFetcher < SalesOrder > fetch ) {
if ( to = = null ) {
throw new IllegalArgumentException ( "to must not be null" ) ;
}
Instant watermark = cursorStore . get ( spec . provider ( ) . name ( ) , spec . accountKey ( ) , spec . stream ( ) ) ;
Instant from = spec . from ( watermark ) ;
if ( from = = null ) {
throw new IllegalStateException ( "no watermark for " + spec . accountKey ( )
+ "/" + spec . stream ( ) + ", run full bootstrap first" ) ;
}
if ( ! to . isAfter ( from ) ) {
throw new IllegalArgumentException ( "to (" + to + ") must be after from (" + from + ")" ) ;
}
int pages = 0 ;
int items = 0 ;
String cursor = null ;
// 首页需要 from,后续页沿用同一 from(时间窗固定,靠游标翻页)
// 时间窗固定为 [from, to):翻页靠游标,窗口不变
while ( true ) {
Page < SalesOrder > page = fetch . apply ( from , cursor ) ;
Page < SalesOrder > page = fetch . fetch ( from , to , cursor ) ;
if ( page = = null | | page . size ( ) = = 0 ) {
break ;
}
@ -89,4 +107,126 @@ public final class SyncEngine {
@@ -89,4 +107,126 @@ public final class SyncEngine {
LOG . debug ( "sync {} pages={} items={} from={} to={}" , spec . stream ( ) , pages , items , from , to ) ;
return SyncResult . of ( pages , items , true ) ;
}
/ * *
* 增量同步商品 / Listing 。
*
* @see # syncSalesOrders ( SyncSpec , Instant , PageFetcher )
* /
public SyncResult syncProducts ( SyncSpec spec , Instant to , PageFetcher < ProductListing > fetch ) {
if ( to = = null ) {
throw new IllegalArgumentException ( "to must not be null" ) ;
}
Instant watermark = cursorStore . get ( spec . provider ( ) . name ( ) , spec . accountKey ( ) , spec . stream ( ) ) ;
Instant from = spec . from ( watermark ) ;
if ( from = = null ) {
throw new IllegalStateException ( "no watermark for " + spec . accountKey ( )
+ "/" + spec . stream ( ) + ", run full bootstrap first" ) ;
}
if ( ! to . isAfter ( from ) ) {
throw new IllegalArgumentException ( "to (" + to + ") must be after from (" + from + ")" ) ;
}
int pages = 0 ;
int items = 0 ;
String cursor = null ;
while ( true ) {
Page < ProductListing > page = fetch . fetch ( from , to , cursor ) ;
if ( page = = null | | page . size ( ) = = 0 ) {
break ;
}
sink . upsertProducts ( page ) ;
pages + + ;
items + = page . size ( ) ;
cursor = page . nextCursor ( ) ;
if ( cursor = = null ) {
break ;
}
}
cursorStore . advance ( spec . provider ( ) . name ( ) , spec . accountKey ( ) , spec . stream ( ) , to ) ;
LOG . debug ( "sync {} pages={} items={} from={} to={}" , spec . stream ( ) , pages , items , from , to ) ;
return SyncResult . of ( pages , items , true ) ;
}
/ * *
* 首次全量建底 : 拉完整个窗口并落库 , 然后把水位线播种到 { @code to } 。
*
* < p > 增量同步要求先有水位线 ; 首次接入时用本方法把历史数据补一遍 。
* 与增量一样 , 整批成功才写入水位线 。 < / p >
*
* @param spec 同步维度
* @param from 全量起点 ( 含 )
* @param to 全量终点 ( 不含 ) ; 建成后的水位线
* @param fetch 拉一页 : ( from , to , cursor ) → page
* @return 本次同步统计
* @throws IllegalArgumentException 任一时间点为空 , 或 { @code from } 不早于 { @code to }
* @throws IllegalArgumentException 该账户 / 流已有水位线 ( 避免误把线上水位线回退 )
* /
public SyncResult bootstrapSalesOrders ( SyncSpec spec , Instant from , Instant to ,
PageFetcher < SalesOrder > fetch ) {
return bootstrap ( spec , from , to , fetch , new SinkStep < SalesOrder > ( ) {
@Override
public void upsert ( Page < SalesOrder > page ) {
sink . upsertSalesOrders ( page ) ;
}
} ) ;
}
/ * *
* 商品 / Listing 的首次全量建底 。
*
* @see # bootstrapSalesOrders ( SyncSpec , Instant , Instant , PageFetcher )
* /
public SyncResult bootstrapProducts ( SyncSpec spec , Instant from , Instant to ,
PageFetcher < ProductListing > fetch ) {
return bootstrap ( spec , from , to , fetch , new SinkStep < ProductListing > ( ) {
@Override
public void upsert ( Page < ProductListing > page ) {
sink . upsertProducts ( page ) ;
}
} ) ;
}
/** 落库动作;只为复用 bootstrap 的分页与水位线逻辑。 */
private interface SinkStep < T > {
void upsert ( Page < T > page ) ;
}
/** 建底通用流程:分页拉取 → 落库 → 成功后播种水位线。 */
private < T > SyncResult bootstrap ( SyncSpec spec , Instant from , Instant to ,
PageFetcher < T > fetch , SinkStep < T > sinkStep ) {
if ( from = = null | | to = = null ) {
throw new IllegalArgumentException ( "from and to must not be null" ) ;
}
if ( ! to . isAfter ( from ) ) {
throw new IllegalArgumentException ( "to (" + to + ") must be after from (" + from + ")" ) ;
}
Instant existing = cursorStore . get ( spec . provider ( ) . name ( ) , spec . accountKey ( ) , spec . stream ( ) ) ;
if ( existing ! = null ) {
throw new IllegalArgumentException ( "watermark already exists for " + spec . accountKey ( )
+ "/" + spec . stream ( ) + " (" + existing + "); bootstrap would roll it back" ) ;
}
int pages = 0 ;
int items = 0 ;
String cursor = null ;
while ( true ) {
Page < T > page = fetch . fetch ( from , to , cursor ) ;
if ( page = = null | | page . size ( ) = = 0 ) {
break ;
}
sinkStep . upsert ( page ) ;
pages + + ;
items + = page . size ( ) ;
cursor = page . nextCursor ( ) ;
if ( cursor = = null ) {
break ;
}
}
cursorStore . advance ( spec . provider ( ) . name ( ) , spec . accountKey ( ) , spec . stream ( ) , to ) ;
LOG . info ( "bootstrap {} pages={} items={} from={} to={}" , spec . stream ( ) , pages , items , from , to ) ;
return SyncResult . of ( pages , items , true ) ;
}
}