|
|
|
@ -4,26 +4,28 @@ import common.ecommerce.shared.support.Page; |
|
|
|
import common.ecommerce.shared.sync.SyncResult; |
|
|
|
import common.ecommerce.shared.sync.SyncResult; |
|
|
|
import common.ecommerce.shared.sync.SyncSpec; |
|
|
|
import common.ecommerce.shared.sync.SyncSpec; |
|
|
|
import common.ecommerce.channel.model.order.SalesOrder; |
|
|
|
import common.ecommerce.channel.model.order.SalesOrder; |
|
|
|
import common.ecommerce.channel.model.order.SalesOrderQuery; |
|
|
|
import common.ecommerce.channel.model.product.ProductListing; |
|
|
|
import common.ecommerce.channel.spi.Sink; |
|
|
|
import common.ecommerce.channel.spi.Sink; |
|
|
|
import common.ecommerce.shared.spi.SyncCursorStore; |
|
|
|
import common.ecommerce.shared.spi.SyncCursorStore; |
|
|
|
import org.slf4j.Logger; |
|
|
|
import org.slf4j.Logger; |
|
|
|
import org.slf4j.LoggerFactory; |
|
|
|
import org.slf4j.LoggerFactory; |
|
|
|
|
|
|
|
|
|
|
|
import java.time.Instant; |
|
|
|
import java.time.Instant; |
|
|
|
import java.util.function.BiFunction; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
|
* 增量同步引擎。 |
|
|
|
* 增量同步引擎。 |
|
|
|
* |
|
|
|
* |
|
|
|
* <pre> |
|
|
|
* <pre> |
|
|
|
* from = lastWatermark - overlapWindow |
|
|
|
* from = lastWatermark - overlapWindow |
|
|
|
* to = now |
|
|
|
* to = 本次批次终点(宿主传入,不取系统时间) |
|
|
|
* loop 拉一页 → 经 Sink upsert → cursor = page.nextCursor |
|
|
|
* loop 拉一页([from, to)) → 经 Sink upsert → cursor = page.nextCursor |
|
|
|
* 直到 cursor == null |
|
|
|
* 直到 cursor == null |
|
|
|
* 全部成功才推进水位线,失败不推进 |
|
|
|
* 全部成功才推进水位线,失败不推进 |
|
|
|
* </pre> |
|
|
|
* </pre> |
|
|
|
* |
|
|
|
* |
|
|
|
|
|
|
|
* <p><b>查询窗口与水位线一致</b>:拉取用 {@code [from, to)},推进也到 {@code to}。 |
|
|
|
|
|
|
|
* 若查询不带上界、水位线却推到 {@code to},窗口内尚未入账的数据会被永久跳过。</p> |
|
|
|
|
|
|
|
* |
|
|
|
* <p>引擎不持有线程或定时器,由宿主调度触发。</p> |
|
|
|
* <p>引擎不持有线程或定时器,由宿主调度触发。</p> |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
public final class SyncEngine { |
|
|
|
public final class SyncEngine { |
|
|
|
@ -46,33 +48,49 @@ public final class SyncEngine { |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
|
* 同步销售订单。 |
|
|
|
* 拉一页的函数。 |
|
|
|
|
|
|
|
* |
|
|
|
|
|
|
|
* <p>{@code from} 与 {@code to} 构成左闭右开的时间窗;首页 {@code cursor} 为 null, |
|
|
|
|
|
|
|
* 后续页原样传回上一页的 {@code nextCursor}。</p> |
|
|
|
|
|
|
|
*/ |
|
|
|
|
|
|
|
public interface PageFetcher<T> { |
|
|
|
|
|
|
|
Page<T> fetch(Instant from, Instant to, String cursor); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
|
|
* 增量同步销售订单。 |
|
|
|
* |
|
|
|
* |
|
|
|
* <p>整批成功才推进水位线;任何一页抛异常都会直接冒出, |
|
|
|
* <p>整批成功才推进水位线;任何一页抛异常都会直接冒出, |
|
|
|
* {@code SyncResult} 不会返回,水位线保持原值。引擎不私起线程/定时器。</p> |
|
|
|
* {@code SyncResult} 不会返回,水位线保持原值。引擎不私起线程/定时器。</p> |
|
|
|
* |
|
|
|
* |
|
|
|
* @param spec 同步维度(provider + account + stream + overlap) |
|
|
|
* @param spec 同步维度(provider + account + stream + overlap) |
|
|
|
* @param to 本次增量终点;会作为新水位线写入,需非 null |
|
|
|
* @param to 本次增量终点(不含);查询上界,同时作为新水位线 |
|
|
|
* @param fetch 拉一页的函数:(from, cursor) → page;首页 {@code cursor} 为 null |
|
|
|
* @param fetch 拉一页:(from, to, cursor) → page |
|
|
|
* @return 本次同步统计;成功路径恒为 {@code watermarkAdvanced=true} |
|
|
|
* @return 本次同步统计;成功路径恒为 {@code watermarkAdvanced=true} |
|
|
|
* @throws IllegalStateException 账户/流还没有水位线(必须先跑一次全量建底) |
|
|
|
* @throws IllegalArgumentException {@code to} 为 null 或不晚于起点 |
|
|
|
|
|
|
|
* @throws IllegalStateException 账户/流还没有水位线(必须先跑 {@link #bootstrapSalesOrders}) |
|
|
|
* @throws RuntimeException {@code fetch} / {@code sink} 抛出的异常原样冒出 |
|
|
|
* @throws RuntimeException {@code fetch} / {@code sink} 抛出的异常原样冒出 |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
public SyncResult syncSalesOrders(SyncSpec spec, Instant to, |
|
|
|
public SyncResult syncSalesOrders(SyncSpec spec, Instant to, PageFetcher<SalesOrder> fetch) { |
|
|
|
BiFunction<Instant, String, Page<SalesOrder>> fetch) { |
|
|
|
if (to == null) { |
|
|
|
|
|
|
|
throw new IllegalArgumentException("to must not be null"); |
|
|
|
|
|
|
|
} |
|
|
|
Instant watermark = cursorStore.get(spec.provider().name(), spec.accountKey(), spec.stream()); |
|
|
|
Instant watermark = cursorStore.get(spec.provider().name(), spec.accountKey(), spec.stream()); |
|
|
|
Instant from = spec.from(watermark); |
|
|
|
Instant from = spec.from(watermark); |
|
|
|
if (from == null) { |
|
|
|
if (from == null) { |
|
|
|
throw new IllegalStateException("no watermark for " + spec.accountKey() |
|
|
|
throw new IllegalStateException("no watermark for " + spec.accountKey() |
|
|
|
+ "/" + spec.stream() + ", run full bootstrap first"); |
|
|
|
+ "/" + spec.stream() + ", run full bootstrap first"); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
if (!to.isAfter(from)) { |
|
|
|
|
|
|
|
throw new IllegalArgumentException("to (" + to + ") must be after from (" + from + ")"); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
int pages = 0; |
|
|
|
int pages = 0; |
|
|
|
int items = 0; |
|
|
|
int items = 0; |
|
|
|
String cursor = null; |
|
|
|
String cursor = null; |
|
|
|
// 首页需要 from,后续页沿用同一 from(时间窗固定,靠游标翻页)
|
|
|
|
// 时间窗固定为 [from, to):翻页靠游标,窗口不变
|
|
|
|
while (true) { |
|
|
|
while (true) { |
|
|
|
Page<SalesOrder> page = fetch.apply(from, cursor); |
|
|
|
Page<SalesOrder> page = fetch.fetch(from, to, cursor); |
|
|
|
if (page == null || page.size() == 0) { |
|
|
|
if (page == null || page.size() == 0) { |
|
|
|
break; |
|
|
|
break; |
|
|
|
} |
|
|
|
} |
|
|
|
@ -89,4 +107,126 @@ public final class SyncEngine { |
|
|
|
LOG.debug("sync {} pages={} items={} from={} to={}", spec.stream(), pages, items, from, to); |
|
|
|
LOG.debug("sync {} pages={} items={} from={} to={}", spec.stream(), pages, items, from, to); |
|
|
|
return SyncResult.of(pages, items, true); |
|
|
|
return SyncResult.of(pages, items, true); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
|
|
* 增量同步商品 / Listing。 |
|
|
|
|
|
|
|
* |
|
|
|
|
|
|
|
* @see #syncSalesOrders(SyncSpec, Instant, PageFetcher) |
|
|
|
|
|
|
|
*/ |
|
|
|
|
|
|
|
public SyncResult syncProducts(SyncSpec spec, Instant to, PageFetcher<ProductListing> fetch) { |
|
|
|
|
|
|
|
if (to == null) { |
|
|
|
|
|
|
|
throw new IllegalArgumentException("to must not be null"); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
Instant watermark = cursorStore.get(spec.provider().name(), spec.accountKey(), spec.stream()); |
|
|
|
|
|
|
|
Instant from = spec.from(watermark); |
|
|
|
|
|
|
|
if (from == null) { |
|
|
|
|
|
|
|
throw new IllegalStateException("no watermark for " + spec.accountKey() |
|
|
|
|
|
|
|
+ "/" + spec.stream() + ", run full bootstrap first"); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
if (!to.isAfter(from)) { |
|
|
|
|
|
|
|
throw new IllegalArgumentException("to (" + to + ") must be after from (" + from + ")"); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
int pages = 0; |
|
|
|
|
|
|
|
int items = 0; |
|
|
|
|
|
|
|
String cursor = null; |
|
|
|
|
|
|
|
while (true) { |
|
|
|
|
|
|
|
Page<ProductListing> page = fetch.fetch(from, to, cursor); |
|
|
|
|
|
|
|
if (page == null || page.size() == 0) { |
|
|
|
|
|
|
|
break; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
sink.upsertProducts(page); |
|
|
|
|
|
|
|
pages++; |
|
|
|
|
|
|
|
items += page.size(); |
|
|
|
|
|
|
|
cursor = page.nextCursor(); |
|
|
|
|
|
|
|
if (cursor == null) { |
|
|
|
|
|
|
|
break; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
cursorStore.advance(spec.provider().name(), spec.accountKey(), spec.stream(), to); |
|
|
|
|
|
|
|
LOG.debug("sync {} pages={} items={} from={} to={}", spec.stream(), pages, items, from, to); |
|
|
|
|
|
|
|
return SyncResult.of(pages, items, true); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
|
|
* 首次全量建底:拉完整个窗口并落库,然后把水位线播种到 {@code to}。 |
|
|
|
|
|
|
|
* |
|
|
|
|
|
|
|
* <p>增量同步要求先有水位线;首次接入时用本方法把历史数据补一遍。 |
|
|
|
|
|
|
|
* 与增量一样,整批成功才写入水位线。</p> |
|
|
|
|
|
|
|
* |
|
|
|
|
|
|
|
* @param spec 同步维度 |
|
|
|
|
|
|
|
* @param from 全量起点(含) |
|
|
|
|
|
|
|
* @param to 全量终点(不含);建成后的水位线 |
|
|
|
|
|
|
|
* @param fetch 拉一页:(from, to, cursor) → page |
|
|
|
|
|
|
|
* @return 本次同步统计 |
|
|
|
|
|
|
|
* @throws IllegalArgumentException 任一时间点为空,或 {@code from} 不早于 {@code to} |
|
|
|
|
|
|
|
* @throws IllegalArgumentException 该账户/流已有水位线(避免误把线上水位线回退) |
|
|
|
|
|
|
|
*/ |
|
|
|
|
|
|
|
public SyncResult bootstrapSalesOrders(SyncSpec spec, Instant from, Instant to, |
|
|
|
|
|
|
|
PageFetcher<SalesOrder> fetch) { |
|
|
|
|
|
|
|
return bootstrap(spec, from, to, fetch, new SinkStep<SalesOrder>() { |
|
|
|
|
|
|
|
@Override |
|
|
|
|
|
|
|
public void upsert(Page<SalesOrder> page) { |
|
|
|
|
|
|
|
sink.upsertSalesOrders(page); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
}); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
|
|
* 商品 / Listing 的首次全量建底。 |
|
|
|
|
|
|
|
* |
|
|
|
|
|
|
|
* @see #bootstrapSalesOrders(SyncSpec, Instant, Instant, PageFetcher) |
|
|
|
|
|
|
|
*/ |
|
|
|
|
|
|
|
public SyncResult bootstrapProducts(SyncSpec spec, Instant from, Instant to, |
|
|
|
|
|
|
|
PageFetcher<ProductListing> fetch) { |
|
|
|
|
|
|
|
return bootstrap(spec, from, to, fetch, new SinkStep<ProductListing>() { |
|
|
|
|
|
|
|
@Override |
|
|
|
|
|
|
|
public void upsert(Page<ProductListing> page) { |
|
|
|
|
|
|
|
sink.upsertProducts(page); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
}); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** 落库动作;只为复用 bootstrap 的分页与水位线逻辑。 */ |
|
|
|
|
|
|
|
private interface SinkStep<T> { |
|
|
|
|
|
|
|
void upsert(Page<T> page); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** 建底通用流程:分页拉取 → 落库 → 成功后播种水位线。 */ |
|
|
|
|
|
|
|
private <T> SyncResult bootstrap(SyncSpec spec, Instant from, Instant to, |
|
|
|
|
|
|
|
PageFetcher<T> fetch, SinkStep<T> sinkStep) { |
|
|
|
|
|
|
|
if (from == null || to == null) { |
|
|
|
|
|
|
|
throw new IllegalArgumentException("from and to must not be null"); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
if (!to.isAfter(from)) { |
|
|
|
|
|
|
|
throw new IllegalArgumentException("to (" + to + ") must be after from (" + from + ")"); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
Instant existing = cursorStore.get(spec.provider().name(), spec.accountKey(), spec.stream()); |
|
|
|
|
|
|
|
if (existing != null) { |
|
|
|
|
|
|
|
throw new IllegalArgumentException("watermark already exists for " + spec.accountKey() |
|
|
|
|
|
|
|
+ "/" + spec.stream() + " (" + existing + "); bootstrap would roll it back"); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
int pages = 0; |
|
|
|
|
|
|
|
int items = 0; |
|
|
|
|
|
|
|
String cursor = null; |
|
|
|
|
|
|
|
while (true) { |
|
|
|
|
|
|
|
Page<T> page = fetch.fetch(from, to, cursor); |
|
|
|
|
|
|
|
if (page == null || page.size() == 0) { |
|
|
|
|
|
|
|
break; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
sinkStep.upsert(page); |
|
|
|
|
|
|
|
pages++; |
|
|
|
|
|
|
|
items += page.size(); |
|
|
|
|
|
|
|
cursor = page.nextCursor(); |
|
|
|
|
|
|
|
if (cursor == null) { |
|
|
|
|
|
|
|
break; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
cursorStore.advance(spec.provider().name(), spec.accountKey(), spec.stream(), to); |
|
|
|
|
|
|
|
LOG.info("bootstrap {} pages={} items={} from={} to={}", spec.stream(), pages, items, from, to); |
|
|
|
|
|
|
|
return SyncResult.of(pages, items, true); |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|