diff --git a/channel-sdk-core/src/main/java/common/ecommerce/channel/sync/SyncEngine.java b/channel-sdk-core/src/main/java/common/ecommerce/channel/sync/SyncEngine.java index 168646f..4a8ec44 100644 --- a/channel-sdk-core/src/main/java/common/ecommerce/channel/sync/SyncEngine.java +++ b/channel-sdk-core/src/main/java/common/ecommerce/channel/sync/SyncEngine.java @@ -4,26 +4,28 @@ import common.ecommerce.shared.support.Page; import common.ecommerce.shared.sync.SyncResult; import common.ecommerce.shared.sync.SyncSpec; import common.ecommerce.channel.model.order.SalesOrder; -import common.ecommerce.channel.model.order.SalesOrderQuery; +import common.ecommerce.channel.model.product.ProductListing; import common.ecommerce.channel.spi.Sink; import common.ecommerce.shared.spi.SyncCursorStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.time.Instant; -import java.util.function.BiFunction; /** * 增量同步引擎。 * *
  * from = lastWatermark - overlapWindow
- * to   = now
- * loop 拉一页 → 经 Sink upsert → cursor = page.nextCursor
+ * to   = 本次批次终点(宿主传入,不取系统时间)
+ * loop 拉一页([from, to)) → 经 Sink upsert → cursor = page.nextCursor
  * 直到 cursor == null
  * 全部成功才推进水位线,失败不推进
  * 
* + *

查询窗口与水位线一致:拉取用 {@code [from, to)},推进也到 {@code to}。 + * 若查询不带上界、水位线却推到 {@code to},窗口内尚未入账的数据会被永久跳过。

+ * *

引擎不持有线程或定时器,由宿主调度触发。

*/ public final class SyncEngine { @@ -46,33 +48,49 @@ public final class SyncEngine { } /** - * 同步销售订单。 + * 拉一页的函数。 + * + *

{@code from} 与 {@code to} 构成左闭右开的时间窗;首页 {@code cursor} 为 null, + * 后续页原样传回上一页的 {@code nextCursor}。

+ */ + public interface PageFetcher { + Page fetch(Instant from, Instant to, String cursor); + } + + /** + * 增量同步销售订单。 * *

整批成功才推进水位线;任何一页抛异常都会直接冒出, * {@code SyncResult} 不会返回,水位线保持原值。引擎不私起线程/定时器。

* * @param spec 同步维度(provider + account + stream + overlap) - * @param to 本次增量终点;会作为新水位线写入,需非 null - * @param fetch 拉一页的函数:(from, cursor) → page;首页 {@code cursor} 为 null + * @param to 本次增量终点(不含);查询上界,同时作为新水位线 + * @param fetch 拉一页:(from, to, cursor) → page * @return 本次同步统计;成功路径恒为 {@code watermarkAdvanced=true} - * @throws IllegalStateException 账户/流还没有水位线(必须先跑一次全量建底) - * @throws RuntimeException {@code fetch} / {@code sink} 抛出的异常原样冒出 + * @throws IllegalArgumentException {@code to} 为 null 或不晚于起点 + * @throws IllegalStateException 账户/流还没有水位线(必须先跑 {@link #bootstrapSalesOrders}) + * @throws RuntimeException {@code fetch} / {@code sink} 抛出的异常原样冒出 */ - public SyncResult syncSalesOrders(SyncSpec spec, Instant to, - BiFunction> fetch) { + public SyncResult syncSalesOrders(SyncSpec spec, Instant to, PageFetcher fetch) { + if (to == null) { + throw new IllegalArgumentException("to must not be null"); + } Instant watermark = cursorStore.get(spec.provider().name(), spec.accountKey(), spec.stream()); Instant from = spec.from(watermark); if (from == null) { throw new IllegalStateException("no watermark for " + spec.accountKey() + "/" + spec.stream() + ", run full bootstrap first"); } + if (!to.isAfter(from)) { + throw new IllegalArgumentException("to (" + to + ") must be after from (" + from + ")"); + } int pages = 0; int items = 0; String cursor = null; - // 首页需要 from,后续页沿用同一 from(时间窗固定,靠游标翻页) + // 时间窗固定为 [from, to):翻页靠游标,窗口不变 while (true) { - Page page = fetch.apply(from, cursor); + Page page = fetch.fetch(from, to, cursor); if (page == null || page.size() == 0) { break; } @@ -89,4 +107,126 @@ public final class SyncEngine { LOG.debug("sync {} pages={} items={} from={} to={}", spec.stream(), pages, items, from, to); return SyncResult.of(pages, items, true); } + + /** + * 增量同步商品 / Listing。 + * + * @see #syncSalesOrders(SyncSpec, Instant, PageFetcher) + */ + public SyncResult syncProducts(SyncSpec spec, Instant to, PageFetcher fetch) { + if (to == null) { + throw new IllegalArgumentException("to must not be null"); + } + Instant watermark = cursorStore.get(spec.provider().name(), spec.accountKey(), spec.stream()); + Instant from = spec.from(watermark); + if (from == null) { + throw new IllegalStateException("no watermark for " + spec.accountKey() + + "/" + spec.stream() + ", run full bootstrap first"); + } + if (!to.isAfter(from)) { + throw new IllegalArgumentException("to (" + to + ") must be after from (" + from + ")"); + } + + int pages = 0; + int items = 0; + String cursor = null; + while (true) { + Page page = fetch.fetch(from, to, cursor); + if (page == null || page.size() == 0) { + break; + } + sink.upsertProducts(page); + pages++; + items += page.size(); + cursor = page.nextCursor(); + if (cursor == null) { + break; + } + } + + cursorStore.advance(spec.provider().name(), spec.accountKey(), spec.stream(), to); + LOG.debug("sync {} pages={} items={} from={} to={}", spec.stream(), pages, items, from, to); + return SyncResult.of(pages, items, true); + } + + /** + * 首次全量建底:拉完整个窗口并落库,然后把水位线播种到 {@code to}。 + * + *

增量同步要求先有水位线;首次接入时用本方法把历史数据补一遍。 + * 与增量一样,整批成功才写入水位线。

+ * + * @param spec 同步维度 + * @param from 全量起点(含) + * @param to 全量终点(不含);建成后的水位线 + * @param fetch 拉一页:(from, to, cursor) → page + * @return 本次同步统计 + * @throws IllegalArgumentException 任一时间点为空,或 {@code from} 不早于 {@code to} + * @throws IllegalArgumentException 该账户/流已有水位线(避免误把线上水位线回退) + */ + public SyncResult bootstrapSalesOrders(SyncSpec spec, Instant from, Instant to, + PageFetcher fetch) { + return bootstrap(spec, from, to, fetch, new SinkStep() { + @Override + public void upsert(Page page) { + sink.upsertSalesOrders(page); + } + }); + } + + /** + * 商品 / Listing 的首次全量建底。 + * + * @see #bootstrapSalesOrders(SyncSpec, Instant, Instant, PageFetcher) + */ + public SyncResult bootstrapProducts(SyncSpec spec, Instant from, Instant to, + PageFetcher fetch) { + return bootstrap(spec, from, to, fetch, new SinkStep() { + @Override + public void upsert(Page page) { + sink.upsertProducts(page); + } + }); + } + + /** 落库动作;只为复用 bootstrap 的分页与水位线逻辑。 */ + private interface SinkStep { + void upsert(Page page); + } + + /** 建底通用流程:分页拉取 → 落库 → 成功后播种水位线。 */ + private SyncResult bootstrap(SyncSpec spec, Instant from, Instant to, + PageFetcher fetch, SinkStep sinkStep) { + if (from == null || to == null) { + throw new IllegalArgumentException("from and to must not be null"); + } + if (!to.isAfter(from)) { + throw new IllegalArgumentException("to (" + to + ") must be after from (" + from + ")"); + } + Instant existing = cursorStore.get(spec.provider().name(), spec.accountKey(), spec.stream()); + if (existing != null) { + throw new IllegalArgumentException("watermark already exists for " + spec.accountKey() + + "/" + spec.stream() + " (" + existing + "); bootstrap would roll it back"); + } + + int pages = 0; + int items = 0; + String cursor = null; + while (true) { + Page page = fetch.fetch(from, to, cursor); + if (page == null || page.size() == 0) { + break; + } + sinkStep.upsert(page); + pages++; + items += page.size(); + cursor = page.nextCursor(); + if (cursor == null) { + break; + } + } + + cursorStore.advance(spec.provider().name(), spec.accountKey(), spec.stream(), to); + LOG.info("bootstrap {} pages={} items={} from={} to={}", spec.stream(), pages, items, from, to); + return SyncResult.of(pages, items, true); + } } diff --git a/sdk-testkit/src/test/java/common/ecommerce/shared/testkit/FakeDirectAdapterTest.java b/sdk-testkit/src/test/java/common/ecommerce/shared/testkit/FakeDirectAdapterTest.java index bd29dfb..d52c8d7 100644 --- a/sdk-testkit/src/test/java/common/ecommerce/shared/testkit/FakeDirectAdapterTest.java +++ b/sdk-testkit/src/test/java/common/ecommerce/shared/testkit/FakeDirectAdapterTest.java @@ -83,8 +83,8 @@ class FakeDirectAdapterTest { SyncSpec spec = SyncSpec.of(ChannelProvider.DIRECT, "oms-test-001", stream, Duration.ofMinutes(15)); Instant to = Instant.parse("2026-10-29T00:00:00Z"); SyncResult result = new SyncEngine(cursors, sink).syncSalesOrders(spec, to, - (from, cursor) -> adapter.pull(ctx(), SalesOrderQuery.builder() - .updatedAfter(from).updatedBefore(to).cursor(cursor).build())); + (from, end, cursor) -> adapter.pull(ctx(), SalesOrderQuery.builder() + .updatedAfter(from).updatedBefore(end).cursor(cursor).build())); assertEquals(3, result.pages()); assertEquals(5, result.items()); @@ -94,6 +94,85 @@ class FakeDirectAdapterTest { assertEquals(5, sink.salesOrders().size()); // 起点回退了重叠窗口 assertEquals(Instant.parse("2026-10-27T23:45:00Z"), adapter.orderQueries().get(0).updatedAfter()); + // 查询上界必须等于推进的水位线,否则窗口内未入账的数据会被永久跳过 + for (SalesOrderQuery q : adapter.orderQueries()) { + assertEquals(to, q.updatedBefore(), "每页查询上界都应等于 to"); + } + } + + @Test + void syncWithoutWatermarkFails() { + FakeDirectAdapter adapter = FakeDirectAdapter.builder().orders(2).build(); + InMemoryCursorStore cursors = new InMemoryCursorStore(); // 未播种 + SyncSpec spec = SyncSpec.of(ChannelProvider.DIRECT, "oms-test-001", + SyncSpec.STREAM_SALES_ORDERS, Duration.ofMinutes(15)); + + IllegalStateException ex = assertThrows(IllegalStateException.class, + () -> new SyncEngine(cursors, new InMemorySink()).syncSalesOrders(spec, + Instant.parse("2026-10-29T00:00:00Z"), + (from, end, cursor) -> adapter.pull(ctx(), SalesOrderQuery.builder() + .updatedAfter(from).updatedBefore(end).cursor(cursor).build()))); + assertTrue(ex.getMessage().contains("bootstrap"), ex.getMessage()); + } + + @Test + void syncRejectsToNotAfterFrom() { + FakeDirectAdapter adapter = FakeDirectAdapter.builder().orders(2).build(); + InMemoryCursorStore cursors = new InMemoryCursorStore(); + // 水位线 10:00 + 15 分钟重叠回退 => 起点 09:45 + cursors.seed(ChannelProvider.DIRECT.name(), "oms-test-001", + SyncSpec.STREAM_SALES_ORDERS, Instant.parse("2026-10-28T10:00:00Z")); + SyncSpec spec = SyncSpec.of(ChannelProvider.DIRECT, "oms-test-001", + SyncSpec.STREAM_SALES_ORDERS, Duration.ofMinutes(15)); + + // to 早于回退后的起点,属于调用方算错,明确拒绝 + assertThrows(IllegalArgumentException.class, + () -> new SyncEngine(cursors, new InMemorySink()).syncSalesOrders(spec, + Instant.parse("2026-10-28T09:30:00Z"), + (from, end, cursor) -> adapter.pull(ctx(), SalesOrderQuery.builder() + .updatedAfter(from).updatedBefore(end).cursor(cursor).build()))); + } + + @Test + void bootstrapSeedsWatermarkAndSinksEverything() { + FakeDirectAdapter adapter = FakeDirectAdapter.builder().orders(5).pageSize(2).build(); + InMemoryCursorStore cursors = new InMemoryCursorStore(); + InMemorySink sink = new InMemorySink(); + SyncSpec spec = SyncSpec.of(ChannelProvider.DIRECT, "oms-test-001", + SyncSpec.STREAM_SALES_ORDERS, Duration.ofMinutes(15)); + + Instant from = Instant.parse("2026-10-01T00:00:00Z"); + Instant to = Instant.parse("2026-10-29T00:00:00Z"); + SyncResult result = new SyncEngine(cursors, sink).bootstrapSalesOrders(spec, from, to, + (start, end, cursor) -> adapter.pull(ctx(), SalesOrderQuery.builder() + .updatedAfter(start).updatedBefore(end).cursor(cursor).build())); + + assertEquals(3, result.pages()); + assertEquals(5, result.items()); + assertEquals(5, sink.salesOrders().size()); + assertEquals(to, cursors.watermark(ChannelProvider.DIRECT.name(), "oms-test-001", + SyncSpec.STREAM_SALES_ORDERS)); + // 建底不受 overlap 影响,直接用传入的 from + assertEquals(from, adapter.orderQueries().get(0).updatedAfter()); + } + + @Test + void bootstrapRefusesToOverwriteExistingWatermark() { + FakeDirectAdapter adapter = FakeDirectAdapter.builder().orders(3).build(); + InMemoryCursorStore cursors = new InMemoryCursorStore(); + cursors.seed(ChannelProvider.DIRECT.name(), "oms-test-001", + SyncSpec.STREAM_SALES_ORDERS, Instant.parse("2026-10-28T00:00:00Z")); + SyncSpec spec = SyncSpec.of(ChannelProvider.DIRECT, "oms-test-001", + SyncSpec.STREAM_SALES_ORDERS, Duration.ofMinutes(15)); + + assertThrows(IllegalArgumentException.class, + () -> new SyncEngine(cursors, new InMemorySink()).bootstrapSalesOrders(spec, + Instant.parse("2026-10-01T00:00:00Z"), Instant.parse("2026-10-29T00:00:00Z"), + (start, end, cursor) -> adapter.pull(ctx(), SalesOrderQuery.builder() + .updatedAfter(start).updatedBefore(end).cursor(cursor).build()))); + // 原水位线没被改动 + assertEquals(Instant.parse("2026-10-28T00:00:00Z"), cursors.watermark( + ChannelProvider.DIRECT.name(), "oms-test-001", SyncSpec.STREAM_SALES_ORDERS)); } @Test diff --git a/warehouse-sdk-edayun/src/main/java/common/ecommerce/edayun/adapter/EdayunAdapter.java b/warehouse-sdk-edayun/src/main/java/common/ecommerce/edayun/adapter/EdayunAdapter.java index 6272a6f..2ae67a4 100644 --- a/warehouse-sdk-edayun/src/main/java/common/ecommerce/edayun/adapter/EdayunAdapter.java +++ b/warehouse-sdk-edayun/src/main/java/common/ecommerce/edayun/adapter/EdayunAdapter.java @@ -607,6 +607,16 @@ public final class EdayunAdapter implements WarehouseAdapter, WarehouseProductCa */ @Override public Page query(WarehouseContext ctx, InventoryQuery query) { + if (query.updatedAfter() != null) { + throw new common.ecommerce.shared.error.ValidationException( + WarehouseException.builder() + .message("edayun inventory has no updatedAfter filter; full-pull and diff instead") + .operation("warehouse.inventory.query") + .warehouseCode(ctx.warehouseCode()) + .provider(provider().name()) + .requestId(ctx.requestId()) + .retryable(false)); + } int pageNum = EdayunInventoryMapper.pageNumber(query.cursor()); EdayunInventoryDtos.QueryInventoryRequest body = EdayunInventoryMapper.toRequest(query, ctx.providerWarehouseId(), pageNum); diff --git a/warehouse-sdk-xincang/src/main/java/common/ecommerce/xincang/adapter/XincangAdapter.java b/warehouse-sdk-xincang/src/main/java/common/ecommerce/xincang/adapter/XincangAdapter.java index e386c46..b233717 100644 --- a/warehouse-sdk-xincang/src/main/java/common/ecommerce/xincang/adapter/XincangAdapter.java +++ b/warehouse-sdk-xincang/src/main/java/common/ecommerce/xincang/adapter/XincangAdapter.java @@ -668,6 +668,16 @@ public final class XincangAdapter implements WarehouseProductCapable, OutboundCa */ @Override public Page query(WarehouseContext ctx, InventoryQuery query) { + if (query.updatedAfter() != null) { + throw new common.ecommerce.shared.error.ValidationException( + WarehouseException.builder() + .message("xincang inventory has no updatedAfter filter; full-pull and diff instead") + .operation("warehouse.inventory.query") + .warehouseCode(ctx.warehouseCode()) + .provider(provider().name()) + .requestId(ctx.requestId()) + .retryable(false)); + } Map params = XincangInventoryMapper.toStockParams(query, ctx.providerWarehouseId()); String raw = executeGet(ctx, XincangConstants.PATH_STOCK_LIST, params, login(ctx), diff --git a/warehouse-sdk-yqn/src/main/java/common/ecommerce/yqn/adapter/YqnAdapter.java b/warehouse-sdk-yqn/src/main/java/common/ecommerce/yqn/adapter/YqnAdapter.java index cb5c37d..01b3699 100644 --- a/warehouse-sdk-yqn/src/main/java/common/ecommerce/yqn/adapter/YqnAdapter.java +++ b/warehouse-sdk-yqn/src/main/java/common/ecommerce/yqn/adapter/YqnAdapter.java @@ -436,6 +436,16 @@ public final class YqnAdapter implements OutboundCapable, WarehouseProductCapabl */ @Override public Page query(WarehouseContext ctx, InventoryQuery query) { + if (query.updatedAfter() != null) { + throw new common.ecommerce.shared.error.ValidationException( + WarehouseException.builder() + .message("yqn inventory has no updatedAfter filter; full-pull and diff instead") + .operation("warehouse.inventory.query") + .warehouseCode(ctx.warehouseCode()) + .provider(provider().name()) + .requestId(ctx.requestId()) + .retryable(false)); + } Map> params = YqnInventoryMapper.toParams(query, ctx.providerWarehouseId()); rateLimiter.acquire(YqnConstants.PATH_INVENTORY_PAGE, YqnConstants.PERMITS_PER_SECOND); HttpRequest req = requestFactory.getMulti(ctx.endpointBaseUrl(), YqnConstants.PATH_INVENTORY_PAGE,