Browse Source

fix: 同步引擎查询窗口与水位线对齐,库存增量字段显式拒绝

1. SyncEngine 的查询窗口补上上界
   原来 fetch(from, cursor) 只传起点,水位线却推进到 to。窗口内尚未入账的
   数据会被永久跳过。改为 PageFetcher(from, to, cursor),查询与推进都限定在
   [from, to)。同时补 syncProducts。

2. 新增首次全量建底 API
   增量同步要求先有水位线,但此前没有任何方法能建底,只能手工 advance。
   新增 bootstrapSalesOrders / bootstrapProducts:分页拉全量 → 落库 →
   成功后播种水位线;已有水位线时拒绝执行,避免把线上水位线回退。

3. 库存 updatedAfter 四家仓统一显式拒绝
   恒健原本就拒绝;运去哪 / 易达云 / 信仓此前静默忽略该字段,调用方以为
   是按增量拉、实际拿到全量,容易误用。现统一抛 ValidationException,
   与各仓 inventorySyncStrategy()=FULL_PULL_DIFF 保持一致。

补 4 个同步测试:窗口上界、无水位线、to 早于起点、建底与重复建底拒绝。

Co-Authored-By: Claude <noreply@anthropic.com>
master
chenchuchuan 19 hours ago
parent
commit
a6bfba6fa9
  1. 166
      channel-sdk-core/src/main/java/common/ecommerce/channel/sync/SyncEngine.java
  2. 83
      sdk-testkit/src/test/java/common/ecommerce/shared/testkit/FakeDirectAdapterTest.java
  3. 10
      warehouse-sdk-edayun/src/main/java/common/ecommerce/edayun/adapter/EdayunAdapter.java
  4. 10
      warehouse-sdk-xincang/src/main/java/common/ecommerce/xincang/adapter/XincangAdapter.java
  5. 10
      warehouse-sdk-yqn/src/main/java/common/ecommerce/yqn/adapter/YqnAdapter.java

166
channel-sdk-core/src/main/java/common/ecommerce/channel/sync/SyncEngine.java

@ -4,26 +4,28 @@ import common.ecommerce.shared.support.Page; @@ -4,26 +4,28 @@ import common.ecommerce.shared.support.Page;
import common.ecommerce.shared.sync.SyncResult;
import common.ecommerce.shared.sync.SyncSpec;
import common.ecommerce.channel.model.order.SalesOrder;
import common.ecommerce.channel.model.order.SalesOrderQuery;
import common.ecommerce.channel.model.product.ProductListing;
import common.ecommerce.channel.spi.Sink;
import common.ecommerce.shared.spi.SyncCursorStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.time.Instant;
import java.util.function.BiFunction;
/**
* 增量同步引擎。
*
* <pre>
* from = lastWatermark - overlapWindow
* to = now
* loop 拉一页 → 经 Sink upsert → cursor = page.nextCursor
* to = 本次批次终点(宿主传入,不取系统时间)
* loop 拉一页([from, to)) → 经 Sink upsert → cursor = page.nextCursor
* 直到 cursor == null
* 全部成功才推进水位线,失败不推进
* </pre>
*
* <p><b>查询窗口与水位线一致</b>:拉取用 {@code [from, to)},推进也到 {@code to}。
* 若查询不带上界、水位线却推到 {@code to},窗口内尚未入账的数据会被永久跳过。</p>
*
* <p>引擎不持有线程或定时器,由宿主调度触发。</p>
*/
public final class SyncEngine {
@ -46,33 +48,49 @@ public final class SyncEngine { @@ -46,33 +48,49 @@ public final class SyncEngine {
}
/**
* 同步销售订单。
* 拉一页的函数。
*
* <p>{@code from} 与 {@code to} 构成左闭右开的时间窗;首页 {@code cursor} 为 null,
* 后续页原样传回上一页的 {@code nextCursor}。</p>
*/
public interface PageFetcher<T> {
Page<T> fetch(Instant from, Instant to, String cursor);
}
/**
* 增量同步销售订单。
*
* <p>整批成功才推进水位线;任何一页抛异常都会直接冒出,
* {@code SyncResult} 不会返回,水位线保持原值。引擎不私起线程/定时器。</p>
*
* @param spec 同步维度(provider + account + stream + overlap)
* @param to 本次增量终点;会作为新水位线写入,需非 null
* @param fetch 拉一页的函数:(from, cursor) → page;首页 {@code cursor} 为 null
* @param to 本次增量终点(不含);查询上界,同时作为新水位线
* @param fetch 拉一页:(from, to, cursor) → page
* @return 本次同步统计;成功路径恒为 {@code watermarkAdvanced=true}
* @throws IllegalStateException 账户/流还没有水位线(必须先跑一次全量建底)
* @throws RuntimeException {@code fetch} / {@code sink} 抛出的异常原样冒出
* @throws IllegalArgumentException {@code to} 为 null 或不晚于起点
* @throws IllegalStateException 账户/流还没有水位线(必须先跑 {@link #bootstrapSalesOrders})
* @throws RuntimeException {@code fetch} / {@code sink} 抛出的异常原样冒出
*/
public SyncResult syncSalesOrders(SyncSpec spec, Instant to,
BiFunction<Instant, String, Page<SalesOrder>> fetch) {
public SyncResult syncSalesOrders(SyncSpec spec, Instant to, PageFetcher<SalesOrder> fetch) {
if (to == null) {
throw new IllegalArgumentException("to must not be null");
}
Instant watermark = cursorStore.get(spec.provider().name(), spec.accountKey(), spec.stream());
Instant from = spec.from(watermark);
if (from == null) {
throw new IllegalStateException("no watermark for " + spec.accountKey()
+ "/" + spec.stream() + ", run full bootstrap first");
}
if (!to.isAfter(from)) {
throw new IllegalArgumentException("to (" + to + ") must be after from (" + from + ")");
}
int pages = 0;
int items = 0;
String cursor = null;
// 首页需要 from,后续页沿用同一 from(时间窗固定,靠游标翻页)
// 时间窗固定为 [from, to):翻页靠游标,窗口不变
while (true) {
Page<SalesOrder> page = fetch.apply(from, cursor);
Page<SalesOrder> page = fetch.fetch(from, to, cursor);
if (page == null || page.size() == 0) {
break;
}
@ -89,4 +107,126 @@ public final class SyncEngine { @@ -89,4 +107,126 @@ public final class SyncEngine {
LOG.debug("sync {} pages={} items={} from={} to={}", spec.stream(), pages, items, from, to);
return SyncResult.of(pages, items, true);
}
/**
* 增量同步商品 / Listing。
*
* @see #syncSalesOrders(SyncSpec, Instant, PageFetcher)
*/
public SyncResult syncProducts(SyncSpec spec, Instant to, PageFetcher<ProductListing> fetch) {
if (to == null) {
throw new IllegalArgumentException("to must not be null");
}
Instant watermark = cursorStore.get(spec.provider().name(), spec.accountKey(), spec.stream());
Instant from = spec.from(watermark);
if (from == null) {
throw new IllegalStateException("no watermark for " + spec.accountKey()
+ "/" + spec.stream() + ", run full bootstrap first");
}
if (!to.isAfter(from)) {
throw new IllegalArgumentException("to (" + to + ") must be after from (" + from + ")");
}
int pages = 0;
int items = 0;
String cursor = null;
while (true) {
Page<ProductListing> page = fetch.fetch(from, to, cursor);
if (page == null || page.size() == 0) {
break;
}
sink.upsertProducts(page);
pages++;
items += page.size();
cursor = page.nextCursor();
if (cursor == null) {
break;
}
}
cursorStore.advance(spec.provider().name(), spec.accountKey(), spec.stream(), to);
LOG.debug("sync {} pages={} items={} from={} to={}", spec.stream(), pages, items, from, to);
return SyncResult.of(pages, items, true);
}
/**
* 首次全量建底:拉完整个窗口并落库,然后把水位线播种到 {@code to}。
*
* <p>增量同步要求先有水位线;首次接入时用本方法把历史数据补一遍。
* 与增量一样,整批成功才写入水位线。</p>
*
* @param spec 同步维度
* @param from 全量起点(含)
* @param to 全量终点(不含);建成后的水位线
* @param fetch 拉一页:(from, to, cursor) → page
* @return 本次同步统计
* @throws IllegalArgumentException 任一时间点为空,或 {@code from} 不早于 {@code to}
* @throws IllegalArgumentException 该账户/流已有水位线(避免误把线上水位线回退)
*/
public SyncResult bootstrapSalesOrders(SyncSpec spec, Instant from, Instant to,
PageFetcher<SalesOrder> fetch) {
return bootstrap(spec, from, to, fetch, new SinkStep<SalesOrder>() {
@Override
public void upsert(Page<SalesOrder> page) {
sink.upsertSalesOrders(page);
}
});
}
/**
* 商品 / Listing 的首次全量建底。
*
* @see #bootstrapSalesOrders(SyncSpec, Instant, Instant, PageFetcher)
*/
public SyncResult bootstrapProducts(SyncSpec spec, Instant from, Instant to,
PageFetcher<ProductListing> fetch) {
return bootstrap(spec, from, to, fetch, new SinkStep<ProductListing>() {
@Override
public void upsert(Page<ProductListing> page) {
sink.upsertProducts(page);
}
});
}
/** 落库动作;只为复用 bootstrap 的分页与水位线逻辑。 */
private interface SinkStep<T> {
void upsert(Page<T> page);
}
/** 建底通用流程:分页拉取 → 落库 → 成功后播种水位线。 */
private <T> SyncResult bootstrap(SyncSpec spec, Instant from, Instant to,
PageFetcher<T> fetch, SinkStep<T> sinkStep) {
if (from == null || to == null) {
throw new IllegalArgumentException("from and to must not be null");
}
if (!to.isAfter(from)) {
throw new IllegalArgumentException("to (" + to + ") must be after from (" + from + ")");
}
Instant existing = cursorStore.get(spec.provider().name(), spec.accountKey(), spec.stream());
if (existing != null) {
throw new IllegalArgumentException("watermark already exists for " + spec.accountKey()
+ "/" + spec.stream() + " (" + existing + "); bootstrap would roll it back");
}
int pages = 0;
int items = 0;
String cursor = null;
while (true) {
Page<T> page = fetch.fetch(from, to, cursor);
if (page == null || page.size() == 0) {
break;
}
sinkStep.upsert(page);
pages++;
items += page.size();
cursor = page.nextCursor();
if (cursor == null) {
break;
}
}
cursorStore.advance(spec.provider().name(), spec.accountKey(), spec.stream(), to);
LOG.info("bootstrap {} pages={} items={} from={} to={}", spec.stream(), pages, items, from, to);
return SyncResult.of(pages, items, true);
}
}

83
sdk-testkit/src/test/java/common/ecommerce/shared/testkit/FakeDirectAdapterTest.java

@ -83,8 +83,8 @@ class FakeDirectAdapterTest { @@ -83,8 +83,8 @@ class FakeDirectAdapterTest {
SyncSpec spec = SyncSpec.of(ChannelProvider.DIRECT, "oms-test-001", stream, Duration.ofMinutes(15));
Instant to = Instant.parse("2026-10-29T00:00:00Z");
SyncResult result = new SyncEngine(cursors, sink).syncSalesOrders(spec, to,
(from, cursor) -> adapter.pull(ctx(), SalesOrderQuery.builder()
.updatedAfter(from).updatedBefore(to).cursor(cursor).build()));
(from, end, cursor) -> adapter.pull(ctx(), SalesOrderQuery.builder()
.updatedAfter(from).updatedBefore(end).cursor(cursor).build()));
assertEquals(3, result.pages());
assertEquals(5, result.items());
@ -94,6 +94,85 @@ class FakeDirectAdapterTest { @@ -94,6 +94,85 @@ class FakeDirectAdapterTest {
assertEquals(5, sink.salesOrders().size());
// 起点回退了重叠窗口
assertEquals(Instant.parse("2026-10-27T23:45:00Z"), adapter.orderQueries().get(0).updatedAfter());
// 查询上界必须等于推进的水位线,否则窗口内未入账的数据会被永久跳过
for (SalesOrderQuery q : adapter.orderQueries()) {
assertEquals(to, q.updatedBefore(), "每页查询上界都应等于 to");
}
}
@Test
void syncWithoutWatermarkFails() {
FakeDirectAdapter adapter = FakeDirectAdapter.builder().orders(2).build();
InMemoryCursorStore cursors = new InMemoryCursorStore(); // 未播种
SyncSpec spec = SyncSpec.of(ChannelProvider.DIRECT, "oms-test-001",
SyncSpec.STREAM_SALES_ORDERS, Duration.ofMinutes(15));
IllegalStateException ex = assertThrows(IllegalStateException.class,
() -> new SyncEngine(cursors, new InMemorySink()).syncSalesOrders(spec,
Instant.parse("2026-10-29T00:00:00Z"),
(from, end, cursor) -> adapter.pull(ctx(), SalesOrderQuery.builder()
.updatedAfter(from).updatedBefore(end).cursor(cursor).build())));
assertTrue(ex.getMessage().contains("bootstrap"), ex.getMessage());
}
@Test
void syncRejectsToNotAfterFrom() {
FakeDirectAdapter adapter = FakeDirectAdapter.builder().orders(2).build();
InMemoryCursorStore cursors = new InMemoryCursorStore();
// 水位线 10:00 + 15 分钟重叠回退 => 起点 09:45
cursors.seed(ChannelProvider.DIRECT.name(), "oms-test-001",
SyncSpec.STREAM_SALES_ORDERS, Instant.parse("2026-10-28T10:00:00Z"));
SyncSpec spec = SyncSpec.of(ChannelProvider.DIRECT, "oms-test-001",
SyncSpec.STREAM_SALES_ORDERS, Duration.ofMinutes(15));
// to 早于回退后的起点,属于调用方算错,明确拒绝
assertThrows(IllegalArgumentException.class,
() -> new SyncEngine(cursors, new InMemorySink()).syncSalesOrders(spec,
Instant.parse("2026-10-28T09:30:00Z"),
(from, end, cursor) -> adapter.pull(ctx(), SalesOrderQuery.builder()
.updatedAfter(from).updatedBefore(end).cursor(cursor).build())));
}
@Test
void bootstrapSeedsWatermarkAndSinksEverything() {
FakeDirectAdapter adapter = FakeDirectAdapter.builder().orders(5).pageSize(2).build();
InMemoryCursorStore cursors = new InMemoryCursorStore();
InMemorySink sink = new InMemorySink();
SyncSpec spec = SyncSpec.of(ChannelProvider.DIRECT, "oms-test-001",
SyncSpec.STREAM_SALES_ORDERS, Duration.ofMinutes(15));
Instant from = Instant.parse("2026-10-01T00:00:00Z");
Instant to = Instant.parse("2026-10-29T00:00:00Z");
SyncResult result = new SyncEngine(cursors, sink).bootstrapSalesOrders(spec, from, to,
(start, end, cursor) -> adapter.pull(ctx(), SalesOrderQuery.builder()
.updatedAfter(start).updatedBefore(end).cursor(cursor).build()));
assertEquals(3, result.pages());
assertEquals(5, result.items());
assertEquals(5, sink.salesOrders().size());
assertEquals(to, cursors.watermark(ChannelProvider.DIRECT.name(), "oms-test-001",
SyncSpec.STREAM_SALES_ORDERS));
// 建底不受 overlap 影响,直接用传入的 from
assertEquals(from, adapter.orderQueries().get(0).updatedAfter());
}
@Test
void bootstrapRefusesToOverwriteExistingWatermark() {
FakeDirectAdapter adapter = FakeDirectAdapter.builder().orders(3).build();
InMemoryCursorStore cursors = new InMemoryCursorStore();
cursors.seed(ChannelProvider.DIRECT.name(), "oms-test-001",
SyncSpec.STREAM_SALES_ORDERS, Instant.parse("2026-10-28T00:00:00Z"));
SyncSpec spec = SyncSpec.of(ChannelProvider.DIRECT, "oms-test-001",
SyncSpec.STREAM_SALES_ORDERS, Duration.ofMinutes(15));
assertThrows(IllegalArgumentException.class,
() -> new SyncEngine(cursors, new InMemorySink()).bootstrapSalesOrders(spec,
Instant.parse("2026-10-01T00:00:00Z"), Instant.parse("2026-10-29T00:00:00Z"),
(start, end, cursor) -> adapter.pull(ctx(), SalesOrderQuery.builder()
.updatedAfter(start).updatedBefore(end).cursor(cursor).build())));
// 原水位线没被改动
assertEquals(Instant.parse("2026-10-28T00:00:00Z"), cursors.watermark(
ChannelProvider.DIRECT.name(), "oms-test-001", SyncSpec.STREAM_SALES_ORDERS));
}
@Test

10
warehouse-sdk-edayun/src/main/java/common/ecommerce/edayun/adapter/EdayunAdapter.java

@ -607,6 +607,16 @@ public final class EdayunAdapter implements WarehouseAdapter, WarehouseProductCa @@ -607,6 +607,16 @@ public final class EdayunAdapter implements WarehouseAdapter, WarehouseProductCa
*/
@Override
public Page<InventorySnapshot> query(WarehouseContext ctx, InventoryQuery query) {
if (query.updatedAfter() != null) {
throw new common.ecommerce.shared.error.ValidationException(
WarehouseException.builder()
.message("edayun inventory has no updatedAfter filter; full-pull and diff instead")
.operation("warehouse.inventory.query")
.warehouseCode(ctx.warehouseCode())
.provider(provider().name())
.requestId(ctx.requestId())
.retryable(false));
}
int pageNum = EdayunInventoryMapper.pageNumber(query.cursor());
EdayunInventoryDtos.QueryInventoryRequest body =
EdayunInventoryMapper.toRequest(query, ctx.providerWarehouseId(), pageNum);

10
warehouse-sdk-xincang/src/main/java/common/ecommerce/xincang/adapter/XincangAdapter.java

@ -668,6 +668,16 @@ public final class XincangAdapter implements WarehouseProductCapable, OutboundCa @@ -668,6 +668,16 @@ public final class XincangAdapter implements WarehouseProductCapable, OutboundCa
*/
@Override
public Page<InventorySnapshot> query(WarehouseContext ctx, InventoryQuery query) {
if (query.updatedAfter() != null) {
throw new common.ecommerce.shared.error.ValidationException(
WarehouseException.builder()
.message("xincang inventory has no updatedAfter filter; full-pull and diff instead")
.operation("warehouse.inventory.query")
.warehouseCode(ctx.warehouseCode())
.provider(provider().name())
.requestId(ctx.requestId())
.retryable(false));
}
Map<String, Object> params = XincangInventoryMapper.toStockParams(query,
ctx.providerWarehouseId());
String raw = executeGet(ctx, XincangConstants.PATH_STOCK_LIST, params, login(ctx),

10
warehouse-sdk-yqn/src/main/java/common/ecommerce/yqn/adapter/YqnAdapter.java

@ -436,6 +436,16 @@ public final class YqnAdapter implements OutboundCapable, WarehouseProductCapabl @@ -436,6 +436,16 @@ public final class YqnAdapter implements OutboundCapable, WarehouseProductCapabl
*/
@Override
public Page<InventorySnapshot> query(WarehouseContext ctx, InventoryQuery query) {
if (query.updatedAfter() != null) {
throw new common.ecommerce.shared.error.ValidationException(
WarehouseException.builder()
.message("yqn inventory has no updatedAfter filter; full-pull and diff instead")
.operation("warehouse.inventory.query")
.warehouseCode(ctx.warehouseCode())
.provider(provider().name())
.requestId(ctx.requestId())
.retryable(false));
}
Map<String, List<String>> params = YqnInventoryMapper.toParams(query, ctx.providerWarehouseId());
rateLimiter.acquire(YqnConstants.PATH_INVENTORY_PAGE, YqnConstants.PERMITS_PER_SECOND);
HttpRequest req = requestFactory.getMulti(ctx.endpointBaseUrl(), YqnConstants.PATH_INVENTORY_PAGE,

Loading…
Cancel
Save