This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch 
gh-readonly-queue/dev/pr-13127-df59bb4cd1c1fef7f2c38ade6456b8c0c2658da8
in repository https://gitbox.apache.org/repos/asf/seatunnel.git

commit 1bff1866f702dc24fe56b6030b37d971df400b23
Author: Doyeon Kim <[email protected]>
AuthorDate: Fri Nov 6 13:49:26 2026 +0000

    [Fix][Connector-V2] Fix BigQuery sink stream initialization (#13127)
---
 .../bigquery/sink/AbstractBigQuerySinkWriter.java  |  45 ++-
 .../bigquery/sink/BigQuerySinkBatchWriter.java     |  22 +-
 .../bigquery/sink/BigQuerySinkStreamWriter.java    |  10 +-
 .../bigquery/sink/committer/BigQueryCommitter.java |  29 +-
 .../bigquery/sink/writer/BigQueryBatchWriter.java  |  54 ++++
 .../sink/writer/BigQueryWriteStreamUtil.java       |  94 ++++++
 .../bigquery/sink/BigQuerySinkBatchWriterTest.java | 305 +++++++++++++++++++
 .../e2e/connector/bigquery/BigQueryCDCRealIT.java  | 323 ++++++++++++++++++++-
 .../fake_to_bigquery_batch_real_restore.conf       |  51 ++++
 .../e2e/common/container/TestContainer.java        |  16 +
 .../container/seatunnel/SeaTunnelContainer.java    |  14 +-
 11 files changed, 911 insertions(+), 52 deletions(-)

diff --git 
a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/AbstractBigQuerySinkWriter.java
 
b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/AbstractBigQuerySinkWriter.java
index f73d8ff048..5d62d9dba8 100644
--- 
a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/AbstractBigQuerySinkWriter.java
+++ 
b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/AbstractBigQuerySinkWriter.java
@@ -63,11 +63,14 @@ public abstract class AbstractBigQuerySinkWriter
 
     protected final ReadonlyConfig config;
     protected BigQuerySerializer serializer;
-    // Mutable: assigned eagerly by the constructor, or lazily by 
setMultiTableResourceManager()
-    // once the shared client is injected by the multi-table sink runtime.
+    // Mutable: assigned by the constructor, by setMultiTableResourceManager() 
when the multi-table
+    // sink runtime injects the shared client, or by ensureStreamWriter() on 
first use otherwise.
     protected BigQueryWriteClient client;
     protected BigQueryWriter streamWriter;
     protected TableSchema tableSchema;
+    // True only for a client created by ensureStreamWriter(), which close() 
must release. A client
+    // injected by the multi-table runtime is shared with sibling writers and 
must stay open.
+    private boolean ownsClient;
 
     protected final int batchSize;
     protected JSONArray buffer = new JSONArray();
@@ -106,8 +109,36 @@ public abstract class AbstractBigQuerySinkWriter
         return new 
BigQueryMultiTableResourceManager(BigQueryClientFactory.getWriteClient(config));
     }
 
+    @Override
+    public void setMultiTableResourceManager(
+            MultiTableResourceManager<BigQueryWriteClient> manager, int 
queueIndex) {
+        log.info("Injecting shared client and initializing BigQuery stream 
writer...");
+        this.client = manager.getSharedResource().get();
+        ensureStreamWriter();
+    }
+
+    /**
+     * Prepares the stream writer if it does not exist yet. A sink with 
several inputs is not
+     * wrapped in the multi-table sink, so {@link 
#setMultiTableResourceManager} is never called and
+     * the writer has to open its own client on first use.
+     */
+    protected void ensureStreamWriter() {
+        if (streamWriter != null) {
+            return;
+        }
+        if (client == null) {
+            client = BigQueryClientFactory.getWriteClient(config);
+            ownsClient = true;
+        }
+        streamWriter = openStreamWriter();
+    }
+
+    /** Opens the stream this writer appends to, using {@link #client}. */
+    protected abstract BigQueryWriter openStreamWriter();
+
     protected void flush() {
-        if (streamWriter == null || buffer.length() == 0) return;
+        if (buffer.length() == 0) return;
+        ensureStreamWriter();
 
         JSONArray dataToSend = buffer;
         buffer = new JSONArray();
@@ -257,6 +288,7 @@ public abstract class AbstractBigQuerySinkWriter
         BigQuerySerializer evolvedSerializer =
                 new 
BigQuerySerializer(evolvedTableSchema.toPhysicalRowDataType(), config);
 
+        ensureStreamWriter();
         flush();
         getSchemaChangeManager().applySchemaChange(event);
         BigQueryWriter evolvedWriter = streamWriter.refreshSchema(client, 
config);
@@ -291,6 +323,13 @@ public abstract class AbstractBigQuerySinkWriter
             // The shared BigQueryWriteClient is owned by 
BigQueryMultiTableResourceManager and is
             // closed once for the whole multi-table sink writer group; 
closing it per-writer here
             // would break sibling replica writers still using it.
+            if (ownsClient) {
+                try {
+                    client.close();
+                } catch (Exception e) {
+                    log.warn("Failed to close BigQueryWriteClient", e);
+                }
+            }
         }
     }
 }
diff --git 
a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkBatchWriter.java
 
b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkBatchWriter.java
index 62be99119f..88da49eb9a 100644
--- 
a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkBatchWriter.java
+++ 
b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkBatchWriter.java
@@ -18,7 +18,6 @@
 package org.apache.seatunnel.connectors.bigquery.sink;
 
 import org.apache.seatunnel.api.configuration.ReadonlyConfig;
-import org.apache.seatunnel.api.sink.MultiTableResourceManager;
 import org.apache.seatunnel.api.table.catalog.TableSchema;
 import org.apache.seatunnel.api.table.type.SeaTunnelRow;
 import org.apache.seatunnel.connectors.bigquery.convert.BigQuerySerializer;
@@ -33,7 +32,6 @@ import org.json.JSONArray;
 import com.google.api.gax.rpc.ApiException;
 import com.google.api.gax.rpc.StatusCode;
 import com.google.cloud.bigquery.storage.v1.AppendRowsResponse;
-import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient;
 import com.google.rpc.Code;
 import lombok.extern.slf4j.Slf4j;
 
@@ -74,17 +72,14 @@ public class BigQuerySinkBatchWriter extends 
AbstractBigQuerySinkWriter {
     }
 
     @Override
-    public void setMultiTableResourceManager(
-            MultiTableResourceManager<BigQueryWriteClient> manager, int 
queueIndex) {
-        log.info("Injecting shared client and initializing Batch stream 
writer...");
-        this.client = manager.getSharedResource().get();
-        if (restoredStreamName != null) {
-            this.streamWriter =
-                    BigQueryBatchWriter.restore(
-                            client, config, restoredStreamName, 
restoredNextOffset);
-        } else {
-            this.streamWriter = BigQueryBatchWriter.of(client, config);
+    protected BigQueryWriter openStreamWriter() {
+        if (restoredStreamName != null && !restoredStreamName.isEmpty()) {
+            // The restored stream may still hold rows appended after the 
checkpoint. They can
+            // be neither skipped nor removed, so the replay is written to a 
new stream.
+            BigQueryBatchWriter.retireRestoredStream(
+                    client, restoredStreamName, restoredNextOffset);
         }
+        return BigQueryBatchWriter.of(client, config);
     }
 
     @Override
@@ -92,6 +87,7 @@ public class BigQuerySinkBatchWriter extends 
AbstractBigQuerySinkWriter {
         if (buffer.length() == 0) {
             return;
         }
+        ensureStreamWriter();
 
         JSONArray dataToSend = buffer;
         buffer = new JSONArray();
@@ -164,6 +160,7 @@ public class BigQuerySinkBatchWriter extends 
AbstractBigQuerySinkWriter {
 
     @Override
     public Optional<BigQueryCommitInfo> prepareCommit() {
+        ensureStreamWriter();
         flush();
 
         BigQueryBatchWriter batchWriter = (BigQueryBatchWriter) streamWriter;
@@ -184,6 +181,7 @@ public class BigQuerySinkBatchWriter extends 
AbstractBigQuerySinkWriter {
 
     @Override
     public List<BigQuerySinkState> snapshotState(long checkpointId) {
+        ensureStreamWriter();
         flush();
 
         BigQueryBatchWriter batchWriter = (BigQueryBatchWriter) streamWriter;
diff --git 
a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkStreamWriter.java
 
b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkStreamWriter.java
index 8c6137eb37..46e87e48fa 100644
--- 
a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkStreamWriter.java
+++ 
b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkStreamWriter.java
@@ -18,7 +18,6 @@
 package org.apache.seatunnel.connectors.bigquery.sink;
 
 import org.apache.seatunnel.api.configuration.ReadonlyConfig;
-import org.apache.seatunnel.api.sink.MultiTableResourceManager;
 import org.apache.seatunnel.api.table.catalog.TableSchema;
 import org.apache.seatunnel.api.table.type.SeaTunnelRow;
 import org.apache.seatunnel.connectors.bigquery.convert.BigQuerySerializer;
@@ -27,11 +26,9 @@ import 
org.apache.seatunnel.connectors.bigquery.sink.writer.BigQueryStreamWriter
 import org.apache.seatunnel.connectors.bigquery.sink.writer.BigQueryWriter;
 
 import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient;
-import lombok.extern.slf4j.Slf4j;
 
 import java.util.Optional;
 
-@Slf4j
 public class BigQuerySinkStreamWriter extends AbstractBigQuerySinkWriter {
     public static final String STREAMING = "streaming";
 
@@ -50,11 +47,8 @@ public class BigQuerySinkStreamWriter extends 
AbstractBigQuerySinkWriter {
     }
 
     @Override
-    public void setMultiTableResourceManager(
-            MultiTableResourceManager<BigQueryWriteClient> manager, int 
queueIndex) {
-        log.info("Injecting shared client and initializing Streaming stream 
writer...");
-        this.client = manager.getSharedResource().get();
-        this.streamWriter = BigQueryStreamWriter.of(client, config);
+    protected BigQueryWriter openStreamWriter() {
+        return BigQueryStreamWriter.of(client, config);
     }
 
     @Override
diff --git 
a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/committer/BigQueryCommitter.java
 
b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/committer/BigQueryCommitter.java
index 72a7b00da3..8339538b63 100644
--- 
a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/committer/BigQueryCommitter.java
+++ 
b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/committer/BigQueryCommitter.java
@@ -22,12 +22,9 @@ import org.apache.seatunnel.api.sink.SinkCommitter;
 import org.apache.seatunnel.connectors.bigquery.client.BigQueryClientFactory;
 import 
org.apache.seatunnel.connectors.bigquery.exception.BigQueryConnectorErrorCode;
 import 
org.apache.seatunnel.connectors.bigquery.exception.BigQueryConnectorException;
+import 
org.apache.seatunnel.connectors.bigquery.sink.writer.BigQueryWriteStreamUtil;
 
 import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient;
-import com.google.cloud.bigquery.storage.v1.FlushRowsRequest;
-import com.google.cloud.bigquery.storage.v1.FlushRowsResponse;
-import com.google.protobuf.Int64Value;
-import lombok.extern.slf4j.Slf4j;
 
 import java.util.Collections;
 import java.util.List;
@@ -36,7 +33,6 @@ import java.util.stream.Collectors;
 
 import static 
org.apache.seatunnel.connectors.bigquery.sink.writer.BigQueryStreamWriter.DEFAULT_PATH;
 
-@Slf4j
 public class BigQueryCommitter implements SinkCommitter<BigQueryCommitInfo> {
     private final ReadonlyConfig config;
 
@@ -64,27 +60,8 @@ public class BigQueryCommitter implements 
SinkCommitter<BigQueryCommitInfo> {
 
         try (BigQueryWriteClient client = 
BigQueryClientFactory.getWriteClient(config)) {
             for (BigQueryCommitInfo info : bufferedCommitInfos) {
-                FlushRowsRequest request =
-                        FlushRowsRequest.newBuilder()
-                                .setWriteStream(info.getStreamName())
-                                
.setOffset(Int64Value.of(info.getFlushOffset()))
-                                .build();
-
-                FlushRowsResponse response = client.flushRows(request);
-
-                long flushedOffset = response.getOffset();
-                if (flushedOffset < info.getFlushOffset()) {
-                    throw new BigQueryConnectorException(
-                            BigQueryConnectorErrorCode.COMMIT_FAILED,
-                            String.format(
-                                    "FlushRows did not reach expected offset. 
stream=%s, expected=%d, actual=%d",
-                                    info.getStreamName(), 
info.getFlushOffset(), flushedOffset));
-                }
-
-                log.info(
-                        "Successfully flushed BigQuery buffered stream {} to 
offset {}",
-                        info.getStreamName(),
-                        info.getFlushOffset());
+                BigQueryWriteStreamUtil.flushRows(
+                        client, info.getStreamName(), info.getFlushOffset());
             }
         } catch (Exception e) {
             throw new 
BigQueryConnectorException(BigQueryConnectorErrorCode.COMMIT_FAILED, e);
diff --git 
a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/writer/BigQueryBatchWriter.java
 
b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/writer/BigQueryBatchWriter.java
index 7689a11fd5..482c8658bf 100644
--- 
a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/writer/BigQueryBatchWriter.java
+++ 
b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/writer/BigQueryBatchWriter.java
@@ -18,6 +18,8 @@
 package org.apache.seatunnel.connectors.bigquery.sink.writer;
 
 import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import 
org.apache.seatunnel.connectors.bigquery.exception.BigQueryConnectorErrorCode;
+import 
org.apache.seatunnel.connectors.bigquery.exception.BigQueryConnectorException;
 import org.apache.seatunnel.connectors.bigquery.option.BigQuerySinkOptions;
 
 import org.json.JSONArray;
@@ -25,6 +27,7 @@ import org.json.JSONArray;
 import com.google.api.core.ApiFuture;
 import com.google.cloud.bigquery.storage.v1.AppendRowsResponse;
 import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient;
+import com.google.cloud.bigquery.storage.v1.FinalizeWriteStreamResponse;
 import com.google.cloud.bigquery.storage.v1.JsonStreamWriter;
 import com.google.cloud.bigquery.storage.v1.TableName;
 import com.google.cloud.bigquery.storage.v1.TableSchema;
@@ -69,6 +72,57 @@ public class BigQueryBatchWriter implements BigQueryWriter {
                 client, config, getActualTableSchema(config, false), 
streamName, nextOffset);
     }
 
+    /**
+     * Retires a buffered stream recorded in restored checkpoint state, so 
that the replay can be
+     * written to a new stream. Rows before {@code restoredNextOffset} belong 
to the restored
+     * checkpoint and are flushed again in case its commit was missed. Rows 
appended after that
+     * checkpoint stay unflushed, and therefore invisible, because the source 
replays them. The
+     * stream is then finalized so that nothing else is appended to it.
+     *
+     * <p>Called once while a writer is restored. Repeating it is safe: a 
stream that an earlier
+     * restore already finalized was flushed by that restore first.
+     *
+     * @param client write client
+     * @param streamName buffered stream recorded in the restored checkpoint
+     * @param restoredNextOffset next offset recorded in the restored 
checkpoint
+     */
+    public static void retireRestoredStream(
+            BigQueryWriteClient client, String streamName, long 
restoredNextOffset) {
+        if (restoredNextOffset > 0) {
+            flushRestoredStream(client, streamName, restoredNextOffset - 1);
+        }
+
+        try {
+            FinalizeWriteStreamResponse response = 
client.finalizeWriteStream(streamName);
+            log.info(
+                    "Finalized restored Buffered write stream {} with {} rows",
+                    streamName,
+                    response.getRowCount());
+        } catch (RuntimeException e) {
+            if (!BigQueryWriteStreamUtil.isStreamFinalized(e)) {
+                throw new BigQueryConnectorException(
+                        BigQueryConnectorErrorCode.STREAM_FINALIZE_FAILED,
+                        String.format("Failed to finalize restored stream %s", 
streamName),
+                        e);
+            }
+            log.info("Restored Buffered write stream {} was already 
finalized", streamName);
+        }
+    }
+
+    private static void flushRestoredStream(
+            BigQueryWriteClient client, String streamName, long flushOffset) {
+        try {
+            BigQueryWriteStreamUtil.flushRows(client, streamName, flushOffset);
+        } catch (RuntimeException e) {
+            throw new BigQueryConnectorException(
+                    BigQueryConnectorErrorCode.COMMIT_FAILED,
+                    String.format(
+                            "Failed to flush restored stream %s to checkpoint 
offset %d",
+                            streamName, flushOffset),
+                    e);
+        }
+    }
+
     private static BigQueryBatchWriter of(
             BigQueryWriteClient client,
             ReadonlyConfig config,
diff --git 
a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/writer/BigQueryWriteStreamUtil.java
 
b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/writer/BigQueryWriteStreamUtil.java
new file mode 100644
index 0000000000..542739ae9e
--- /dev/null
+++ 
b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/writer/BigQueryWriteStreamUtil.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.seatunnel.connectors.bigquery.sink.writer;
+
+import 
org.apache.seatunnel.connectors.bigquery.exception.BigQueryConnectorErrorCode;
+import 
org.apache.seatunnel.connectors.bigquery.exception.BigQueryConnectorException;
+
+import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient;
+import com.google.cloud.bigquery.storage.v1.Exceptions;
+import com.google.cloud.bigquery.storage.v1.FlushRowsRequest;
+import com.google.cloud.bigquery.storage.v1.FlushRowsResponse;
+import com.google.protobuf.Int64Value;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class BigQueryWriteStreamUtil {
+    private BigQueryWriteStreamUtil() {}
+
+    /**
+     * Makes the rows of a buffered stream visible up to and including {@code 
flushOffset}.
+     *
+     * <p>A finalized stream counts as already flushed. The connector 
finalizes a stream only while
+     * retiring it on restore, after flushing it up to the restored 
checkpoint, and an engine that
+     * replays commits on restore can reach the same stream afterwards.
+     *
+     * @param client write client
+     * @param streamName buffered stream to flush
+     * @param flushOffset last offset to make visible
+     * @throws BigQueryConnectorException if BigQuery reports a flushed offset 
below {@code
+     *     flushOffset}
+     */
+    public static void flushRows(BigQueryWriteClient client, String 
streamName, long flushOffset) {
+        FlushRowsRequest request =
+                FlushRowsRequest.newBuilder()
+                        .setWriteStream(streamName)
+                        .setOffset(Int64Value.of(flushOffset))
+                        .build();
+
+        FlushRowsResponse response;
+        try {
+            response = client.flushRows(request);
+        } catch (RuntimeException e) {
+            if (isStreamFinalized(e)) {
+                log.warn(
+                        "BigQuery buffered stream {} was flushed and finalized 
by an earlier restore, skipping flush to offset {}",
+                        streamName,
+                        flushOffset);
+                return;
+            }
+            throw e;
+        }
+
+        long flushedOffset = response.getOffset();
+        if (flushedOffset < flushOffset) {
+            throw new BigQueryConnectorException(
+                    BigQueryConnectorErrorCode.COMMIT_FAILED,
+                    String.format(
+                            "FlushRows did not reach expected offset. 
stream=%s, expected=%d, actual=%d",
+                            streamName, flushOffset, flushedOffset));
+        }
+
+        log.info(
+                "Successfully flushed BigQuery buffered stream {} to offset 
{}",
+                streamName,
+                flushOffset);
+    }
+
+    /**
+     * Checks whether a failed call reports that the stream is already 
finalized.
+     *
+     * @param throwable the failure of a write stream call
+     * @return true if BigQuery rejected the call because the stream is 
finalized
+     */
+    public static boolean isStreamFinalized(Throwable throwable) {
+        return throwable instanceof Exceptions.StreamFinalizedException
+                || Exceptions.toStorageException(throwable)
+                        instanceof Exceptions.StreamFinalizedException;
+    }
+}
diff --git 
a/seatunnel-connectors-v2/connector-bigquery/src/test/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkBatchWriterTest.java
 
b/seatunnel-connectors-v2/connector-bigquery/src/test/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkBatchWriterTest.java
index fd772273b1..16357e65fb 100644
--- 
a/seatunnel-connectors-v2/connector-bigquery/src/test/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkBatchWriterTest.java
+++ 
b/seatunnel-connectors-v2/connector-bigquery/src/test/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkBatchWriterTest.java
@@ -18,29 +18,69 @@
 package org.apache.seatunnel.connectors.bigquery.sink;
 
 import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.sink.MultiTableResourceManager;
+import org.apache.seatunnel.connectors.bigquery.client.BigQueryClientFactory;
 import 
org.apache.seatunnel.connectors.bigquery.exception.BigQueryConnectorException;
+import org.apache.seatunnel.connectors.bigquery.option.BigQuerySinkOptions;
+import 
org.apache.seatunnel.connectors.bigquery.sink.committer.BigQueryCommitInfo;
+import 
org.apache.seatunnel.connectors.bigquery.sink.committer.BigQueryCommitter;
+import 
org.apache.seatunnel.connectors.bigquery.sink.writer.BigQueryBatchWriter;
 import org.apache.seatunnel.connectors.bigquery.sink.writer.BigQueryWriter;
+import org.apache.seatunnel.connectors.bigquery.sink.writer.TableSchemaUtil;
 
 import org.json.JSONArray;
 import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.InOrder;
+import org.mockito.MockedStatic;
 
 import com.google.api.core.ApiFuture;
 import com.google.api.core.ApiFutures;
+import com.google.api.gax.grpc.GrpcStatusCode;
 import com.google.api.gax.rpc.AlreadyExistsException;
+import com.google.api.gax.rpc.ApiException;
+import com.google.api.gax.rpc.ApiExceptionFactory;
 import com.google.api.gax.rpc.OutOfRangeException;
 import com.google.api.gax.rpc.StatusCode;
 import com.google.cloud.bigquery.storage.v1.AppendRowsResponse;
+import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient;
+import com.google.cloud.bigquery.storage.v1.FinalizeWriteStreamResponse;
+import com.google.cloud.bigquery.storage.v1.FlushRowsRequest;
+import com.google.cloud.bigquery.storage.v1.FlushRowsResponse;
+import com.google.cloud.bigquery.storage.v1.JsonStreamWriter;
+import com.google.cloud.bigquery.storage.v1.StorageError;
+import com.google.cloud.bigquery.storage.v1.TableSchema;
+import com.google.cloud.bigquery.storage.v1.WriteStream;
+import com.google.protobuf.Any;
 import com.google.rpc.Code;
 import com.google.rpc.Status;
+import io.grpc.protobuf.StatusProto;
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertSame;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 class BigQuerySinkBatchWriterTest {
+    private static final String RESTORED_STREAM =
+            
"projects/test-project/datasets/test_dataset/tables/test_table/streams/restored";
+    private static final String NEW_STREAM =
+            
"projects/test-project/datasets/test_dataset/tables/test_table/streams/new";
 
     @Test
     void testAlreadyExistsResponseAdvancesOffsetWithoutRecreatingStream() {
@@ -142,6 +182,271 @@ class BigQuerySinkBatchWriterTest {
         assertEquals(2, sinkWriter.buffer.length());
     }
 
+    @Test
+    void testRestoreFlushesCheckpointedRowsAndSwitchesToNewStream() {
+        BigQueryWriteClient client = mock(BigQueryWriteClient.class);
+        when(client.flushRows(any(FlushRowsRequest.class)))
+                
.thenReturn(FlushRowsResponse.newBuilder().setOffset(99L).build());
+        when(client.finalizeWriteStream(RESTORED_STREAM))
+                
.thenReturn(FinalizeWriteStreamResponse.newBuilder().setRowCount(900L).build());
+        when(client.createWriteStream(anyString(), any(WriteStream.class)))
+                
.thenReturn(WriteStream.newBuilder().setName(NEW_STREAM).build());
+
+        BigQuerySinkBatchWriter sinkWriter = restoreSinkWriter(client, 100L);
+
+        // Rows up to the checkpoint become visible before the old stream is 
closed off. Rows
+        // after it are never flushed, so the replay goes to a new stream from 
offset 0.
+        InOrder order = inOrder(client);
+        ArgumentCaptor<FlushRowsRequest> flushRequest =
+                ArgumentCaptor.forClass(FlushRowsRequest.class);
+        order.verify(client).flushRows(flushRequest.capture());
+        order.verify(client).finalizeWriteStream(RESTORED_STREAM);
+        order.verify(client).createWriteStream(anyString(), 
any(WriteStream.class));
+        assertEquals(RESTORED_STREAM, 
flushRequest.getValue().getWriteStream());
+        assertEquals(99L, flushRequest.getValue().getOffset().getValue());
+
+        BigQueryBatchWriter batchWriter = (BigQueryBatchWriter) 
sinkWriter.streamWriter;
+        assertEquals(NEW_STREAM, batchWriter.getStreamName());
+        assertEquals(0L, batchWriter.getNextOffset());
+    }
+
+    @Test
+    void testRestoreAcceptsStreamFinalizedByEarlierRestore() {
+        BigQueryWriteClient client = mock(BigQueryWriteClient.class);
+        
when(client.flushRows(any(FlushRowsRequest.class))).thenThrow(streamFinalized());
+        
when(client.finalizeWriteStream(RESTORED_STREAM)).thenThrow(streamFinalized());
+        when(client.createWriteStream(anyString(), any(WriteStream.class)))
+                
.thenReturn(WriteStream.newBuilder().setName(NEW_STREAM).build());
+
+        BigQuerySinkBatchWriter sinkWriter = restoreSinkWriter(client, 100L);
+
+        BigQueryBatchWriter batchWriter = (BigQueryBatchWriter) 
sinkWriter.streamWriter;
+        assertEquals(NEW_STREAM, batchWriter.getStreamName());
+        assertEquals(0L, batchWriter.getNextOffset());
+    }
+
+    @Test
+    void testRestoreFailsWhenCheckpointedRowsCannotBeFlushed() {
+        BigQueryWriteClient client = mock(BigQueryWriteClient.class);
+        when(client.flushRows(any(FlushRowsRequest.class)))
+                .thenThrow(
+                        ApiExceptionFactory.createException(
+                                new IOException("unavailable"),
+                                
GrpcStatusCode.of(io.grpc.Status.Code.UNAVAILABLE),
+                                true));
+
+        assertThrows(BigQueryConnectorException.class, () -> 
restoreSinkWriter(client, 100L));
+
+        verify(client, never()).finalizeWriteStream(anyString());
+        verify(client, never()).createWriteStream(anyString(), 
any(WriteStream.class));
+    }
+
+    @Test
+    void testRestoreFailsWhenFlushDoesNotReachCheckpointOffset() {
+        BigQueryWriteClient client = mock(BigQueryWriteClient.class);
+        when(client.flushRows(any(FlushRowsRequest.class)))
+                
.thenReturn(FlushRowsResponse.newBuilder().setOffset(50L).build());
+
+        assertThrows(BigQueryConnectorException.class, () -> 
restoreSinkWriter(client, 100L));
+
+        verify(client, never()).finalizeWriteStream(anyString());
+        verify(client, never()).createWriteStream(anyString(), 
any(WriteStream.class));
+    }
+
+    @Test
+    void testSharedClientIsNotClosedByWriter() {
+        BigQueryWriteClient client = mock(BigQueryWriteClient.class);
+        when(client.flushRows(any(FlushRowsRequest.class)))
+                
.thenReturn(FlushRowsResponse.newBuilder().setOffset(99L).build());
+        when(client.finalizeWriteStream(RESTORED_STREAM))
+                .thenReturn(FinalizeWriteStreamResponse.getDefaultInstance());
+        when(client.createWriteStream(anyString(), any(WriteStream.class)))
+                
.thenReturn(WriteStream.newBuilder().setName(NEW_STREAM).build());
+
+        restoreSinkWriter(client, 100L).close();
+
+        verify(client, never()).close();
+    }
+
+    @Test
+    void testBatchWriterWithoutSharedClientOpensItsOwnClient() throws 
Exception {
+        BigQueryWriteClient client = mock(BigQueryWriteClient.class);
+        when(client.createWriteStream(anyString(), any(WriteStream.class)))
+                
.thenReturn(WriteStream.newBuilder().setName(NEW_STREAM).build());
+        JsonStreamWriter jsonStreamWriter = mock(JsonStreamWriter.class);
+        when(jsonStreamWriter.append(any(JSONArray.class), anyLong()))
+                
.thenReturn(ApiFutures.immediateFuture(AppendRowsResponse.getDefaultInstance()));
+        BigQuerySinkBatchWriter sinkWriter =
+                new BigQuerySinkBatchWriter(testConfig(), null, null, null, 
0L);
+        sinkWriter.buffer.put("row");
+
+        try (MockedStatic<BigQueryClientFactory> clientFactory = 
mockClientFactory(client);
+                MockedStatic<TableSchemaUtil> ignored = 
mockTableSchemaUtil(jsonStreamWriter)) {
+            sinkWriter.flush();
+        }
+
+        verify(jsonStreamWriter).append(any(JSONArray.class), eq(0L));
+        BigQueryBatchWriter batchWriter = (BigQueryBatchWriter) 
sinkWriter.streamWriter;
+        assertEquals(NEW_STREAM, batchWriter.getStreamName());
+        assertEquals(1L, batchWriter.getNextOffset());
+
+        sinkWriter.close();
+        verify(client).close();
+    }
+
+    @Test
+    void testRestoredWriterWithoutSharedClientRetiresStreamAtFirstCheckpoint() 
{
+        BigQueryWriteClient client = mock(BigQueryWriteClient.class);
+        when(client.flushRows(any(FlushRowsRequest.class)))
+                
.thenReturn(FlushRowsResponse.newBuilder().setOffset(99L).build());
+        when(client.finalizeWriteStream(RESTORED_STREAM))
+                .thenReturn(FinalizeWriteStreamResponse.getDefaultInstance());
+        when(client.createWriteStream(anyString(), any(WriteStream.class)))
+                
.thenReturn(WriteStream.newBuilder().setName(NEW_STREAM).build());
+        BigQuerySinkBatchWriter sinkWriter =
+                new BigQuerySinkBatchWriter(testConfig(), null, null, 
RESTORED_STREAM, 100L);
+
+        // No row arrives after the restore, so the first barrier has to 
retire the old stream.
+        try (MockedStatic<BigQueryClientFactory> clientFactory = 
mockClientFactory(client);
+                MockedStatic<TableSchemaUtil> ignored =
+                        mockTableSchemaUtil(mock(JsonStreamWriter.class))) {
+            assertEquals(Optional.empty(), sinkWriter.prepareCommit());
+        }
+
+        InOrder order = inOrder(client);
+        order.verify(client).flushRows(any(FlushRowsRequest.class));
+        order.verify(client).finalizeWriteStream(RESTORED_STREAM);
+        order.verify(client).createWriteStream(anyString(), 
any(WriteStream.class));
+    }
+
+    @Test
+    void testStreamingWriterWithoutSharedClientDoesNotDropRows() throws 
Exception {
+        BigQueryWriteClient client = mock(BigQueryWriteClient.class);
+        JsonStreamWriter jsonStreamWriter = mock(JsonStreamWriter.class);
+        when(jsonStreamWriter.append(any(JSONArray.class)))
+                
.thenReturn(ApiFutures.immediateFuture(AppendRowsResponse.getDefaultInstance()));
+        BigQuerySinkStreamWriter sinkWriter =
+                new BigQuerySinkStreamWriter(testConfig(), null, null);
+        sinkWriter.buffer.put("row");
+
+        try (MockedStatic<BigQueryClientFactory> clientFactory = 
mockClientFactory(client);
+                MockedStatic<TableSchemaUtil> ignored = 
mockTableSchemaUtil(jsonStreamWriter)) {
+            sinkWriter.prepareCommit();
+        }
+
+        verify(jsonStreamWriter).append(any(JSONArray.class));
+        assertEquals(0, sinkWriter.buffer.length());
+    }
+
+    @Test
+    void testCommitterAcceptsStreamFinalizedByRestore() {
+        BigQueryWriteClient client = mock(BigQueryWriteClient.class);
+        
when(client.flushRows(any(FlushRowsRequest.class))).thenThrow(streamFinalized());
+
+        // An engine that replays commits on restore can reach the stream 
after it was retired.
+        try (MockedStatic<BigQueryClientFactory> clientFactory = 
mockClientFactory(client)) {
+            new BigQueryCommitter(testConfig())
+                    .commit(
+                            Collections.singletonList(
+                                    new BigQueryCommitInfo(RESTORED_STREAM, 
99L)));
+        }
+
+        verify(client).flushRows(any(FlushRowsRequest.class));
+    }
+
+    @Test
+    void testCommitterFailsWhenFlushFails() {
+        BigQueryWriteClient client = mock(BigQueryWriteClient.class);
+        when(client.flushRows(any(FlushRowsRequest.class)))
+                .thenThrow(
+                        ApiExceptionFactory.createException(
+                                new IOException("unavailable"),
+                                
GrpcStatusCode.of(io.grpc.Status.Code.UNAVAILABLE),
+                                true));
+        BigQueryCommitter committer = new BigQueryCommitter(testConfig());
+
+        try (MockedStatic<BigQueryClientFactory> clientFactory = 
mockClientFactory(client)) {
+            assertThrows(
+                    BigQueryConnectorException.class,
+                    () ->
+                            committer.commit(
+                                    Collections.singletonList(
+                                            new 
BigQueryCommitInfo(RESTORED_STREAM, 99L))));
+        }
+    }
+
+    private static BigQuerySinkBatchWriter restoreSinkWriter(
+            BigQueryWriteClient client, long restoredNextOffset) {
+        BigQuerySinkBatchWriter sinkWriter =
+                new BigQuerySinkBatchWriter(
+                        testConfig(), null, null, RESTORED_STREAM, 
restoredNextOffset);
+
+        @SuppressWarnings("unchecked")
+        MultiTableResourceManager<BigQueryWriteClient> manager =
+                mock(MultiTableResourceManager.class);
+        when(manager.getSharedResource()).thenReturn(Optional.of(client));
+
+        try (MockedStatic<TableSchemaUtil> ignored =
+                mockTableSchemaUtil(mock(JsonStreamWriter.class))) {
+            sinkWriter.setMultiTableResourceManager(manager, 0);
+        }
+        return sinkWriter;
+    }
+
+    private static ReadonlyConfig testConfig() {
+        Map<String, Object> options = new HashMap<>();
+        options.put(BigQuerySinkOptions.PROJECT_ID.key(), "test-project");
+        options.put(BigQuerySinkOptions.DATASET_ID.key(), "test_dataset");
+        options.put(BigQuerySinkOptions.TABLE_ID.key(), "test_table");
+        return ReadonlyConfig.fromMap(options);
+    }
+
+    private static MockedStatic<BigQueryClientFactory> mockClientFactory(
+            BigQueryWriteClient client) {
+        MockedStatic<BigQueryClientFactory> clientFactory = 
mockStatic(BigQueryClientFactory.class);
+        clientFactory
+                .when(() -> 
BigQueryClientFactory.getWriteClient(any(ReadonlyConfig.class)))
+                .thenReturn(client);
+        return clientFactory;
+    }
+
+    private static MockedStatic<TableSchemaUtil> mockTableSchemaUtil(
+            JsonStreamWriter jsonStreamWriter) {
+        return mockStatic(
+                TableSchemaUtil.class,
+                invocation -> {
+                    switch (invocation.getMethod().getName()) {
+                        case "getActualTableSchema":
+                            return TableSchema.getDefaultInstance();
+                        case "createStreamWriter":
+                            return jsonStreamWriter;
+                        default:
+                            return invocation.callRealMethod();
+                    }
+                });
+    }
+
+    /** The error a unary call returns when the stream was finalized, as gax 
surfaces it. */
+    private static ApiException streamFinalized() {
+        Status status =
+                Status.newBuilder()
+                        .setCode(Code.INVALID_ARGUMENT.getNumber())
+                        .setMessage("Stream is finalized")
+                        .addDetails(
+                                Any.pack(
+                                        StorageError.newBuilder()
+                                                .setCode(
+                                                        
StorageError.StorageErrorCode
+                                                                
.STREAM_FINALIZED)
+                                                .setEntity(RESTORED_STREAM)
+                                                .build()))
+                        .build();
+        return ApiExceptionFactory.createException(
+                StatusProto.toStatusRuntimeException(status),
+                GrpcStatusCode.of(io.grpc.Status.Code.INVALID_ARGUMENT),
+                false);
+    }
+
     private static BigQuerySinkBatchWriter createSinkWriter(BigQueryWriter 
streamWriter) {
         return new BigQuerySinkBatchWriter(
                 ReadonlyConfig.fromMap(Collections.emptyMap()), streamWriter, 
null);
diff --git 
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-bigquery-e2e/src/test/java/org/apache/seatunnel/e2e/connector/bigquery/BigQueryCDCRealIT.java
 
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-bigquery-e2e/src/test/java/org/apache/seatunnel/e2e/connector/bigquery/BigQueryCDCRealIT.java
index 7cdc3081fe..527358f2f5 100644
--- 
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-bigquery-e2e/src/test/java/org/apache/seatunnel/e2e/connector/bigquery/BigQueryCDCRealIT.java
+++ 
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-bigquery-e2e/src/test/java/org/apache/seatunnel/e2e/connector/bigquery/BigQueryCDCRealIT.java
@@ -25,6 +25,7 @@ import org.apache.seatunnel.e2e.common.container.EngineType;
 import org.apache.seatunnel.e2e.common.container.TestContainer;
 import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer;
 import org.apache.seatunnel.e2e.common.junit.TestContainerExtension;
+import org.apache.seatunnel.e2e.common.util.JobIdGenerator;
 
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
@@ -60,11 +61,15 @@ import java.sql.DriverManager;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -73,10 +78,12 @@ import java.util.stream.StreamSupport;
 import static org.awaitility.Awaitility.await;
 
 /**
- * Runs BigQuery CDC and ADD COLUMN schema evolution against the real BigQuery 
service.
+ * Runs BigQuery CDC and ADD COLUMN schema evolution, and the batch-mode 
restore from a checkpoint,
+ * against the real BigQuery service.
  *
  * <p>This test is opt-in because it requires GCP credentials, creates 
billable BigQuery jobs, and
- * cannot be verified correctly by goccy/bigquery-emulator.
+ * cannot be verified correctly by goccy/bigquery-emulator. The emulator 
ignores append offsets and
+ * inserts the buffered rows again on every FlushRows call.
  *
  * <p>Set {@code BIGQUERY_E2E_REAL=true}, {@code BIGQUERY_E2E_PROJECT_ID}, 
{@code
  * BIGQUERY_E2E_DATASET_ID}, and {@code GOOGLE_APPLICATION_CREDENTIALS}. The 
dataset must already
@@ -106,6 +113,20 @@ public class BigQueryCDCRealIT extends TestSuiteBase {
             Pattern.compile("[a-z][a-z0-9-]{4,28}[a-z0-9]");
     private static final Pattern DATASET_ID_PATTERN = 
Pattern.compile("[A-Za-z0-9_]+");
     private static final MySqlContainer MYSQL_CONTAINER = 
createMySqlContainer(MySqlVersion.V8_0);
+    private static final String BATCH_RESTORE_CONF = 
"/fake_to_bigquery_batch_real_restore.conf";
+    private static final long BATCH_RESTORE_ROW_COUNT = 60L;
+    private static final int BATCH_RESTORE_TIMEOUT_SECONDS = 300;
+    private static final long BATCH_RESTORE_CHECKPOINT_INTERVAL_MILLIS = 
10_000L;
+    // Long enough that no later checkpoint completes before the cancel takes 
effect, and short
+    // enough that the source is still emitting rows after the first 
checkpoint.
+    private static final long CANCELLED_JOB_CHECKPOINT_INTERVAL_MILLIS = 
20_000L;
+    // Longer than a restored job runs, so that job never completes a 
checkpoint of its own.
+    private static final long NO_CHECKPOINT_INTERVAL_MILLIS = 3_600_000L;
+    private static final Pattern FLUSHED_STREAM_PATTERN =
+            Pattern.compile(
+                    "Successfully flushed BigQuery buffered stream (\\S+) to 
offset (\\d+)");
+    private static final Pattern FINALIZED_STREAM_PATTERN =
+            Pattern.compile("Finalized restored Buffered write stream (\\S+) 
with (\\d+) rows");
 
     private String projectId;
     private String datasetId;
@@ -325,6 +346,304 @@ public class BigQueryCDCRealIT extends TestSuiteBase {
                         });
     }
 
+    /**
+     * Cancels a batch-mode job after rows were appended past its last 
completed checkpoint, then
+     * restores the job from that checkpoint. The rows left unflushed in the 
old buffered stream
+     * must stay invisible, and the replay must make every remaining row 
visible exactly once.
+     */
+    @TestTemplate
+    @DisabledOnContainer(
+            value = {},
+            type = {EngineType.SPARK, EngineType.FLINK},
+            disabledReason =
+                    "Cancelling and restoring a job are only implemented by 
the SeaTunnel test container")
+    public void testBatchModeRestoreAfterUncheckpointedAppends(TestContainer 
container)
+            throws Exception {
+        createBatchRestoreTable();
+        String jobId = runBatchRestoreJobUntilCancelled(container);
+
+        assertRestoredJobWritesEveryRowOnce(container, jobId);
+    }
+
+    /**
+     * Restores one checkpoint twice. The first restored job finalizes the 
buffered stream recorded
+     * in that checkpoint and is cancelled without completing a checkpoint of 
its own. The second
+     * restored job retires the same stream again, so FlushRows and 
FinalizeWriteStream reach a
+     * stream that is already finalized. The real service must answer in a way 
the sink accepts, and
+     * every row must still become visible exactly once.
+     */
+    @TestTemplate
+    @DisabledOnContainer(
+            value = {},
+            type = {EngineType.SPARK, EngineType.FLINK},
+            disabledReason =
+                    "Cancelling and restoring a job are only implemented by 
the SeaTunnel test container")
+    public void testBatchModeRestoreOfAlreadyFinalizedStream(TestContainer 
container)
+            throws Exception {
+        createBatchRestoreTable();
+        String jobId = runBatchRestoreJobUntilCancelled(container);
+
+        String firstRestoredJobId = String.valueOf(JobIdGenerator.newJobId());
+        CompletableFuture<Container.ExecResult> firstRestoredRun =
+                restoreBatchJob(
+                        container, jobId, firstRestoredJobId, 
NO_CHECKPOINT_INTERVAL_MILLIS);
+        try {
+            await().atMost(BATCH_RESTORE_TIMEOUT_SECONDS, TimeUnit.SECONDS)
+                    .pollInterval(1, TimeUnit.SECONDS)
+                    .until(
+                            () ->
+                                    countStreamRetirements(container) >= 1
+                                            && "RUNNING"
+                                                    .equals(
+                                                            
container.getJobStatus(
+                                                                    
firstRestoredJobId)));
+            Assertions.assertEquals(
+                    0L,
+                    container.getCompletedCheckpointCount(firstRestoredJobId),
+                    "The first restored job must not commit rows of its own");
+        } finally {
+            cancelBatchJob(container, firstRestoredJobId, firstRestoredRun);
+        }
+
+        assertRestoredJobWritesEveryRowOnce(container, jobId);
+
+        long retirements = countStreamRetirements(container);
+        Assertions.assertTrue(
+                retirements >= 2,
+                "The second restored job must retire the finalized stream 
again, but "
+                        + retirements
+                        + " retirements were logged");
+    }
+
+    /**
+     * Runs the batch restore job until the server logs show rows appended 
past its committed
+     * checkpoint, then cancels it. A later checkpoint can still cover those 
rows before the cancel
+     * takes effect, so {@link #assertRetiredStreamHeldUncheckpointedRows} 
checks the restored
+     * checkpoint afterwards.
+     *
+     * @return the id of the cancelled job
+     */
+    private String runBatchRestoreJobUntilCancelled(TestContainer container) 
throws Exception {
+        String jobId = String.valueOf(JobIdGenerator.newJobId());
+        String[] variables =
+                batchRestoreVariables(CANCELLED_JOB_CHECKPOINT_INTERVAL_MILLIS)
+                        .toArray(new String[0]);
+
+        CompletableFuture<Container.ExecResult> firstRun =
+                CompletableFuture.supplyAsync(
+                        () -> {
+                            try {
+                                return 
container.executeJob(BATCH_RESTORE_CONF, jobId, variables);
+                            } catch (Exception e) {
+                                throw new RuntimeException(e);
+                            }
+                        });
+        try {
+            await().atMost(INITIAL_SNAPSHOT_TIMEOUT_SECONDS, TimeUnit.SECONDS)
+                    .pollInterval(1, TimeUnit.SECONDS)
+                    .until(() -> container.getCompletedCheckpointCount(jobId) 
> 0);
+            await("rows appended past the committed checkpoint")
+                    .atMost(2, TimeUnit.MINUTES)
+                    .pollInterval(1, TimeUnit.SECONDS)
+                    .until(() -> hasUncheckpointedRows(container, jobId));
+        } finally {
+            cancelBatchJob(container, jobId, firstRun);
+        }
+
+        long rowsBeforeRestore = countRows();
+        Assertions.assertTrue(
+                rowsBeforeRestore > 0 && rowsBeforeRestore < 
BATCH_RESTORE_ROW_COUNT,
+                "The job must be cancelled after the first commit and before 
every row is written,"
+                        + " but "
+                        + rowsBeforeRestore
+                        + " rows were visible");
+        return jobId;
+    }
+
+    private void assertRestoredJobWritesEveryRowOnce(TestContainer container, 
String sourceJobId)
+            throws Exception {
+        List<Long> idsBeforeRestore = readRowIds();
+        // A job restored from a checkpoint runs under a new job id.
+        String restoredJobId = String.valueOf(JobIdGenerator.newJobId());
+        CompletableFuture<Container.ExecResult> restoredRun =
+                restoreBatchJob(
+                        container,
+                        sourceJobId,
+                        restoredJobId,
+                        BATCH_RESTORE_CHECKPOINT_INTERVAL_MILLIS);
+        try {
+            await().atMost(BATCH_RESTORE_TIMEOUT_SECONDS, TimeUnit.SECONDS)
+                    .pollInterval(2, TimeUnit.SECONDS)
+                    .untilAsserted(
+                            () -> 
Assertions.assertEquals(BATCH_RESTORE_ROW_COUNT, countRows()));
+
+            // Later checkpoints flush the same offsets again, which must not 
add rows.
+            long checkpointsAtFullCount = 
container.getCompletedCheckpointCount(restoredJobId);
+            await().atMost(2, TimeUnit.MINUTES)
+                    .pollInterval(1, TimeUnit.SECONDS)
+                    .until(
+                            () ->
+                                    
container.getCompletedCheckpointCount(restoredJobId)
+                                            > checkpointsAtFullCount);
+
+            // FakeSource draws a new random id for every row, replayed rows 
included, so ids can
+            // only show that committed rows survive and that no row became 
visible twice.
+            List<Long> idsAfterRestore = readRowIds();
+            Assertions.assertEquals(BATCH_RESTORE_ROW_COUNT, 
idsAfterRestore.size());
+            Assertions.assertEquals(
+                    BATCH_RESTORE_ROW_COUNT,
+                    new HashSet<>(idsAfterRestore).size(),
+                    "A row became visible more than once");
+            Assertions.assertTrue(
+                    idsAfterRestore.containsAll(idsBeforeRestore),
+                    "A row committed before the restore disappeared");
+            assertRetiredStreamHeldUncheckpointedRows(container);
+        } finally {
+            container.cancelJob(restoredJobId);
+            restoredRun.get(2, TimeUnit.MINUTES);
+        }
+    }
+
+    /**
+     * Proves that the restore ran inside the failure window: the retired 
stream held more rows than
+     * the restored checkpoint flushed. BigQuery reports the row count of a 
stream only when it is
+     * first finalized, so this can only be checked after the restore.
+     */
+    private void assertRetiredStreamHeldUncheckpointedRows(TestContainer 
container) {
+        Map<String, Long> finalizedRows =
+                maxLoggedValuePerStream(container, FINALIZED_STREAM_PATTERN);
+        Assertions.assertEquals(1, finalizedRows.size(), "Exactly one stream 
must be retired");
+
+        Map.Entry<String, Long> retiredStream = 
finalizedRows.entrySet().iterator().next();
+        long flushedRows =
+                maxLoggedValuePerStream(container, FLUSHED_STREAM_PATTERN)
+                                .get(retiredStream.getKey())
+                        + 1;
+        Assertions.assertTrue(
+                retiredStream.getValue() > flushedRows,
+                "The cancelled job must leave rows past its checkpoint, but 
the stream held "
+                        + retiredStream.getValue()
+                        + " rows and the checkpoint covered "
+                        + flushedRows);
+    }
+
+    private CompletableFuture<Container.ExecResult> restoreBatchJob(
+            TestContainer container,
+            String sourceJobId,
+            String restoredJobId,
+            long checkpointIntervalMillis) {
+        List<String> variables = 
batchRestoreVariables(checkpointIntervalMillis);
+        return CompletableFuture.supplyAsync(
+                () -> {
+                    try {
+                        return container.restoreJobWithCheckpoint(
+                                BATCH_RESTORE_CONF, sourceJobId, 
restoredJobId, variables);
+                    } catch (Exception e) {
+                        throw new RuntimeException(e);
+                    }
+                });
+    }
+
+    private void cancelBatchJob(
+            TestContainer container, String jobId, 
CompletableFuture<Container.ExecResult> run)
+            throws Exception {
+        Container.ExecResult cancelResult = container.cancelJob(jobId);
+        Assertions.assertEquals(0, cancelResult.getExitCode(), 
cancelResult.getStderr());
+        await().atMost(2, TimeUnit.MINUTES)
+                .pollInterval(1, TimeUnit.SECONDS)
+                .until(() -> "CANCELED".equals(container.getJobStatus(jobId)));
+        run.get(2, TimeUnit.MINUTES);
+    }
+
+    private List<String> batchRestoreVariables(long checkpointIntervalMillis) {
+        return Arrays.asList(
+                "bigquery_project_id=" + projectId,
+                "bigquery_dataset_id=" + datasetId,
+                "bigquery_table_id=" + tableId.getTable(),
+                "checkpoint_interval=" + checkpointIntervalMillis);
+    }
+
+    /** Tells whether a job appended more rows than its committed checkpoint 
made visible. */
+    private boolean hasUncheckpointedRows(TestContainer container, String 
jobId) {
+        long appendedRows = countAppendedRows(container, jobId);
+        return maxLoggedValuePerStream(container, 
FLUSHED_STREAM_PATTERN).values().stream()
+                .anyMatch(flushedOffset -> appendedRows > flushedOffset + 1);
+    }
+
+    /** Counts the rows a job appended. The restore config appends one row per 
call. */
+    private long countAppendedRows(TestContainer container, String jobId) {
+        String jobLogPrefix = "[" + jobId + "]";
+        return Arrays.stream(container.getServerLogs().split("\n"))
+                .filter(line -> line.contains(jobLogPrefix))
+                .filter(line -> line.contains("Successfully appended 1 rows."))
+                .count();
+    }
+
+    /** Collects the largest value the pattern logged for each stream of the 
current table. */
+    private Map<String, Long> maxLoggedValuePerStream(TestContainer container, 
Pattern pattern) {
+        String streamsOfTable = "/tables/" + tableId.getTable() + "/streams/";
+        Map<String, Long> values = new HashMap<>();
+        Matcher matcher = pattern.matcher(container.getServerLogs());
+        while (matcher.find()) {
+            String stream = matcher.group(1);
+            if (stream.contains(streamsOfTable)) {
+                values.merge(stream, Long.parseLong(matcher.group(2)), 
Math::max);
+            }
+        }
+        return values;
+    }
+
+    /** Counts how often a stream of the current table was retired, whatever 
BigQuery answered. */
+    private long countStreamRetirements(TestContainer container) {
+        String streamsOfTable = "/tables/" + tableId.getTable() + "/streams/";
+        return Arrays.stream(container.getServerLogs().split("\n"))
+                .filter(line -> line.contains(streamsOfTable))
+                .filter(
+                        line ->
+                                line.contains("Finalized restored Buffered 
write stream")
+                                        || line.contains("was already 
finalized"))
+                .count();
+    }
+
+    private void createBatchRestoreTable() throws InterruptedException {
+        tableId =
+                TableId.of(
+                        projectId,
+                        datasetId,
+                        "seatunnel_batch_restore_" + 
UUID.randomUUID().toString().replace("-", ""));
+        String sql =
+                String.format(
+                        "CREATE TABLE `%s.%s.%s` (id INT64, name STRING) 
OPTIONS ("
+                                + "expiration_timestamp = "
+                                + "TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 
1 DAY)"
+                                + ")",
+                        projectId, datasetId, tableId.getTable());
+        bigQuery.query(QueryJobConfiguration.of(sql));
+    }
+
+    private long countRows() throws InterruptedException {
+        String query =
+                String.format(
+                        "SELECT COUNT(*) FROM `%s.%s.%s`",
+                        projectId, datasetId, tableId.getTable());
+        TableResult result =
+                bigQuery.query(
+                        
QueryJobConfiguration.newBuilder(query).setUseQueryCache(false).build());
+        return result.iterateAll().iterator().next().get(0).getLongValue();
+    }
+
+    private List<Long> readRowIds() throws InterruptedException {
+        String query =
+                String.format(
+                        "SELECT id FROM `%s.%s.%s`", projectId, datasetId, 
tableId.getTable());
+        TableResult result =
+                bigQuery.query(
+                        
QueryJobConfiguration.newBuilder(query).setUseQueryCache(false).build());
+        return StreamSupport.stream(result.iterateAll().spliterator(), false)
+                .map(row -> row.get(0).getLongValue())
+                .collect(Collectors.toList());
+    }
+
     private void createBigQueryTable() throws InterruptedException {
         String sql =
                 String.format(
diff --git 
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-bigquery-e2e/src/test/resources/fake_to_bigquery_batch_real_restore.conf
 
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-bigquery-e2e/src/test/resources/fake_to_bigquery_batch_real_restore.conf
new file mode 100644
index 0000000000..3853559aa5
--- /dev/null
+++ 
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-bigquery-e2e/src/test/resources/fake_to_bigquery_batch_real_restore.conf
@@ -0,0 +1,51 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+  parallelism = 1
+  job.mode = "STREAMING"
+  # Set by the test. It is long enough for rows to be appended after the last 
completed
+  # checkpoint before the test cancels the job.
+  checkpoint.interval = ${checkpoint_interval}
+  checkpoint.retain-after-job-cancelled = true
+}
+
+source {
+  FakeSource {
+    row.num = 60
+    split.num = 30
+    split.read-interval = 1000
+    schema = {
+      fields {
+        id = bigint
+        name = string
+      }
+    }
+  }
+}
+
+sink {
+  BigQuery {
+    project_id = "${bigquery_project_id}"
+    dataset_id = "${bigquery_dataset_id}"
+    table_id = "${bigquery_table_id}"
+    service_account_key_path = "/tmp/seatunnel/bigquery-service-account.json"
+    write_mode = "batch"
+    # Every row is appended to the buffered stream as soon as it arrives.
+    batch_size = 1
+  }
+}
diff --git 
a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/TestContainer.java
 
b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/TestContainer.java
index db5a8ce100..92172a4e8c 100644
--- 
a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/TestContainer.java
+++ 
b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/TestContainer.java
@@ -81,6 +81,22 @@ public interface TestContainer extends TestResource {
         throw new UnsupportedOperationException("Not implemented");
     }
 
+    /**
+     * Starts a new job from the latest checkpoint of an ended job.
+     *
+     * @param confFile job config file
+     * @param sourceJobId ended job whose checkpoint is restored
+     * @param restoreJobId id of the new job, which must differ from {@code 
sourceJobId}
+     * @param variables {@code key=value} entries passed to the job config as 
{@code -i} options, or
+     *     null if the config has no variables
+     * @return result of the restore command
+     */
+    default Container.ExecResult restoreJobWithCheckpoint(
+            String confFile, String sourceJobId, String restoreJobId, 
List<String> variables)
+            throws IOException, InterruptedException {
+        throw new UnsupportedOperationException("Not implemented");
+    }
+
     default Container.ExecResult cancelJob(String jobId) throws IOException, 
InterruptedException {
         throw new UnsupportedOperationException("Not implemented");
     }
diff --git 
a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java
 
b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java
index 6a2846589e..773f3aab18 100644
--- 
a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java
+++ 
b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java
@@ -959,10 +959,22 @@ public class SeaTunnelContainer extends 
AbstractTestContainer implements Reusabl
     public Container.ExecResult restoreJobWithCheckpoint(
             String confFile, String sourceJobId, String restoreJobId)
             throws IOException, InterruptedException {
+        return restoreJobWithCheckpoint(confFile, sourceJobId, restoreJobId, 
(List<String>) null);
+    }
+
+    @Override
+    public Container.ExecResult restoreJobWithCheckpoint(
+            String confFile, String sourceJobId, String restoreJobId, 
List<String> variables)
+            throws IOException, InterruptedException {
         runningCount.incrementAndGet();
         try {
             return restoreJob(
-                    server, confFile, sourceJobId, restoreJobId, null, 
"--restore-with-checkpoint");
+                    server,
+                    confFile,
+                    sourceJobId,
+                    restoreJobId,
+                    variables,
+                    "--restore-with-checkpoint");
         } finally {
             runningCount.decrementAndGet();
         }

Reply via email to