This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-13127-df59bb4cd1c1fef7f2c38ade6456b8c0c2658da8 in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit 1bff1866f702dc24fe56b6030b37d971df400b23 Author: Doyeon Kim <[email protected]> AuthorDate: Fri Nov 6 13:49:26 2026 +0000 [Fix][Connector-V2] Fix BigQuery sink stream initialization (#13127) --- .../bigquery/sink/AbstractBigQuerySinkWriter.java | 45 ++- .../bigquery/sink/BigQuerySinkBatchWriter.java | 22 +- .../bigquery/sink/BigQuerySinkStreamWriter.java | 10 +- .../bigquery/sink/committer/BigQueryCommitter.java | 29 +- .../bigquery/sink/writer/BigQueryBatchWriter.java | 54 ++++ .../sink/writer/BigQueryWriteStreamUtil.java | 94 ++++++ .../bigquery/sink/BigQuerySinkBatchWriterTest.java | 305 +++++++++++++++++++ .../e2e/connector/bigquery/BigQueryCDCRealIT.java | 323 ++++++++++++++++++++- .../fake_to_bigquery_batch_real_restore.conf | 51 ++++ .../e2e/common/container/TestContainer.java | 16 + .../container/seatunnel/SeaTunnelContainer.java | 14 +- 11 files changed, 911 insertions(+), 52 deletions(-) diff --git a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/AbstractBigQuerySinkWriter.java b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/AbstractBigQuerySinkWriter.java index f73d8ff048..5d62d9dba8 100644 --- a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/AbstractBigQuerySinkWriter.java +++ b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/AbstractBigQuerySinkWriter.java @@ -63,11 +63,14 @@ public abstract class AbstractBigQuerySinkWriter protected final ReadonlyConfig config; protected BigQuerySerializer serializer; - // Mutable: assigned eagerly by the constructor, or lazily by setMultiTableResourceManager() - // once the shared client is injected by the multi-table sink runtime. + // Mutable: assigned by the constructor, by setMultiTableResourceManager() when the multi-table + // sink runtime injects the shared client, or by ensureStreamWriter() on first use otherwise. protected BigQueryWriteClient client; protected BigQueryWriter streamWriter; protected TableSchema tableSchema; + // True only for a client created by ensureStreamWriter(), which close() must release. A client + // injected by the multi-table runtime is shared with sibling writers and must stay open. + private boolean ownsClient; protected final int batchSize; protected JSONArray buffer = new JSONArray(); @@ -106,8 +109,36 @@ public abstract class AbstractBigQuerySinkWriter return new BigQueryMultiTableResourceManager(BigQueryClientFactory.getWriteClient(config)); } + @Override + public void setMultiTableResourceManager( + MultiTableResourceManager<BigQueryWriteClient> manager, int queueIndex) { + log.info("Injecting shared client and initializing BigQuery stream writer..."); + this.client = manager.getSharedResource().get(); + ensureStreamWriter(); + } + + /** + * Prepares the stream writer if it does not exist yet. A sink with several inputs is not + * wrapped in the multi-table sink, so {@link #setMultiTableResourceManager} is never called and + * the writer has to open its own client on first use. + */ + protected void ensureStreamWriter() { + if (streamWriter != null) { + return; + } + if (client == null) { + client = BigQueryClientFactory.getWriteClient(config); + ownsClient = true; + } + streamWriter = openStreamWriter(); + } + + /** Opens the stream this writer appends to, using {@link #client}. */ + protected abstract BigQueryWriter openStreamWriter(); + protected void flush() { - if (streamWriter == null || buffer.length() == 0) return; + if (buffer.length() == 0) return; + ensureStreamWriter(); JSONArray dataToSend = buffer; buffer = new JSONArray(); @@ -257,6 +288,7 @@ public abstract class AbstractBigQuerySinkWriter BigQuerySerializer evolvedSerializer = new BigQuerySerializer(evolvedTableSchema.toPhysicalRowDataType(), config); + ensureStreamWriter(); flush(); getSchemaChangeManager().applySchemaChange(event); BigQueryWriter evolvedWriter = streamWriter.refreshSchema(client, config); @@ -291,6 +323,13 @@ public abstract class AbstractBigQuerySinkWriter // The shared BigQueryWriteClient is owned by BigQueryMultiTableResourceManager and is // closed once for the whole multi-table sink writer group; closing it per-writer here // would break sibling replica writers still using it. + if (ownsClient) { + try { + client.close(); + } catch (Exception e) { + log.warn("Failed to close BigQueryWriteClient", e); + } + } } } } diff --git a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkBatchWriter.java b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkBatchWriter.java index 62be99119f..88da49eb9a 100644 --- a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkBatchWriter.java +++ b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkBatchWriter.java @@ -18,7 +18,6 @@ package org.apache.seatunnel.connectors.bigquery.sink; import org.apache.seatunnel.api.configuration.ReadonlyConfig; -import org.apache.seatunnel.api.sink.MultiTableResourceManager; import org.apache.seatunnel.api.table.catalog.TableSchema; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.connectors.bigquery.convert.BigQuerySerializer; @@ -33,7 +32,6 @@ import org.json.JSONArray; import com.google.api.gax.rpc.ApiException; import com.google.api.gax.rpc.StatusCode; import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; -import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; import com.google.rpc.Code; import lombok.extern.slf4j.Slf4j; @@ -74,17 +72,14 @@ public class BigQuerySinkBatchWriter extends AbstractBigQuerySinkWriter { } @Override - public void setMultiTableResourceManager( - MultiTableResourceManager<BigQueryWriteClient> manager, int queueIndex) { - log.info("Injecting shared client and initializing Batch stream writer..."); - this.client = manager.getSharedResource().get(); - if (restoredStreamName != null) { - this.streamWriter = - BigQueryBatchWriter.restore( - client, config, restoredStreamName, restoredNextOffset); - } else { - this.streamWriter = BigQueryBatchWriter.of(client, config); + protected BigQueryWriter openStreamWriter() { + if (restoredStreamName != null && !restoredStreamName.isEmpty()) { + // The restored stream may still hold rows appended after the checkpoint. They can + // be neither skipped nor removed, so the replay is written to a new stream. + BigQueryBatchWriter.retireRestoredStream( + client, restoredStreamName, restoredNextOffset); } + return BigQueryBatchWriter.of(client, config); } @Override @@ -92,6 +87,7 @@ public class BigQuerySinkBatchWriter extends AbstractBigQuerySinkWriter { if (buffer.length() == 0) { return; } + ensureStreamWriter(); JSONArray dataToSend = buffer; buffer = new JSONArray(); @@ -164,6 +160,7 @@ public class BigQuerySinkBatchWriter extends AbstractBigQuerySinkWriter { @Override public Optional<BigQueryCommitInfo> prepareCommit() { + ensureStreamWriter(); flush(); BigQueryBatchWriter batchWriter = (BigQueryBatchWriter) streamWriter; @@ -184,6 +181,7 @@ public class BigQuerySinkBatchWriter extends AbstractBigQuerySinkWriter { @Override public List<BigQuerySinkState> snapshotState(long checkpointId) { + ensureStreamWriter(); flush(); BigQueryBatchWriter batchWriter = (BigQueryBatchWriter) streamWriter; diff --git a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkStreamWriter.java b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkStreamWriter.java index 8c6137eb37..46e87e48fa 100644 --- a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkStreamWriter.java +++ b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkStreamWriter.java @@ -18,7 +18,6 @@ package org.apache.seatunnel.connectors.bigquery.sink; import org.apache.seatunnel.api.configuration.ReadonlyConfig; -import org.apache.seatunnel.api.sink.MultiTableResourceManager; import org.apache.seatunnel.api.table.catalog.TableSchema; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.connectors.bigquery.convert.BigQuerySerializer; @@ -27,11 +26,9 @@ import org.apache.seatunnel.connectors.bigquery.sink.writer.BigQueryStreamWriter import org.apache.seatunnel.connectors.bigquery.sink.writer.BigQueryWriter; import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; -import lombok.extern.slf4j.Slf4j; import java.util.Optional; -@Slf4j public class BigQuerySinkStreamWriter extends AbstractBigQuerySinkWriter { public static final String STREAMING = "streaming"; @@ -50,11 +47,8 @@ public class BigQuerySinkStreamWriter extends AbstractBigQuerySinkWriter { } @Override - public void setMultiTableResourceManager( - MultiTableResourceManager<BigQueryWriteClient> manager, int queueIndex) { - log.info("Injecting shared client and initializing Streaming stream writer..."); - this.client = manager.getSharedResource().get(); - this.streamWriter = BigQueryStreamWriter.of(client, config); + protected BigQueryWriter openStreamWriter() { + return BigQueryStreamWriter.of(client, config); } @Override diff --git a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/committer/BigQueryCommitter.java b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/committer/BigQueryCommitter.java index 72a7b00da3..8339538b63 100644 --- a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/committer/BigQueryCommitter.java +++ b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/committer/BigQueryCommitter.java @@ -22,12 +22,9 @@ import org.apache.seatunnel.api.sink.SinkCommitter; import org.apache.seatunnel.connectors.bigquery.client.BigQueryClientFactory; import org.apache.seatunnel.connectors.bigquery.exception.BigQueryConnectorErrorCode; import org.apache.seatunnel.connectors.bigquery.exception.BigQueryConnectorException; +import org.apache.seatunnel.connectors.bigquery.sink.writer.BigQueryWriteStreamUtil; import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; -import com.google.cloud.bigquery.storage.v1.FlushRowsRequest; -import com.google.cloud.bigquery.storage.v1.FlushRowsResponse; -import com.google.protobuf.Int64Value; -import lombok.extern.slf4j.Slf4j; import java.util.Collections; import java.util.List; @@ -36,7 +33,6 @@ import java.util.stream.Collectors; import static org.apache.seatunnel.connectors.bigquery.sink.writer.BigQueryStreamWriter.DEFAULT_PATH; -@Slf4j public class BigQueryCommitter implements SinkCommitter<BigQueryCommitInfo> { private final ReadonlyConfig config; @@ -64,27 +60,8 @@ public class BigQueryCommitter implements SinkCommitter<BigQueryCommitInfo> { try (BigQueryWriteClient client = BigQueryClientFactory.getWriteClient(config)) { for (BigQueryCommitInfo info : bufferedCommitInfos) { - FlushRowsRequest request = - FlushRowsRequest.newBuilder() - .setWriteStream(info.getStreamName()) - .setOffset(Int64Value.of(info.getFlushOffset())) - .build(); - - FlushRowsResponse response = client.flushRows(request); - - long flushedOffset = response.getOffset(); - if (flushedOffset < info.getFlushOffset()) { - throw new BigQueryConnectorException( - BigQueryConnectorErrorCode.COMMIT_FAILED, - String.format( - "FlushRows did not reach expected offset. stream=%s, expected=%d, actual=%d", - info.getStreamName(), info.getFlushOffset(), flushedOffset)); - } - - log.info( - "Successfully flushed BigQuery buffered stream {} to offset {}", - info.getStreamName(), - info.getFlushOffset()); + BigQueryWriteStreamUtil.flushRows( + client, info.getStreamName(), info.getFlushOffset()); } } catch (Exception e) { throw new BigQueryConnectorException(BigQueryConnectorErrorCode.COMMIT_FAILED, e); diff --git a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/writer/BigQueryBatchWriter.java b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/writer/BigQueryBatchWriter.java index 7689a11fd5..482c8658bf 100644 --- a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/writer/BigQueryBatchWriter.java +++ b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/writer/BigQueryBatchWriter.java @@ -18,6 +18,8 @@ package org.apache.seatunnel.connectors.bigquery.sink.writer; import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.connectors.bigquery.exception.BigQueryConnectorErrorCode; +import org.apache.seatunnel.connectors.bigquery.exception.BigQueryConnectorException; import org.apache.seatunnel.connectors.bigquery.option.BigQuerySinkOptions; import org.json.JSONArray; @@ -25,6 +27,7 @@ import org.json.JSONArray; import com.google.api.core.ApiFuture; import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; +import com.google.cloud.bigquery.storage.v1.FinalizeWriteStreamResponse; import com.google.cloud.bigquery.storage.v1.JsonStreamWriter; import com.google.cloud.bigquery.storage.v1.TableName; import com.google.cloud.bigquery.storage.v1.TableSchema; @@ -69,6 +72,57 @@ public class BigQueryBatchWriter implements BigQueryWriter { client, config, getActualTableSchema(config, false), streamName, nextOffset); } + /** + * Retires a buffered stream recorded in restored checkpoint state, so that the replay can be + * written to a new stream. Rows before {@code restoredNextOffset} belong to the restored + * checkpoint and are flushed again in case its commit was missed. Rows appended after that + * checkpoint stay unflushed, and therefore invisible, because the source replays them. The + * stream is then finalized so that nothing else is appended to it. + * + * <p>Called once while a writer is restored. Repeating it is safe: a stream that an earlier + * restore already finalized was flushed by that restore first. + * + * @param client write client + * @param streamName buffered stream recorded in the restored checkpoint + * @param restoredNextOffset next offset recorded in the restored checkpoint + */ + public static void retireRestoredStream( + BigQueryWriteClient client, String streamName, long restoredNextOffset) { + if (restoredNextOffset > 0) { + flushRestoredStream(client, streamName, restoredNextOffset - 1); + } + + try { + FinalizeWriteStreamResponse response = client.finalizeWriteStream(streamName); + log.info( + "Finalized restored Buffered write stream {} with {} rows", + streamName, + response.getRowCount()); + } catch (RuntimeException e) { + if (!BigQueryWriteStreamUtil.isStreamFinalized(e)) { + throw new BigQueryConnectorException( + BigQueryConnectorErrorCode.STREAM_FINALIZE_FAILED, + String.format("Failed to finalize restored stream %s", streamName), + e); + } + log.info("Restored Buffered write stream {} was already finalized", streamName); + } + } + + private static void flushRestoredStream( + BigQueryWriteClient client, String streamName, long flushOffset) { + try { + BigQueryWriteStreamUtil.flushRows(client, streamName, flushOffset); + } catch (RuntimeException e) { + throw new BigQueryConnectorException( + BigQueryConnectorErrorCode.COMMIT_FAILED, + String.format( + "Failed to flush restored stream %s to checkpoint offset %d", + streamName, flushOffset), + e); + } + } + private static BigQueryBatchWriter of( BigQueryWriteClient client, ReadonlyConfig config, diff --git a/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/writer/BigQueryWriteStreamUtil.java b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/writer/BigQueryWriteStreamUtil.java new file mode 100644 index 0000000000..542739ae9e --- /dev/null +++ b/seatunnel-connectors-v2/connector-bigquery/src/main/java/org/apache/seatunnel/connectors/bigquery/sink/writer/BigQueryWriteStreamUtil.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.bigquery.sink.writer; + +import org.apache.seatunnel.connectors.bigquery.exception.BigQueryConnectorErrorCode; +import org.apache.seatunnel.connectors.bigquery.exception.BigQueryConnectorException; + +import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; +import com.google.cloud.bigquery.storage.v1.Exceptions; +import com.google.cloud.bigquery.storage.v1.FlushRowsRequest; +import com.google.cloud.bigquery.storage.v1.FlushRowsResponse; +import com.google.protobuf.Int64Value; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class BigQueryWriteStreamUtil { + private BigQueryWriteStreamUtil() {} + + /** + * Makes the rows of a buffered stream visible up to and including {@code flushOffset}. + * + * <p>A finalized stream counts as already flushed. The connector finalizes a stream only while + * retiring it on restore, after flushing it up to the restored checkpoint, and an engine that + * replays commits on restore can reach the same stream afterwards. + * + * @param client write client + * @param streamName buffered stream to flush + * @param flushOffset last offset to make visible + * @throws BigQueryConnectorException if BigQuery reports a flushed offset below {@code + * flushOffset} + */ + public static void flushRows(BigQueryWriteClient client, String streamName, long flushOffset) { + FlushRowsRequest request = + FlushRowsRequest.newBuilder() + .setWriteStream(streamName) + .setOffset(Int64Value.of(flushOffset)) + .build(); + + FlushRowsResponse response; + try { + response = client.flushRows(request); + } catch (RuntimeException e) { + if (isStreamFinalized(e)) { + log.warn( + "BigQuery buffered stream {} was flushed and finalized by an earlier restore, skipping flush to offset {}", + streamName, + flushOffset); + return; + } + throw e; + } + + long flushedOffset = response.getOffset(); + if (flushedOffset < flushOffset) { + throw new BigQueryConnectorException( + BigQueryConnectorErrorCode.COMMIT_FAILED, + String.format( + "FlushRows did not reach expected offset. stream=%s, expected=%d, actual=%d", + streamName, flushOffset, flushedOffset)); + } + + log.info( + "Successfully flushed BigQuery buffered stream {} to offset {}", + streamName, + flushOffset); + } + + /** + * Checks whether a failed call reports that the stream is already finalized. + * + * @param throwable the failure of a write stream call + * @return true if BigQuery rejected the call because the stream is finalized + */ + public static boolean isStreamFinalized(Throwable throwable) { + return throwable instanceof Exceptions.StreamFinalizedException + || Exceptions.toStorageException(throwable) + instanceof Exceptions.StreamFinalizedException; + } +} diff --git a/seatunnel-connectors-v2/connector-bigquery/src/test/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkBatchWriterTest.java b/seatunnel-connectors-v2/connector-bigquery/src/test/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkBatchWriterTest.java index fd772273b1..16357e65fb 100644 --- a/seatunnel-connectors-v2/connector-bigquery/src/test/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkBatchWriterTest.java +++ b/seatunnel-connectors-v2/connector-bigquery/src/test/java/org/apache/seatunnel/connectors/bigquery/sink/BigQuerySinkBatchWriterTest.java @@ -18,29 +18,69 @@ package org.apache.seatunnel.connectors.bigquery.sink; import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.sink.MultiTableResourceManager; +import org.apache.seatunnel.connectors.bigquery.client.BigQueryClientFactory; import org.apache.seatunnel.connectors.bigquery.exception.BigQueryConnectorException; +import org.apache.seatunnel.connectors.bigquery.option.BigQuerySinkOptions; +import org.apache.seatunnel.connectors.bigquery.sink.committer.BigQueryCommitInfo; +import org.apache.seatunnel.connectors.bigquery.sink.committer.BigQueryCommitter; +import org.apache.seatunnel.connectors.bigquery.sink.writer.BigQueryBatchWriter; import org.apache.seatunnel.connectors.bigquery.sink.writer.BigQueryWriter; +import org.apache.seatunnel.connectors.bigquery.sink.writer.TableSchemaUtil; import org.json.JSONArray; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; +import org.mockito.MockedStatic; import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutures; +import com.google.api.gax.grpc.GrpcStatusCode; import com.google.api.gax.rpc.AlreadyExistsException; +import com.google.api.gax.rpc.ApiException; +import com.google.api.gax.rpc.ApiExceptionFactory; import com.google.api.gax.rpc.OutOfRangeException; import com.google.api.gax.rpc.StatusCode; import com.google.cloud.bigquery.storage.v1.AppendRowsResponse; +import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; +import com.google.cloud.bigquery.storage.v1.FinalizeWriteStreamResponse; +import com.google.cloud.bigquery.storage.v1.FlushRowsRequest; +import com.google.cloud.bigquery.storage.v1.FlushRowsResponse; +import com.google.cloud.bigquery.storage.v1.JsonStreamWriter; +import com.google.cloud.bigquery.storage.v1.StorageError; +import com.google.cloud.bigquery.storage.v1.TableSchema; +import com.google.cloud.bigquery.storage.v1.WriteStream; +import com.google.protobuf.Any; import com.google.rpc.Code; import com.google.rpc.Status; +import io.grpc.protobuf.StatusProto; import java.io.IOException; import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; class BigQuerySinkBatchWriterTest { + private static final String RESTORED_STREAM = + "projects/test-project/datasets/test_dataset/tables/test_table/streams/restored"; + private static final String NEW_STREAM = + "projects/test-project/datasets/test_dataset/tables/test_table/streams/new"; @Test void testAlreadyExistsResponseAdvancesOffsetWithoutRecreatingStream() { @@ -142,6 +182,271 @@ class BigQuerySinkBatchWriterTest { assertEquals(2, sinkWriter.buffer.length()); } + @Test + void testRestoreFlushesCheckpointedRowsAndSwitchesToNewStream() { + BigQueryWriteClient client = mock(BigQueryWriteClient.class); + when(client.flushRows(any(FlushRowsRequest.class))) + .thenReturn(FlushRowsResponse.newBuilder().setOffset(99L).build()); + when(client.finalizeWriteStream(RESTORED_STREAM)) + .thenReturn(FinalizeWriteStreamResponse.newBuilder().setRowCount(900L).build()); + when(client.createWriteStream(anyString(), any(WriteStream.class))) + .thenReturn(WriteStream.newBuilder().setName(NEW_STREAM).build()); + + BigQuerySinkBatchWriter sinkWriter = restoreSinkWriter(client, 100L); + + // Rows up to the checkpoint become visible before the old stream is closed off. Rows + // after it are never flushed, so the replay goes to a new stream from offset 0. + InOrder order = inOrder(client); + ArgumentCaptor<FlushRowsRequest> flushRequest = + ArgumentCaptor.forClass(FlushRowsRequest.class); + order.verify(client).flushRows(flushRequest.capture()); + order.verify(client).finalizeWriteStream(RESTORED_STREAM); + order.verify(client).createWriteStream(anyString(), any(WriteStream.class)); + assertEquals(RESTORED_STREAM, flushRequest.getValue().getWriteStream()); + assertEquals(99L, flushRequest.getValue().getOffset().getValue()); + + BigQueryBatchWriter batchWriter = (BigQueryBatchWriter) sinkWriter.streamWriter; + assertEquals(NEW_STREAM, batchWriter.getStreamName()); + assertEquals(0L, batchWriter.getNextOffset()); + } + + @Test + void testRestoreAcceptsStreamFinalizedByEarlierRestore() { + BigQueryWriteClient client = mock(BigQueryWriteClient.class); + when(client.flushRows(any(FlushRowsRequest.class))).thenThrow(streamFinalized()); + when(client.finalizeWriteStream(RESTORED_STREAM)).thenThrow(streamFinalized()); + when(client.createWriteStream(anyString(), any(WriteStream.class))) + .thenReturn(WriteStream.newBuilder().setName(NEW_STREAM).build()); + + BigQuerySinkBatchWriter sinkWriter = restoreSinkWriter(client, 100L); + + BigQueryBatchWriter batchWriter = (BigQueryBatchWriter) sinkWriter.streamWriter; + assertEquals(NEW_STREAM, batchWriter.getStreamName()); + assertEquals(0L, batchWriter.getNextOffset()); + } + + @Test + void testRestoreFailsWhenCheckpointedRowsCannotBeFlushed() { + BigQueryWriteClient client = mock(BigQueryWriteClient.class); + when(client.flushRows(any(FlushRowsRequest.class))) + .thenThrow( + ApiExceptionFactory.createException( + new IOException("unavailable"), + GrpcStatusCode.of(io.grpc.Status.Code.UNAVAILABLE), + true)); + + assertThrows(BigQueryConnectorException.class, () -> restoreSinkWriter(client, 100L)); + + verify(client, never()).finalizeWriteStream(anyString()); + verify(client, never()).createWriteStream(anyString(), any(WriteStream.class)); + } + + @Test + void testRestoreFailsWhenFlushDoesNotReachCheckpointOffset() { + BigQueryWriteClient client = mock(BigQueryWriteClient.class); + when(client.flushRows(any(FlushRowsRequest.class))) + .thenReturn(FlushRowsResponse.newBuilder().setOffset(50L).build()); + + assertThrows(BigQueryConnectorException.class, () -> restoreSinkWriter(client, 100L)); + + verify(client, never()).finalizeWriteStream(anyString()); + verify(client, never()).createWriteStream(anyString(), any(WriteStream.class)); + } + + @Test + void testSharedClientIsNotClosedByWriter() { + BigQueryWriteClient client = mock(BigQueryWriteClient.class); + when(client.flushRows(any(FlushRowsRequest.class))) + .thenReturn(FlushRowsResponse.newBuilder().setOffset(99L).build()); + when(client.finalizeWriteStream(RESTORED_STREAM)) + .thenReturn(FinalizeWriteStreamResponse.getDefaultInstance()); + when(client.createWriteStream(anyString(), any(WriteStream.class))) + .thenReturn(WriteStream.newBuilder().setName(NEW_STREAM).build()); + + restoreSinkWriter(client, 100L).close(); + + verify(client, never()).close(); + } + + @Test + void testBatchWriterWithoutSharedClientOpensItsOwnClient() throws Exception { + BigQueryWriteClient client = mock(BigQueryWriteClient.class); + when(client.createWriteStream(anyString(), any(WriteStream.class))) + .thenReturn(WriteStream.newBuilder().setName(NEW_STREAM).build()); + JsonStreamWriter jsonStreamWriter = mock(JsonStreamWriter.class); + when(jsonStreamWriter.append(any(JSONArray.class), anyLong())) + .thenReturn(ApiFutures.immediateFuture(AppendRowsResponse.getDefaultInstance())); + BigQuerySinkBatchWriter sinkWriter = + new BigQuerySinkBatchWriter(testConfig(), null, null, null, 0L); + sinkWriter.buffer.put("row"); + + try (MockedStatic<BigQueryClientFactory> clientFactory = mockClientFactory(client); + MockedStatic<TableSchemaUtil> ignored = mockTableSchemaUtil(jsonStreamWriter)) { + sinkWriter.flush(); + } + + verify(jsonStreamWriter).append(any(JSONArray.class), eq(0L)); + BigQueryBatchWriter batchWriter = (BigQueryBatchWriter) sinkWriter.streamWriter; + assertEquals(NEW_STREAM, batchWriter.getStreamName()); + assertEquals(1L, batchWriter.getNextOffset()); + + sinkWriter.close(); + verify(client).close(); + } + + @Test + void testRestoredWriterWithoutSharedClientRetiresStreamAtFirstCheckpoint() { + BigQueryWriteClient client = mock(BigQueryWriteClient.class); + when(client.flushRows(any(FlushRowsRequest.class))) + .thenReturn(FlushRowsResponse.newBuilder().setOffset(99L).build()); + when(client.finalizeWriteStream(RESTORED_STREAM)) + .thenReturn(FinalizeWriteStreamResponse.getDefaultInstance()); + when(client.createWriteStream(anyString(), any(WriteStream.class))) + .thenReturn(WriteStream.newBuilder().setName(NEW_STREAM).build()); + BigQuerySinkBatchWriter sinkWriter = + new BigQuerySinkBatchWriter(testConfig(), null, null, RESTORED_STREAM, 100L); + + // No row arrives after the restore, so the first barrier has to retire the old stream. + try (MockedStatic<BigQueryClientFactory> clientFactory = mockClientFactory(client); + MockedStatic<TableSchemaUtil> ignored = + mockTableSchemaUtil(mock(JsonStreamWriter.class))) { + assertEquals(Optional.empty(), sinkWriter.prepareCommit()); + } + + InOrder order = inOrder(client); + order.verify(client).flushRows(any(FlushRowsRequest.class)); + order.verify(client).finalizeWriteStream(RESTORED_STREAM); + order.verify(client).createWriteStream(anyString(), any(WriteStream.class)); + } + + @Test + void testStreamingWriterWithoutSharedClientDoesNotDropRows() throws Exception { + BigQueryWriteClient client = mock(BigQueryWriteClient.class); + JsonStreamWriter jsonStreamWriter = mock(JsonStreamWriter.class); + when(jsonStreamWriter.append(any(JSONArray.class))) + .thenReturn(ApiFutures.immediateFuture(AppendRowsResponse.getDefaultInstance())); + BigQuerySinkStreamWriter sinkWriter = + new BigQuerySinkStreamWriter(testConfig(), null, null); + sinkWriter.buffer.put("row"); + + try (MockedStatic<BigQueryClientFactory> clientFactory = mockClientFactory(client); + MockedStatic<TableSchemaUtil> ignored = mockTableSchemaUtil(jsonStreamWriter)) { + sinkWriter.prepareCommit(); + } + + verify(jsonStreamWriter).append(any(JSONArray.class)); + assertEquals(0, sinkWriter.buffer.length()); + } + + @Test + void testCommitterAcceptsStreamFinalizedByRestore() { + BigQueryWriteClient client = mock(BigQueryWriteClient.class); + when(client.flushRows(any(FlushRowsRequest.class))).thenThrow(streamFinalized()); + + // An engine that replays commits on restore can reach the stream after it was retired. + try (MockedStatic<BigQueryClientFactory> clientFactory = mockClientFactory(client)) { + new BigQueryCommitter(testConfig()) + .commit( + Collections.singletonList( + new BigQueryCommitInfo(RESTORED_STREAM, 99L))); + } + + verify(client).flushRows(any(FlushRowsRequest.class)); + } + + @Test + void testCommitterFailsWhenFlushFails() { + BigQueryWriteClient client = mock(BigQueryWriteClient.class); + when(client.flushRows(any(FlushRowsRequest.class))) + .thenThrow( + ApiExceptionFactory.createException( + new IOException("unavailable"), + GrpcStatusCode.of(io.grpc.Status.Code.UNAVAILABLE), + true)); + BigQueryCommitter committer = new BigQueryCommitter(testConfig()); + + try (MockedStatic<BigQueryClientFactory> clientFactory = mockClientFactory(client)) { + assertThrows( + BigQueryConnectorException.class, + () -> + committer.commit( + Collections.singletonList( + new BigQueryCommitInfo(RESTORED_STREAM, 99L)))); + } + } + + private static BigQuerySinkBatchWriter restoreSinkWriter( + BigQueryWriteClient client, long restoredNextOffset) { + BigQuerySinkBatchWriter sinkWriter = + new BigQuerySinkBatchWriter( + testConfig(), null, null, RESTORED_STREAM, restoredNextOffset); + + @SuppressWarnings("unchecked") + MultiTableResourceManager<BigQueryWriteClient> manager = + mock(MultiTableResourceManager.class); + when(manager.getSharedResource()).thenReturn(Optional.of(client)); + + try (MockedStatic<TableSchemaUtil> ignored = + mockTableSchemaUtil(mock(JsonStreamWriter.class))) { + sinkWriter.setMultiTableResourceManager(manager, 0); + } + return sinkWriter; + } + + private static ReadonlyConfig testConfig() { + Map<String, Object> options = new HashMap<>(); + options.put(BigQuerySinkOptions.PROJECT_ID.key(), "test-project"); + options.put(BigQuerySinkOptions.DATASET_ID.key(), "test_dataset"); + options.put(BigQuerySinkOptions.TABLE_ID.key(), "test_table"); + return ReadonlyConfig.fromMap(options); + } + + private static MockedStatic<BigQueryClientFactory> mockClientFactory( + BigQueryWriteClient client) { + MockedStatic<BigQueryClientFactory> clientFactory = mockStatic(BigQueryClientFactory.class); + clientFactory + .when(() -> BigQueryClientFactory.getWriteClient(any(ReadonlyConfig.class))) + .thenReturn(client); + return clientFactory; + } + + private static MockedStatic<TableSchemaUtil> mockTableSchemaUtil( + JsonStreamWriter jsonStreamWriter) { + return mockStatic( + TableSchemaUtil.class, + invocation -> { + switch (invocation.getMethod().getName()) { + case "getActualTableSchema": + return TableSchema.getDefaultInstance(); + case "createStreamWriter": + return jsonStreamWriter; + default: + return invocation.callRealMethod(); + } + }); + } + + /** The error a unary call returns when the stream was finalized, as gax surfaces it. */ + private static ApiException streamFinalized() { + Status status = + Status.newBuilder() + .setCode(Code.INVALID_ARGUMENT.getNumber()) + .setMessage("Stream is finalized") + .addDetails( + Any.pack( + StorageError.newBuilder() + .setCode( + StorageError.StorageErrorCode + .STREAM_FINALIZED) + .setEntity(RESTORED_STREAM) + .build())) + .build(); + return ApiExceptionFactory.createException( + StatusProto.toStatusRuntimeException(status), + GrpcStatusCode.of(io.grpc.Status.Code.INVALID_ARGUMENT), + false); + } + private static BigQuerySinkBatchWriter createSinkWriter(BigQueryWriter streamWriter) { return new BigQuerySinkBatchWriter( ReadonlyConfig.fromMap(Collections.emptyMap()), streamWriter, null); diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-bigquery-e2e/src/test/java/org/apache/seatunnel/e2e/connector/bigquery/BigQueryCDCRealIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-bigquery-e2e/src/test/java/org/apache/seatunnel/e2e/connector/bigquery/BigQueryCDCRealIT.java index 7cdc3081fe..527358f2f5 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-bigquery-e2e/src/test/java/org/apache/seatunnel/e2e/connector/bigquery/BigQueryCDCRealIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-bigquery-e2e/src/test/java/org/apache/seatunnel/e2e/connector/bigquery/BigQueryCDCRealIT.java @@ -25,6 +25,7 @@ import org.apache.seatunnel.e2e.common.container.EngineType; import org.apache.seatunnel.e2e.common.container.TestContainer; import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer; import org.apache.seatunnel.e2e.common.junit.TestContainerExtension; +import org.apache.seatunnel.e2e.common.util.JobIdGenerator; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; @@ -60,11 +61,15 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -73,10 +78,12 @@ import java.util.stream.StreamSupport; import static org.awaitility.Awaitility.await; /** - * Runs BigQuery CDC and ADD COLUMN schema evolution against the real BigQuery service. + * Runs BigQuery CDC and ADD COLUMN schema evolution, and the batch-mode restore from a checkpoint, + * against the real BigQuery service. * * <p>This test is opt-in because it requires GCP credentials, creates billable BigQuery jobs, and - * cannot be verified correctly by goccy/bigquery-emulator. + * cannot be verified correctly by goccy/bigquery-emulator. The emulator ignores append offsets and + * inserts the buffered rows again on every FlushRows call. * * <p>Set {@code BIGQUERY_E2E_REAL=true}, {@code BIGQUERY_E2E_PROJECT_ID}, {@code * BIGQUERY_E2E_DATASET_ID}, and {@code GOOGLE_APPLICATION_CREDENTIALS}. The dataset must already @@ -106,6 +113,20 @@ public class BigQueryCDCRealIT extends TestSuiteBase { Pattern.compile("[a-z][a-z0-9-]{4,28}[a-z0-9]"); private static final Pattern DATASET_ID_PATTERN = Pattern.compile("[A-Za-z0-9_]+"); private static final MySqlContainer MYSQL_CONTAINER = createMySqlContainer(MySqlVersion.V8_0); + private static final String BATCH_RESTORE_CONF = "/fake_to_bigquery_batch_real_restore.conf"; + private static final long BATCH_RESTORE_ROW_COUNT = 60L; + private static final int BATCH_RESTORE_TIMEOUT_SECONDS = 300; + private static final long BATCH_RESTORE_CHECKPOINT_INTERVAL_MILLIS = 10_000L; + // Long enough that no later checkpoint completes before the cancel takes effect, and short + // enough that the source is still emitting rows after the first checkpoint. + private static final long CANCELLED_JOB_CHECKPOINT_INTERVAL_MILLIS = 20_000L; + // Longer than a restored job runs, so that job never completes a checkpoint of its own. + private static final long NO_CHECKPOINT_INTERVAL_MILLIS = 3_600_000L; + private static final Pattern FLUSHED_STREAM_PATTERN = + Pattern.compile( + "Successfully flushed BigQuery buffered stream (\\S+) to offset (\\d+)"); + private static final Pattern FINALIZED_STREAM_PATTERN = + Pattern.compile("Finalized restored Buffered write stream (\\S+) with (\\d+) rows"); private String projectId; private String datasetId; @@ -325,6 +346,304 @@ public class BigQueryCDCRealIT extends TestSuiteBase { }); } + /** + * Cancels a batch-mode job after rows were appended past its last completed checkpoint, then + * restores the job from that checkpoint. The rows left unflushed in the old buffered stream + * must stay invisible, and the replay must make every remaining row visible exactly once. + */ + @TestTemplate + @DisabledOnContainer( + value = {}, + type = {EngineType.SPARK, EngineType.FLINK}, + disabledReason = + "Cancelling and restoring a job are only implemented by the SeaTunnel test container") + public void testBatchModeRestoreAfterUncheckpointedAppends(TestContainer container) + throws Exception { + createBatchRestoreTable(); + String jobId = runBatchRestoreJobUntilCancelled(container); + + assertRestoredJobWritesEveryRowOnce(container, jobId); + } + + /** + * Restores one checkpoint twice. The first restored job finalizes the buffered stream recorded + * in that checkpoint and is cancelled without completing a checkpoint of its own. The second + * restored job retires the same stream again, so FlushRows and FinalizeWriteStream reach a + * stream that is already finalized. The real service must answer in a way the sink accepts, and + * every row must still become visible exactly once. + */ + @TestTemplate + @DisabledOnContainer( + value = {}, + type = {EngineType.SPARK, EngineType.FLINK}, + disabledReason = + "Cancelling and restoring a job are only implemented by the SeaTunnel test container") + public void testBatchModeRestoreOfAlreadyFinalizedStream(TestContainer container) + throws Exception { + createBatchRestoreTable(); + String jobId = runBatchRestoreJobUntilCancelled(container); + + String firstRestoredJobId = String.valueOf(JobIdGenerator.newJobId()); + CompletableFuture<Container.ExecResult> firstRestoredRun = + restoreBatchJob( + container, jobId, firstRestoredJobId, NO_CHECKPOINT_INTERVAL_MILLIS); + try { + await().atMost(BATCH_RESTORE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .until( + () -> + countStreamRetirements(container) >= 1 + && "RUNNING" + .equals( + container.getJobStatus( + firstRestoredJobId))); + Assertions.assertEquals( + 0L, + container.getCompletedCheckpointCount(firstRestoredJobId), + "The first restored job must not commit rows of its own"); + } finally { + cancelBatchJob(container, firstRestoredJobId, firstRestoredRun); + } + + assertRestoredJobWritesEveryRowOnce(container, jobId); + + long retirements = countStreamRetirements(container); + Assertions.assertTrue( + retirements >= 2, + "The second restored job must retire the finalized stream again, but " + + retirements + + " retirements were logged"); + } + + /** + * Runs the batch restore job until the server logs show rows appended past its committed + * checkpoint, then cancels it. A later checkpoint can still cover those rows before the cancel + * takes effect, so {@link #assertRetiredStreamHeldUncheckpointedRows} checks the restored + * checkpoint afterwards. + * + * @return the id of the cancelled job + */ + private String runBatchRestoreJobUntilCancelled(TestContainer container) throws Exception { + String jobId = String.valueOf(JobIdGenerator.newJobId()); + String[] variables = + batchRestoreVariables(CANCELLED_JOB_CHECKPOINT_INTERVAL_MILLIS) + .toArray(new String[0]); + + CompletableFuture<Container.ExecResult> firstRun = + CompletableFuture.supplyAsync( + () -> { + try { + return container.executeJob(BATCH_RESTORE_CONF, jobId, variables); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + try { + await().atMost(INITIAL_SNAPSHOT_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .until(() -> container.getCompletedCheckpointCount(jobId) > 0); + await("rows appended past the committed checkpoint") + .atMost(2, TimeUnit.MINUTES) + .pollInterval(1, TimeUnit.SECONDS) + .until(() -> hasUncheckpointedRows(container, jobId)); + } finally { + cancelBatchJob(container, jobId, firstRun); + } + + long rowsBeforeRestore = countRows(); + Assertions.assertTrue( + rowsBeforeRestore > 0 && rowsBeforeRestore < BATCH_RESTORE_ROW_COUNT, + "The job must be cancelled after the first commit and before every row is written," + + " but " + + rowsBeforeRestore + + " rows were visible"); + return jobId; + } + + private void assertRestoredJobWritesEveryRowOnce(TestContainer container, String sourceJobId) + throws Exception { + List<Long> idsBeforeRestore = readRowIds(); + // A job restored from a checkpoint runs under a new job id. + String restoredJobId = String.valueOf(JobIdGenerator.newJobId()); + CompletableFuture<Container.ExecResult> restoredRun = + restoreBatchJob( + container, + sourceJobId, + restoredJobId, + BATCH_RESTORE_CHECKPOINT_INTERVAL_MILLIS); + try { + await().atMost(BATCH_RESTORE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .pollInterval(2, TimeUnit.SECONDS) + .untilAsserted( + () -> Assertions.assertEquals(BATCH_RESTORE_ROW_COUNT, countRows())); + + // Later checkpoints flush the same offsets again, which must not add rows. + long checkpointsAtFullCount = container.getCompletedCheckpointCount(restoredJobId); + await().atMost(2, TimeUnit.MINUTES) + .pollInterval(1, TimeUnit.SECONDS) + .until( + () -> + container.getCompletedCheckpointCount(restoredJobId) + > checkpointsAtFullCount); + + // FakeSource draws a new random id for every row, replayed rows included, so ids can + // only show that committed rows survive and that no row became visible twice. + List<Long> idsAfterRestore = readRowIds(); + Assertions.assertEquals(BATCH_RESTORE_ROW_COUNT, idsAfterRestore.size()); + Assertions.assertEquals( + BATCH_RESTORE_ROW_COUNT, + new HashSet<>(idsAfterRestore).size(), + "A row became visible more than once"); + Assertions.assertTrue( + idsAfterRestore.containsAll(idsBeforeRestore), + "A row committed before the restore disappeared"); + assertRetiredStreamHeldUncheckpointedRows(container); + } finally { + container.cancelJob(restoredJobId); + restoredRun.get(2, TimeUnit.MINUTES); + } + } + + /** + * Proves that the restore ran inside the failure window: the retired stream held more rows than + * the restored checkpoint flushed. BigQuery reports the row count of a stream only when it is + * first finalized, so this can only be checked after the restore. + */ + private void assertRetiredStreamHeldUncheckpointedRows(TestContainer container) { + Map<String, Long> finalizedRows = + maxLoggedValuePerStream(container, FINALIZED_STREAM_PATTERN); + Assertions.assertEquals(1, finalizedRows.size(), "Exactly one stream must be retired"); + + Map.Entry<String, Long> retiredStream = finalizedRows.entrySet().iterator().next(); + long flushedRows = + maxLoggedValuePerStream(container, FLUSHED_STREAM_PATTERN) + .get(retiredStream.getKey()) + + 1; + Assertions.assertTrue( + retiredStream.getValue() > flushedRows, + "The cancelled job must leave rows past its checkpoint, but the stream held " + + retiredStream.getValue() + + " rows and the checkpoint covered " + + flushedRows); + } + + private CompletableFuture<Container.ExecResult> restoreBatchJob( + TestContainer container, + String sourceJobId, + String restoredJobId, + long checkpointIntervalMillis) { + List<String> variables = batchRestoreVariables(checkpointIntervalMillis); + return CompletableFuture.supplyAsync( + () -> { + try { + return container.restoreJobWithCheckpoint( + BATCH_RESTORE_CONF, sourceJobId, restoredJobId, variables); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + private void cancelBatchJob( + TestContainer container, String jobId, CompletableFuture<Container.ExecResult> run) + throws Exception { + Container.ExecResult cancelResult = container.cancelJob(jobId); + Assertions.assertEquals(0, cancelResult.getExitCode(), cancelResult.getStderr()); + await().atMost(2, TimeUnit.MINUTES) + .pollInterval(1, TimeUnit.SECONDS) + .until(() -> "CANCELED".equals(container.getJobStatus(jobId))); + run.get(2, TimeUnit.MINUTES); + } + + private List<String> batchRestoreVariables(long checkpointIntervalMillis) { + return Arrays.asList( + "bigquery_project_id=" + projectId, + "bigquery_dataset_id=" + datasetId, + "bigquery_table_id=" + tableId.getTable(), + "checkpoint_interval=" + checkpointIntervalMillis); + } + + /** Tells whether a job appended more rows than its committed checkpoint made visible. */ + private boolean hasUncheckpointedRows(TestContainer container, String jobId) { + long appendedRows = countAppendedRows(container, jobId); + return maxLoggedValuePerStream(container, FLUSHED_STREAM_PATTERN).values().stream() + .anyMatch(flushedOffset -> appendedRows > flushedOffset + 1); + } + + /** Counts the rows a job appended. The restore config appends one row per call. */ + private long countAppendedRows(TestContainer container, String jobId) { + String jobLogPrefix = "[" + jobId + "]"; + return Arrays.stream(container.getServerLogs().split("\n")) + .filter(line -> line.contains(jobLogPrefix)) + .filter(line -> line.contains("Successfully appended 1 rows.")) + .count(); + } + + /** Collects the largest value the pattern logged for each stream of the current table. */ + private Map<String, Long> maxLoggedValuePerStream(TestContainer container, Pattern pattern) { + String streamsOfTable = "/tables/" + tableId.getTable() + "/streams/"; + Map<String, Long> values = new HashMap<>(); + Matcher matcher = pattern.matcher(container.getServerLogs()); + while (matcher.find()) { + String stream = matcher.group(1); + if (stream.contains(streamsOfTable)) { + values.merge(stream, Long.parseLong(matcher.group(2)), Math::max); + } + } + return values; + } + + /** Counts how often a stream of the current table was retired, whatever BigQuery answered. */ + private long countStreamRetirements(TestContainer container) { + String streamsOfTable = "/tables/" + tableId.getTable() + "/streams/"; + return Arrays.stream(container.getServerLogs().split("\n")) + .filter(line -> line.contains(streamsOfTable)) + .filter( + line -> + line.contains("Finalized restored Buffered write stream") + || line.contains("was already finalized")) + .count(); + } + + private void createBatchRestoreTable() throws InterruptedException { + tableId = + TableId.of( + projectId, + datasetId, + "seatunnel_batch_restore_" + UUID.randomUUID().toString().replace("-", "")); + String sql = + String.format( + "CREATE TABLE `%s.%s.%s` (id INT64, name STRING) OPTIONS (" + + "expiration_timestamp = " + + "TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 1 DAY)" + + ")", + projectId, datasetId, tableId.getTable()); + bigQuery.query(QueryJobConfiguration.of(sql)); + } + + private long countRows() throws InterruptedException { + String query = + String.format( + "SELECT COUNT(*) FROM `%s.%s.%s`", + projectId, datasetId, tableId.getTable()); + TableResult result = + bigQuery.query( + QueryJobConfiguration.newBuilder(query).setUseQueryCache(false).build()); + return result.iterateAll().iterator().next().get(0).getLongValue(); + } + + private List<Long> readRowIds() throws InterruptedException { + String query = + String.format( + "SELECT id FROM `%s.%s.%s`", projectId, datasetId, tableId.getTable()); + TableResult result = + bigQuery.query( + QueryJobConfiguration.newBuilder(query).setUseQueryCache(false).build()); + return StreamSupport.stream(result.iterateAll().spliterator(), false) + .map(row -> row.get(0).getLongValue()) + .collect(Collectors.toList()); + } + private void createBigQueryTable() throws InterruptedException { String sql = String.format( diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-bigquery-e2e/src/test/resources/fake_to_bigquery_batch_real_restore.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-bigquery-e2e/src/test/resources/fake_to_bigquery_batch_real_restore.conf new file mode 100644 index 0000000000..3853559aa5 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-bigquery-e2e/src/test/resources/fake_to_bigquery_batch_real_restore.conf @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 1 + job.mode = "STREAMING" + # Set by the test. It is long enough for rows to be appended after the last completed + # checkpoint before the test cancels the job. + checkpoint.interval = ${checkpoint_interval} + checkpoint.retain-after-job-cancelled = true +} + +source { + FakeSource { + row.num = 60 + split.num = 30 + split.read-interval = 1000 + schema = { + fields { + id = bigint + name = string + } + } + } +} + +sink { + BigQuery { + project_id = "${bigquery_project_id}" + dataset_id = "${bigquery_dataset_id}" + table_id = "${bigquery_table_id}" + service_account_key_path = "/tmp/seatunnel/bigquery-service-account.json" + write_mode = "batch" + # Every row is appended to the buffered stream as soon as it arrives. + batch_size = 1 + } +} diff --git a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/TestContainer.java b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/TestContainer.java index db5a8ce100..92172a4e8c 100644 --- a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/TestContainer.java +++ b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/TestContainer.java @@ -81,6 +81,22 @@ public interface TestContainer extends TestResource { throw new UnsupportedOperationException("Not implemented"); } + /** + * Starts a new job from the latest checkpoint of an ended job. + * + * @param confFile job config file + * @param sourceJobId ended job whose checkpoint is restored + * @param restoreJobId id of the new job, which must differ from {@code sourceJobId} + * @param variables {@code key=value} entries passed to the job config as {@code -i} options, or + * null if the config has no variables + * @return result of the restore command + */ + default Container.ExecResult restoreJobWithCheckpoint( + String confFile, String sourceJobId, String restoreJobId, List<String> variables) + throws IOException, InterruptedException { + throw new UnsupportedOperationException("Not implemented"); + } + default Container.ExecResult cancelJob(String jobId) throws IOException, InterruptedException { throw new UnsupportedOperationException("Not implemented"); } diff --git a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java index 6a2846589e..773f3aab18 100644 --- a/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java +++ b/seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/seatunnel/SeaTunnelContainer.java @@ -959,10 +959,22 @@ public class SeaTunnelContainer extends AbstractTestContainer implements Reusabl public Container.ExecResult restoreJobWithCheckpoint( String confFile, String sourceJobId, String restoreJobId) throws IOException, InterruptedException { + return restoreJobWithCheckpoint(confFile, sourceJobId, restoreJobId, (List<String>) null); + } + + @Override + public Container.ExecResult restoreJobWithCheckpoint( + String confFile, String sourceJobId, String restoreJobId, List<String> variables) + throws IOException, InterruptedException { runningCount.incrementAndGet(); try { return restoreJob( - server, confFile, sourceJobId, restoreJobId, null, "--restore-with-checkpoint"); + server, + confFile, + sourceJobId, + restoreJobId, + variables, + "--restore-with-checkpoint"); } finally { runningCount.decrementAndGet(); }
