liuxiao2shf commented on code in PR #3619: URL: https://github.com/apache/flink-cdc/pull/3619#discussion_r1793008455
########## flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/metrics/SourceEnumeratorMetrics.java: ########## @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.base.source.metrics; + +import org.apache.flink.metrics.Gauge; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.metrics.groups.SplitEnumeratorMetricGroup; + +import io.debezium.relational.TableId; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** A collection class for handling metrics in {@link SourceEnumeratorMetrics}. */ +public class SourceEnumeratorMetrics { + private static final Logger LOGGER = LoggerFactory.getLogger(SourceEnumeratorMetrics.class); + // Constants + public static final int UNDEFINED = 0; + + // Metric names + public static final String IS_SNAPSHOTTING = "isSnapshotting"; + public static final String IS_STREAM_READING = "isStreamReading"; + public static final String NUM_TABLES_SNAPSHOTTED = "numTablesSnapshotted"; + public static final String NUM_TABLES_REMAINING = "numTablesRemaining"; + public static final String NUM_SNAPSHOT_SPLITS_PROCESSED = "numSnapshotSplitsProcessed"; + public static final String NUM_SNAPSHOT_SPLITS_REMAINING = "numSnapshotSplitsRemaining"; + public static final String NUM_SNAPSHOT_SPLITS_FINISHED = "numSnapshotSplitsFinished"; + public static final String SNAPSHOT_START_TIME = "snapshotStartTime"; + public static final String SNAPSHOT_END_TIME = "snapshotEndTime"; + public static final String DATABASE_GROUP_KEY = "database"; + public static final String SCHEMA_GROUP_KEY = "schema"; + public static final String TABLE_GROUP_KEY = "table"; + + private final SplitEnumeratorMetricGroup metricGroup; + + private volatile int isSnapshotting = UNDEFINED; + private volatile int isStreamReading = UNDEFINED; + private volatile int numTablesRemaining = 0; + + // Map for managing per-table metrics by table identifier + // Key: Identifier of the table + // Value: TableMetrics related to the table + private final Map<TableId, TableMetrics> tableMetricsMap = new ConcurrentHashMap<>(); + + public SourceEnumeratorMetrics(SplitEnumeratorMetricGroup metricGroup) { + this.metricGroup = metricGroup; + metricGroup.gauge(IS_SNAPSHOTTING, () -> isSnapshotting); + metricGroup.gauge(IS_STREAM_READING, () -> isStreamReading); + metricGroup.gauge(NUM_TABLES_REMAINING, () -> numTablesRemaining); + } + + public void enterSnapshotPhase() { + this.isSnapshotting = 1; + } + + public void exitSnapshotPhase() { + this.isSnapshotting = 0; + } + + public void enterStreamReading() { + this.isStreamReading = 1; + } + + public void exitStreamReading() { + this.isStreamReading = 0; + } + + public void registerMetrics( + Gauge<Integer> numTablesSnapshotted, + Gauge<Integer> numSnapshotSplitsProcessed, + Gauge<Integer> numSnapshotSplitsRemaining) { + metricGroup.gauge(NUM_TABLES_SNAPSHOTTED, numTablesSnapshotted); + metricGroup.gauge(NUM_SNAPSHOT_SPLITS_PROCESSED, numSnapshotSplitsProcessed); + metricGroup.gauge(NUM_SNAPSHOT_SPLITS_REMAINING, numSnapshotSplitsRemaining); + } + + public void addNewTables(int numNewTables) { + numTablesRemaining += numNewTables; + } + + public void startSnapshotTables(int numSnapshottedTables) { + numTablesRemaining -= numSnapshottedTables; + } + + public TableMetrics getTableMetrics(TableId tableId) { + return tableMetricsMap.computeIfAbsent( + tableId, + key -> new TableMetrics(key.catalog(), key.schema(), key.table(), metricGroup)); + } + + // ----------------------------------- Helper classes -------------------------------- + + /** + * Collection class for managing metrics of a table. + * + * <p>Metrics of table level are registered in its corresponding subgroup under the {@link + * SplitEnumeratorMetricGroup}. + */ + public static class TableMetrics { + private AtomicInteger numSnapshotSplitsProcessed = new AtomicInteger(0); + private AtomicInteger numSnapshotSplitsRemaining = new AtomicInteger(0); + private AtomicInteger numSnapshotSplitsFinished = new AtomicInteger(0); + private volatile long snapshotStartTime = UNDEFINED; + private volatile long snapshotEndTime = UNDEFINED; + + private Set<String> remainingSplitIds = + Collections.newSetFromMap(new ConcurrentHashMap<>()); + private Set<String> processedSplitIds = + Collections.newSetFromMap(new ConcurrentHashMap<>()); + private Set<String> finishedSplitIds = Collections.newSetFromMap(new ConcurrentHashMap<>()); + + public TableMetrics( + String databaseName, String schemaName, String tableName, MetricGroup parentGroup) { + databaseName = processNull(databaseName); + schemaName = processNull(schemaName); + tableName = processNull(tableName); + MetricGroup metricGroup = + parentGroup + .addGroup(DATABASE_GROUP_KEY, databaseName) + .addGroup(SCHEMA_GROUP_KEY, schemaName) + .addGroup(TABLE_GROUP_KEY, tableName); + metricGroup.gauge( + NUM_SNAPSHOT_SPLITS_PROCESSED, () -> numSnapshotSplitsProcessed.intValue()); + metricGroup.gauge( + NUM_SNAPSHOT_SPLITS_REMAINING, () -> numSnapshotSplitsRemaining.intValue()); + metricGroup.gauge( + NUM_SNAPSHOT_SPLITS_FINISHED, () -> numSnapshotSplitsFinished.intValue()); + metricGroup.gauge(SNAPSHOT_START_TIME, () -> snapshotStartTime); + metricGroup.gauge(SNAPSHOT_END_TIME, () -> snapshotEndTime); + snapshotStartTime = System.currentTimeMillis(); + } + + private String processNull(String name) { + if (StringUtils.isBlank(name)) { + // If null, convert to an empty string + return ""; + } + return name; + } + + public void addNewSplit(String newSplitId) { + if (!remainingSplitIds.contains(newSplitId)) { + remainingSplitIds.add(newSplitId); + numSnapshotSplitsRemaining.getAndAdd(1); + LOGGER.info("add remaining split: {}", newSplitId); + } + } + + public void addNewSplits(List<String> newSplitIds) { + if (newSplitIds != null) { + for (String newSplitId : newSplitIds) { + addNewSplit(newSplitId); Review Comment: 1. When adding, it is necessary to determine whether it is already inside the collection, and using `addAll` is actually more complicated 2.`addAll` is also internally processed in a loop, We will only experience performance loss from executing `getAndAdd` method calls multiple times -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@flink.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
