morningman commented on code in PR #27154:
URL: https://github.com/apache/doris/pull/27154#discussion_r1405409179
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/external/MaxComputeExternalTable.java:
##########
@@ -73,85 +72,105 @@ protected synchronized void makeSureInitialized() {
super.makeSureInitialized();
if (!objectCreated) {
odpsTable = ((MaxComputeExternalCatalog)
catalog).getClient().tables().get(name);
+ initTablePartitions();
objectCreated = true;
}
}
+ public long getTotalRows() throws TunnelException {
+ // use for non-partitioned table
+ makeSureInitialized();
+ MaxComputeMetadataCache metadataCache =
Env.getCurrentEnv().getExtMetaCacheMgr()
+ .getMaxComputeMetadataCache(catalog.getId());
+ MaxComputeExternalCatalog mcCatalog = ((MaxComputeExternalCatalog)
catalog);
+ return metadataCache.getCachedRowCount(dbName, name, null, () ->
mcCatalog.getTableTunnel()
+ .getDownloadSession(dbName, name, null)
+ .getRecordCount());
+ }
+
+ @Override
+ public Set<String> getPartitionNames() {
+ makeSureInitialized();
+ return partitionNameToColumns.keySet();
+ }
+
+ public List<Column> getPartitionColumns() {
+ makeSureInitialized();
+ return new ArrayList<>(partitionNameToColumns.values());
+ }
+
+ public TablePartitionValues getPartitionValues() {
+ makeSureInitialized();
+ // Make sure to call it after initSchema() completes
+ String projectName = odpsTable.getProject();
+ String tableName = odpsTable.getName();
+ MaxComputeMetadataCache metadataCache =
Env.getCurrentEnv().getExtMetaCacheMgr()
+ .getMaxComputeMetadataCache(catalog.getId());
+ return metadataCache.getCachedPartitionValues(
+ new MaxComputeCacheKey(projectName, tableName),
+ () -> {
+ TablePartitionValues partitionValues = new
TablePartitionValues();
+ partitionValues.addPartitions(partitionSpecs,
+ partitionSpecs.stream()
+ .map(p -> parsePartitionValues(new
ArrayList<>(getPartitionNames()), p))
+ .collect(Collectors.toList()),
+ partitionTypes);
+ return partitionValues;
+ });
+ }
+
+ private static List<String> parsePartitionValues(List<String>
partitionColumns, String partitionPath) {
Review Comment:
Add some comment in code to explain the partition format and how to parse it.
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/external/MaxComputeExternalTable.java:
##########
@@ -73,85 +72,105 @@ protected synchronized void makeSureInitialized() {
super.makeSureInitialized();
if (!objectCreated) {
odpsTable = ((MaxComputeExternalCatalog)
catalog).getClient().tables().get(name);
+ initTablePartitions();
objectCreated = true;
}
}
+ public long getTotalRows() throws TunnelException {
+ // use for non-partitioned table
Review Comment:
What about partitioned table?
##########
fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java:
##########
@@ -173,6 +173,11 @@ public void open() throws IOException {
.collect(Collectors.toSet());
List<Column> maxComputeColumns = new ArrayList<>(readColumns);
maxComputeColumns.removeIf(e ->
partitionColumns.contains(e.getName()));
+ if (maxComputeColumns.isEmpty() && !partitionColumns.isEmpty()) {
Review Comment:
how can maxComputeColumns be empty
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
