This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 0f53ae04db Added import IT that uses same table name in a different
namespace (#5333)
0f53ae04db is described below
commit 0f53ae04db71b20b939f8fcb67d47c85ef96c9c6
Author: Dave Marion <[email protected]>
AuthorDate: Thu Feb 13 14:12:04 2025 -0500
Added import IT that uses same table name in a different namespace (#5333)
Closes #5326
---
.../org/apache/accumulo/test/ImportExportIT.java | 214 +++++++++++----------
1 file changed, 115 insertions(+), 99 deletions(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java
b/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java
index 49f01c1b36..995474dcc1 100644
--- a/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java
@@ -85,124 +85,140 @@ public class ImportExportIT extends
AccumuloClusterHarness {
@Test
public void testExportImportThenScan() throws Exception {
try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
-
String[] tableNames = getUniqueNames(2);
- String srcTable = tableNames[0], destTable = tableNames[1];
- client.tableOperations().create(srcTable);
-
- try (BatchWriter bw = client.createBatchWriter(srcTable)) {
- for (int row = 0; row < 1000; row++) {
- Mutation m = new Mutation(Integer.toString(row));
- for (int col = 0; col < 100; col++) {
- m.put(Integer.toString(col), "", Integer.toString(col * 2));
- }
- bw.addMutation(m);
- }
- }
-
- client.tableOperations().compact(srcTable, null, null, true, true);
-
- // Make a directory we can use to throw the export and import directories
- // Must exist on the filesystem the cluster is running.
- FileSystem fs = cluster.getFileSystem();
- log.info("Using FileSystem: " + fs);
- Path baseDir = new Path(cluster.getTemporaryPath(),
getClass().getName());
- fs.deleteOnExit(baseDir);
- if (fs.exists(baseDir)) {
- log.info("{} exists on filesystem, deleting", baseDir);
- assertTrue(fs.delete(baseDir, true), "Failed to deleted " + baseDir);
- }
- log.info("Creating {}", baseDir);
- assertTrue(fs.mkdirs(baseDir), "Failed to create " + baseDir);
- Path exportDir = new Path(baseDir, "export");
- fs.deleteOnExit(exportDir);
- Path importDirA = new Path(baseDir, "import-a");
- Path importDirB = new Path(baseDir, "import-b");
- fs.deleteOnExit(importDirA);
- fs.deleteOnExit(importDirB);
- for (Path p : new Path[] {exportDir, importDirA, importDirB}) {
- assertTrue(fs.mkdirs(p), "Failed to create " + p);
- }
-
- Set<String> importDirs = Set.of(importDirA.toString(),
importDirB.toString());
+ doExportImportThenScan(client, tableNames[0], tableNames[1]);
+ }
+ }
- Path[] importDirAry = new Path[] {importDirA, importDirB};
+ @Test
+ public void testExportImportSameTableNameThenScan() throws Exception {
+ try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ String ns1 = "namespace1";
+ client.namespaceOperations().create(ns1);
+ String ns2 = "namespace2";
+ client.namespaceOperations().create(ns2);
+ String tableName = getUniqueNames(1)[0];
+ doExportImportThenScan(client, ns1 + "." + tableName, ns2 + "." +
tableName);
+ }
+ }
- log.info("Exporting table to {}", exportDir);
- log.info("Importing table from {}", importDirs);
+ public void doExportImportThenScan(AccumuloClient client, String srcTable,
String destTable)
+ throws Exception {
- // test fast fail offline check
- assertThrows(IllegalStateException.class,
- () -> client.tableOperations().exportTable(srcTable,
exportDir.toString()));
+ client.tableOperations().create(srcTable);
- // Offline the table
- client.tableOperations().offline(srcTable, true);
- // Then export it
- client.tableOperations().exportTable(srcTable, exportDir.toString());
+ try (BatchWriter bw = client.createBatchWriter(srcTable)) {
+ for (int row = 0; row < 1000; row++) {
+ Mutation m = new Mutation(Integer.toString(row));
+ for (int col = 0; col < 100; col++) {
+ m.put(Integer.toString(col), "", Integer.toString(col * 2));
+ }
+ bw.addMutation(m);
+ }
+ }
- // Make sure the distcp.txt file that exporttable creates is available
- Path distcp = new Path(exportDir, "distcp.txt");
- fs.deleteOnExit(distcp);
- assertTrue(fs.exists(distcp), "Distcp file doesn't exist");
- FSDataInputStream is = fs.open(distcp);
- BufferedReader reader = new BufferedReader(new InputStreamReader(is));
+ client.tableOperations().compact(srcTable, null, null, true, true);
+
+ // Make a directory we can use to throw the export and import directories
+ // Must exist on the filesystem the cluster is running.
+ FileSystem fs = cluster.getFileSystem();
+ log.info("Using FileSystem: " + fs);
+ Path baseDir = new Path(cluster.getTemporaryPath(), getClass().getName());
+ fs.deleteOnExit(baseDir);
+ if (fs.exists(baseDir)) {
+ log.info("{} exists on filesystem, deleting", baseDir);
+ assertTrue(fs.delete(baseDir, true), "Failed to deleted " + baseDir);
+ }
+ log.info("Creating {}", baseDir);
+ assertTrue(fs.mkdirs(baseDir), "Failed to create " + baseDir);
+ Path exportDir = new Path(baseDir, "export");
+ fs.deleteOnExit(exportDir);
+ Path importDirA = new Path(baseDir, "import-a");
+ Path importDirB = new Path(baseDir, "import-b");
+ fs.deleteOnExit(importDirA);
+ fs.deleteOnExit(importDirB);
+ for (Path p : new Path[] {exportDir, importDirA, importDirB}) {
+ assertTrue(fs.mkdirs(p), "Failed to create " + p);
+ }
- // Copy each file that was exported to one of the imports directory
- String line;
+ Set<String> importDirs = Set.of(importDirA.toString(),
importDirB.toString());
- while ((line = reader.readLine()) != null) {
- Path p = new Path(line.substring(5));
- assertTrue(fs.exists(p), "File doesn't exist: " + p);
- Path importDir = importDirAry[random.nextInt(importDirAry.length)];
- Path dest = new Path(importDir, p.getName());
- assertFalse(fs.exists(dest), "Did not expect " + dest + " to exist");
- FileUtil.copy(fs, p, fs, dest, false, fs.getConf());
- }
+ Path[] importDirAry = new Path[] {importDirA, importDirB};
- reader.close();
+ log.info("Exporting table to {}", exportDir);
+ log.info("Importing table from {}", importDirs);
- log.info("Import dir A: {}", Arrays.toString(fs.listStatus(importDirA)));
- log.info("Import dir B: {}", Arrays.toString(fs.listStatus(importDirB)));
+ // test fast fail offline check
+ assertThrows(IllegalStateException.class,
+ () -> client.tableOperations().exportTable(srcTable,
exportDir.toString()));
- // Import the exported data into a new table
- client.tableOperations().importTable(destTable, importDirs,
ImportConfiguration.empty());
+ // Offline the table
+ client.tableOperations().offline(srcTable, true);
+ // Then export it
+ client.tableOperations().exportTable(srcTable, exportDir.toString());
- // Get the table ID for the table that the importtable command created
- final String tableId =
client.tableOperations().tableIdMap().get(destTable);
- assertNotNull(tableId);
+ // Make sure the distcp.txt file that exporttable creates is available
+ Path distcp = new Path(exportDir, "distcp.txt");
+ fs.deleteOnExit(distcp);
+ assertTrue(fs.exists(distcp), "Distcp file doesn't exist");
+ FSDataInputStream is = fs.open(distcp);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(is));
- // Get all `file` colfams from the metadata table for the new table
- log.info("Imported into table with ID: {}", tableId);
+ // Copy each file that was exported to one of the imports directory
+ String line;
- try (Scanner s = client.createScanner(MetadataTable.NAME,
Authorizations.EMPTY)) {
- s.setRange(TabletsSection.getRange(TableId.of(tableId)));
- s.fetchColumnFamily(DataFileColumnFamily.NAME);
- ServerColumnFamily.DIRECTORY_COLUMN.fetch(s);
+ while ((line = reader.readLine()) != null) {
+ Path p = new Path(line.substring(5));
+ assertTrue(fs.exists(p), "File doesn't exist: " + p);
+ Path importDir = importDirAry[random.nextInt(importDirAry.length)];
+ Path dest = new Path(importDir, p.getName());
+ assertFalse(fs.exists(dest), "Did not expect " + dest + " to exist");
+ FileUtil.copy(fs, p, fs, dest, false, fs.getConf());
+ }
- // Should find a single entry
- for (Entry<Key,Value> fileEntry : s) {
- Key k = fileEntry.getKey();
- String value = fileEntry.getValue().toString();
- if (k.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
- // The file should be an absolute URI (file:///...), not a
relative path
- // (/b-000.../I000001.rf)
- String fileUri = k.getColumnQualifier().toString();
- assertFalse(looksLikeRelativePath(fileUri),
- "Imported files should have absolute URIs, not relative: " +
fileUri);
- } else if (k.getColumnFamily().equals(ServerColumnFamily.NAME)) {
- assertFalse(looksLikeRelativePath(value),
- "Server directory should have absolute URI, not relative: " +
value);
- } else {
- fail("Got expected pair: " + k + "=" + fileEntry.getValue());
- }
+ reader.close();
+
+ log.info("Import dir A: {}", Arrays.toString(fs.listStatus(importDirA)));
+ log.info("Import dir B: {}", Arrays.toString(fs.listStatus(importDirB)));
+
+ // Import the exported data into a new table
+ client.tableOperations().importTable(destTable, importDirs,
ImportConfiguration.empty());
+
+ // Get the table ID for the table that the importtable command created
+ final String tableId =
client.tableOperations().tableIdMap().get(destTable);
+ assertNotNull(tableId);
+
+ // Get all `file` colfams from the metadata table for the new table
+ log.info("Imported into table with ID: {}", tableId);
+
+ try (Scanner s = client.createScanner(MetadataTable.NAME,
Authorizations.EMPTY)) {
+ s.setRange(TabletsSection.getRange(TableId.of(tableId)));
+ s.fetchColumnFamily(DataFileColumnFamily.NAME);
+ ServerColumnFamily.DIRECTORY_COLUMN.fetch(s);
+
+ // Should find a single entry
+ for (Entry<Key,Value> fileEntry : s) {
+ Key k = fileEntry.getKey();
+ String value = fileEntry.getValue().toString();
+ if (k.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
+ // The file should be an absolute URI (file:///...), not a relative
path
+ // (/b-000.../I000001.rf)
+ String fileUri = k.getColumnQualifier().toString();
+ assertFalse(looksLikeRelativePath(fileUri),
+ "Imported files should have absolute URIs, not relative: " +
fileUri);
+ } else if (k.getColumnFamily().equals(ServerColumnFamily.NAME)) {
+ assertFalse(looksLikeRelativePath(value),
+ "Server directory should have absolute URI, not relative: " +
value);
+ } else {
+ fail("Got expected pair: " + k + "=" + fileEntry.getValue());
}
-
}
- // Online the original table before we verify equivalence
- client.tableOperations().online(srcTable, true);
- verifyTableEquality(client, srcTable, destTable);
}
+ // Online the original table before we verify equivalence
+ client.tableOperations().online(srcTable, true);
+
+ verifyTableEquality(client, srcTable, destTable);
}
@Test
