adoroszlai commented on code in PR #8287: URL: https://github.com/apache/ozone/pull/8287#discussion_r2077120860
########## hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/AbstractNSSummaryTaskTest.java: ########## @@ -0,0 +1,767 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.tasks; + +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProvider; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProviderWithFSO; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; +import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.db.RDBBatchOperation; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmConfig; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils; +import org.apache.hadoop.ozone.recon.ReconConstants; +import org.apache.hadoop.ozone.recon.ReconTestInjector; +import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; + +/** + * Abstract Class created to handle common objects and methods. Review Comment: nit: This is pretty vague, the same can be stated about any abstract class. ########## hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/tasks/AbstractNSSummaryTaskTest.java: ########## @@ -0,0 +1,767 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.recon.tasks; + +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProvider; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getMockOzoneManagerServiceProviderWithFSO; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.getTestReconOmMetadataManager; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeDirToOm; +import static org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils.writeKeyToOm; +import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.utils.db.RDBBatchOperation; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmConfig; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.recon.OMMetadataManagerTestUtils; +import org.apache.hadoop.ozone.recon.ReconConstants; +import org.apache.hadoop.ozone.recon.ReconTestInjector; +import org.apache.hadoop.ozone.recon.api.types.NSSummary; +import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager; +import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager; +import org.apache.hadoop.ozone.recon.spi.impl.OzoneManagerServiceProviderImpl; + +/** + * Abstract Class created to handle common objects and methods. + */ +public abstract class AbstractNSSummaryTaskTest { + // User and Volume Constants + protected static final String TEST_USER = "TestUser"; + protected static final String VOL = "vol"; + protected static final long PARENT_OBJECT_ID_ZERO = 0L; + protected static final long VOL_OBJECT_ID = 0L; + // Bucket Constants + protected static final String BUCKET_ONE = "bucket1"; + protected static final String BUCKET_TWO = "bucket2"; + protected static final String BUCKET_THREE = "bucket3"; + protected static final long BUCKET_ONE_OBJECT_ID = 1L; + protected static final long BUCKET_TWO_OBJECT_ID = 2L; + protected static final long BUCKET_THREE_OBJECT_ID = 4L; + // File/Key Constants + protected static final String KEY_ONE = "file1"; + protected static final String KEY_TWO = "file2"; + protected static final String KEY_THREE = "file3"; + protected static final String KEY_FOUR = "file4"; + protected static final String KEY_FIVE = "file5"; + protected static final String KEY_SIX = "key6"; + protected static final String KEY_SEVEN = "/////key7"; + protected static final String KEY_THREE_1 = "dir1/dir2/file3"; + protected static final String FILE_ONE = "file1"; + protected static final String FILE_TWO = "file2"; + protected static final String FILE_THREE = "file3"; + protected static final String FILE_FOUR = "file4"; + protected static final String FILE_FIVE = "file5"; + protected static final long KEY_ONE_OBJECT_ID = 3L; + protected static final long KEY_TWO_OBJECT_ID = 5L; + protected static final long KEY_THREE_OBJECT_ID = 8L; + protected static final long KEY_FOUR_OBJECT_ID = 6L; + protected static final long KEY_FIVE_OBJECT_ID = 9L; + protected static final long KEY_SIX_OBJECT_ID = 10L; + protected static final long KEY_SEVEN_OBJECT_ID = 11L; + protected static final long KEY_ONE_SIZE = 500L; + protected static final long KEY_TWO_SIZE = 1025L; + protected static final long KEY_TWO_OLD_SIZE = 1025L; + protected static final long KEY_TWO_UPDATE_SIZE = 1023L; + protected static final long KEY_THREE_SIZE = ReconConstants.MAX_FILE_SIZE_UPPER_BOUND - 100L; + protected static final long KEY_FOUR_SIZE = 2050L; + protected static final long KEY_FIVE_SIZE = 100L; + protected static final long KEY_SIX_SIZE = 6000L; + protected static final long KEY_SEVEN_SIZE = 7000L; + // Directory Constants + protected static final String DIR_ONE = "dir1"; + protected static final String DIR_ONE_RENAME = "dir1_new"; + protected static final String DIR_TWO = "dir2"; + protected static final String DIR_THREE = "dir3"; + protected static final String DIR_FOUR = "dir4"; + protected static final String DIR_FIVE = "dir5"; + protected static final long DIR_ONE_OBJECT_ID = 4L; + protected static final long DIR_TWO_OBJECT_ID = 7L; + protected static final long DIR_THREE_OBJECT_ID = 10L; + protected static final long DIR_FOUR_OBJECT_ID = 11L; + protected static final long DIR_FIVE_OBJECT_ID = 12L; + + private OzoneConfiguration ozoneConfiguration; + private OzoneConfiguration omConfiguration; + private OMMetadataManager omMetadataManager; + private ReconOMMetadataManager reconOMMetadataManager; + private ReconNamespaceSummaryManager reconNamespaceSummaryManager; + + // Helper Methods + protected void commonSetup(File tmpDir, OMConfigParameter configParameter) throws Exception { + + if (configParameter.overrideConfig) { + setOzoneConfiguration(new OzoneConfiguration()); + getOzoneConfiguration().setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, configParameter.enableFSPaths); + getOzoneConfiguration().setLong(OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD, configParameter.flushThreshold); + } + + initializeNewOmMetadataManager(new File(tmpDir, "om"), configParameter.layout); + OzoneManagerServiceProviderImpl ozoneManagerServiceProvider = + configParameter.isFSO ? getMockOzoneManagerServiceProviderWithFSO() + : getMockOzoneManagerServiceProvider(); + + setReconOMMetadataManager(getTestReconOmMetadataManager(getOmMetadataManager(), new File(tmpDir, "recon"))); + ReconTestInjector reconTestInjector = + new ReconTestInjector.Builder(tmpDir) + .withReconOm(getReconOMMetadataManager()) + .withOmServiceProvider(ozoneManagerServiceProvider) + .withReconSqlDb() + .withContainerDB() + .build(); + + setReconNamespaceSummaryManager(reconTestInjector.getInstance(ReconNamespaceSummaryManager.class)); + assertNull(getReconNamespaceSummaryManager().getNSSummary(BUCKET_ONE_OBJECT_ID)); + + if (!configParameter.legacyPopulate && !configParameter.isFSO && !configParameter.isOBS) { + populateOMDBCommon(); + } else if (configParameter.isOBS) { + populateOMDBOBS(configParameter.layout); + } else if (configParameter.legacyPopulate) { + populateOMDB(configParameter.layout, true); + } else { + populateOMDB(configParameter.layout, false); + } + } + + public List<NSSummary> commonSetUpTestReprocess(Runnable reprocessTask, long... bucketObjectIds) throws IOException { + + List<NSSummary> result = new ArrayList<>(); + NSSummary staleNSSummary = new NSSummary(); + RDBBatchOperation rdbBatchOperation = new RDBBatchOperation(); + getReconNamespaceSummaryManager().batchStoreNSSummaries(rdbBatchOperation, -1L, staleNSSummary); + getReconNamespaceSummaryManager().commitBatchOperation(rdbBatchOperation); + + // Verify commit + assertNotNull(getReconNamespaceSummaryManager().getNSSummary(-1L)); + // Reinitialize Recon RocksDB's namespace CF. + getReconNamespaceSummaryManager().clearNSSummaryTable(); + // Run specific reprocess task + reprocessTask.run(); + // Verify cleanup + assertNull(getReconNamespaceSummaryManager().getNSSummary(-1L)); + // Assign and verify NSSummaries for buckets + if (bucketObjectIds.length > 0) { + NSSummary nsSummaryForBucket1 = getReconNamespaceSummaryManager().getNSSummary(bucketObjectIds[0]); + assertNotNull(nsSummaryForBucket1); + result.add(nsSummaryForBucket1); + } + if (bucketObjectIds.length > 1) { + NSSummary nsSummaryForBucket2 = getReconNamespaceSummaryManager().getNSSummary(bucketObjectIds[1]); + assertNotNull(nsSummaryForBucket2); + result.add(nsSummaryForBucket2); + } + if (bucketObjectIds.length > 2) { + NSSummary nsSummaryForBucket3 = getReconNamespaceSummaryManager().getNSSummary(bucketObjectIds[2]); + assertNotNull(nsSummaryForBucket3); + result.add(nsSummaryForBucket3); + } + return result; + } + + /** + * Build a key info for put/update action. + * + * @param volume volume name + * @param bucket bucket name + * @param key key name + * @param fileName file name + * @param objectID object ID + * @param parentObjectId parent object ID + * @param dataSize file size + * @return the KeyInfo + */ + protected static OmKeyInfo buildOmKeyInfo(String volume, String bucket, String key, String fileName, + long objectID, long parentObjectId, long dataSize) { + return new OmKeyInfo.Builder() + .setBucketName(bucket) + .setVolumeName(volume) + .setKeyName(key) + .setFileName(fileName) + .setReplicationConfig(StandaloneReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)) + .setObjectID(objectID) + .setParentObjectID(parentObjectId) + .setDataSize(dataSize) + .build(); + } + + /** + * Build a key info for delete action. + * + * @param volume volume name + * @param bucket bucket name + * @param key key name + * @param fileName file name + * @param objectID object ID + * @param parentObjectId parent object ID + * @return the KeyInfo + */ + protected static OmKeyInfo buildOmKeyInfo(String volume, String bucket, String key, String fileName, + long objectID, long parentObjectId) { + return new OmKeyInfo.Builder() + .setBucketName(bucket) + .setVolumeName(volume) + .setKeyName(key) + .setFileName(fileName) + .setReplicationConfig(StandaloneReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)) + .setObjectID(objectID) + .setParentObjectID(parentObjectId) + .build(); + } + + protected static OmDirectoryInfo buildOmDirInfo(String dirName, long objectId, long parentObjectId) { + return new OmDirectoryInfo.Builder() + .setName(dirName) + .setObjectID(objectId) + .setParentObjectID(parentObjectId) + .build(); + } + + /** + * Build a directory as key info for put/update action. + * We don't need to set size. + * @param volume volume name + * @param bucket bucket name + * @param key key name + * @param fileName file name + * @param objectID object ID + * @return the KeyInfo + */ + protected static OmKeyInfo buildOmDirKeyInfo(String volume, + String bucket, + String key, + String fileName, + long objectID) { Review Comment: nit: Please do not format method signature like this. Whenever visibility / return type / method name / other modifiers are changed, we would have to reindent all parameters. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@ozone.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@ozone.apache.org For additional commands, e-mail: issues-h...@ozone.apache.org
