dlmarion commented on code in PR #5341:
URL: https://github.com/apache/accumulo/pull/5341#discussion_r2010929108
##########
server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java:
##########
@@ -366,15 +369,35 @@ static long loadFiles(Loader loader, BulkInfo bulkInfo,
Path bulkDir,
ImportTimingStats importTimingStats = new ImportTimingStats();
Timer timer = Timer.startNew();
- try (TabletsMetadata tabletsMetadata =
factory.newTabletsMetadata(startRow)) {
+ KeyExtent prevLastExtent = null; // KeyExtent of last tablet from prior
loadMapEntry
+ TabletsMetadata tabletsMetadata = factory.newTabletsMetadata(startRow);
+ try {
Iterator<TabletMetadata> tabletIter = tabletsMetadata.iterator();
while (lmi.hasNext()) {
loadMapEntry = lmi.next();
+ // If the user set the TABLE_BULK_SKIP_THRESHOLD property, then only
look
+ // at the next skipDistance tablets before recreating the iterator
+ if (skipDistance > 0) {
+ final KeyExtent loadMapKey = loadMapEntry.getKey();
+ if (prevLastExtent != null &&
!loadMapKey.isPreviousExtent(prevLastExtent)) {
+ final KeyExtent search = prevLastExtent;
+ SearchResults results =
+ lmi.advanceTo(e -> e.getKey().isPreviousExtent(search),
skipDistance);
Review Comment:
Changed to the TabletIterator in 35891e9.
##########
server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/PrepBulkImport.java:
##########
@@ -116,82 +120,111 @@ private static boolean equals(Function<KeyExtent,Text>
extractor, KeyExtent ke1,
*/
@VisibleForTesting
static KeyExtent validateLoadMapping(String tableId, LoadMappingIterator lmi,
- TabletIterFactory tabletIterFactory, int maxNumTablets, long tid) throws
Exception {
+ TabletIterFactory tabletIterFactory, int maxNumTablets, long tid, int
skip) throws Exception {
+
var currRange = lmi.next();
Text startRow = currRange.getKey().prevEndRow();
Iterator<KeyExtent> tabletIter = tabletIterFactory.newTabletIter(startRow);
+ PeekingIterator<KeyExtent> pi = new PeekingIterator<>(tabletIter);
- KeyExtent currTablet = tabletIter.next();
+ try {
+ KeyExtent currTablet = pi.next();
- var fileCounts = new HashMap<String,Integer>();
- int count;
+ var fileCounts = new HashMap<String,Integer>();
+ int count;
- KeyExtent firstTablet = currRange.getKey();
- KeyExtent lastTablet = currRange.getKey();
+ KeyExtent firstTablet = currRange.getKey();
+ KeyExtent lastTablet = currRange.getKey();
- if (!tabletIter.hasNext() && equals(KeyExtent::prevEndRow, currTablet,
currRange.getKey())
- && equals(KeyExtent::endRow, currTablet, currRange.getKey())) {
- currRange = null;
- }
+ if (!pi.hasNext() && equals(KeyExtent::prevEndRow, currTablet,
currRange.getKey())
+ && equals(KeyExtent::endRow, currTablet, currRange.getKey())) {
+ currRange = null;
+ }
- while (tabletIter.hasNext()) {
+ while (pi.hasNext()) {
- if (currRange == null) {
- if (!lmi.hasNext()) {
- break;
+ if (currRange == null) {
+ if (!lmi.hasNext()) {
+ break;
+ }
+ currRange = lmi.next();
+ lastTablet = currRange.getKey();
+ }
+ // If the user set the TABLE_BULK_SKIP_THRESHOLD property, then only
look
+ // at the next skipDistance tablets before recreating the iterator
+ if (skip > 0) {
+ final KeyExtent search = currRange.getKey();
+ SearchResults results =
+ pi.advanceTo((ke) ->
ke.prevEndRow().equals(search.prevEndRow()), skip);
Review Comment:
Added checks in 35891e9
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
