From 44363eba747689c270e080a30d918a40e72d5fc1 Mon Sep 17 00:00:00 2001 From: Dom Garguilo Date: Mon, 30 Dec 2024 16:16:56 -0500 Subject: [PATCH 1/5] Improve Ample logging for rejected conditional mutations --- .../accumulo/core/metadata/schema/Ample.java | 11 +++++++++++ .../ConditionalTabletMutatorImpl.java | 12 +++++++++++- .../ConditionalTabletsMutatorImpl.java | 19 +++++++++++++------ .../functional/AmpleConditionalWriterIT.java | 6 ++++-- 4 files changed, 39 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java index 7135b5a978f..367ee6fe64a 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java @@ -24,6 +24,7 @@ import java.util.Set; import java.util.function.Consumer; import java.util.function.Predicate; +import java.util.function.Supplier; import org.apache.accumulo.core.client.ConditionalWriter; import org.apache.accumulo.core.client.admin.TabletAvailability; @@ -623,6 +624,16 @@ ConditionalTabletMutator requireSame(TabletMetadata tabletMetadata, ColumnType t * let the rejected status carry forward in this case. */ void submit(RejectionHandler rejectionHandler); + + /** + * Overloaded version of {@link #submit(RejectionHandler)} that takes a short description of the + * operation to assist with debugging. + * + * @param rejectionHandler The rejection handler + * @param description A short description of the operation (e.g., "bulk import", "compaction") + */ + void submit(RejectionHandler rejectionHandler, Supplier description); + } /** diff --git a/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java b/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java index 4ae46676024..ac0dc4b1126 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java +++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletMutatorImpl.java @@ -36,6 +36,7 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Supplier; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.admin.TabletAvailability; @@ -86,17 +87,20 @@ public class ConditionalTabletMutatorImpl extends TabletMutatorBase> descriptionConsumer; private boolean sawOperationRequirement = false; private boolean checkPrevEndRow = true; protected ConditionalTabletMutatorImpl(ServerContext context, KeyExtent extent, Consumer mutationConsumer, - BiConsumer rejectionHandlerConsumer) { + BiConsumer rejectionHandlerConsumer, + BiConsumer> descriptionConsumer) { super(new ConditionalMutation(extent.toMetaRow())); this.mutation = (ConditionalMutation) super.mutation; this.mutationConsumer = mutationConsumer; this.rejectionHandlerConsumer = rejectionHandlerConsumer; + this.descriptionConsumer = descriptionConsumer; this.extent = extent; this.context = context; this.lock = this.context.getServiceLock(); @@ -390,4 +394,10 @@ public void submit(Ample.RejectionHandler rejectionCheck) { mutationConsumer.accept(mutation); rejectionHandlerConsumer.accept(extent, rejectionCheck); } + + @Override + public void submit(Ample.RejectionHandler rejectionHandler, Supplier description) { + descriptionConsumer.accept(extent, description); + this.submit(rejectionHandler); + } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletsMutatorImpl.java b/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletsMutatorImpl.java index 660453f2998..777364e0c48 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletsMutatorImpl.java +++ b/server/base/src/main/java/org/apache/accumulo/server/metadata/ConditionalTabletsMutatorImpl.java @@ -30,6 +30,7 @@ import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; import org.apache.accumulo.core.client.AccumuloException; @@ -66,6 +67,7 @@ public class ConditionalTabletsMutatorImpl implements Ample.ConditionalTabletsMu private boolean active = true; final Map rejectedHandlers = new HashMap<>(); + private final Map> operationDescriptions = new HashMap<>(); private final Function tableMapper; public ConditionalTabletsMutatorImpl(ServerContext context) { @@ -93,7 +95,8 @@ public Ample.OperationRequirements mutateTablet(KeyExtent extent) { Preconditions.checkState(extents.putIfAbsent(extent.toMetaRow(), extent) == null, "Duplicate extents not handled %s", extent); - return new ConditionalTabletMutatorImpl(context, extent, mutations::add, rejectedHandlers::put); + return new ConditionalTabletMutatorImpl(context, extent, mutations::add, rejectedHandlers::put, + operationDescriptions::put); } protected ConditionalWriter createConditionalWriter(Ample.DataLevel dataLevel) @@ -262,16 +265,20 @@ public Status getStatus() { status = Status.ACCEPTED; } + Supplier descSupplier = operationDescriptions.get(extent); + String desc = (descSupplier == null) ? null : descSupplier.get(); + if (log.isTraceEnabled()) { // log detailed info about tablet metadata and mutation - log.trace("Mutation was rejected, status:{} {} {}", status, tabletMetadata, - result.getMutation().prettyPrint()); + log.trace("Mutation was rejected, status:{}. Operation description: {} {} {}", + status, desc, tabletMetadata, result.getMutation().prettyPrint()); } else if (log.isDebugEnabled()) { // log a single line of info that makes it apparent this happened and gives enough // information to investigate - log.debug("Mutation was rejected, status:{} extent:{} row:{}", status, - tabletMetadata == null ? null : tabletMetadata.getExtent(), - new String(result.getMutation().getRow(), UTF_8)); + log.debug( + "Mutation was rejected, status:{} extent:{} row:{} operation description: {}", + status, tabletMetadata == null ? null : tabletMetadata.getExtent(), + new String(result.getMutation().getRow(), UTF_8), desc); } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java index d8e0e9d2e17..f594f7b9ec7 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java @@ -173,7 +173,8 @@ public void testLocations() throws Exception { // test require absent with a future location set try (var ctmi = new ConditionalTabletsMutatorImpl(context)) { ctmi.mutateTablet(e1).requireAbsentOperation().requireAbsentLocation() - .putLocation(Location.future(ts2)).submit(tm -> false); + .putLocation(Location.future(ts2)).submit(tm -> false, + () -> "Testing that requireAbsentLocation() fails when a future location is set"); assertEquals(Status.REJECTED, ctmi.process().get(e1).getStatus()); } assertEquals(Location.future(ts1), context.getAmple().readTablet(e1).getLocation()); @@ -196,7 +197,8 @@ public void testLocations() throws Exception { try (var ctmi = new ConditionalTabletsMutatorImpl(context)) { ctmi.mutateTablet(e1).requireAbsentOperation().requireAbsentLocation() .putLocation(Location.future(ts2)).submit(tm -> false); - assertEquals(Status.REJECTED, ctmi.process().get(e1).getStatus()); + assertEquals(Status.REJECTED, ctmi.process().get(e1).getStatus(), + () -> "Testing that requireAbsentLocation() fails when a current location is set"); } assertEquals(Location.current(ts1), context.getAmple().readTablet(e1).getLocation()); From 891afe2de038ca111c731771f0ce439b3de0200e Mon Sep 17 00:00:00 2001 From: Dom Garguilo Date: Fri, 3 Jan 2025 13:16:34 -0500 Subject: [PATCH 2/5] Add more descriptions to conditional mutations --- .../accumulo/manager/TabletGroupWatcher.java | 12 ++++----- .../coordinator/CompactionCoordinator.java | 5 ++-- .../coordinator/commit/CommitCompaction.java | 5 ++-- .../availability/SetTabletAvailability.java | 3 ++- .../tableOps/bulkVer2/CleanUpBulkImport.java | 2 +- .../manager/tableOps/bulkVer2/LoadFiles.java | 2 +- .../tableOps/compact/CompactionDriver.java | 19 ++++++++------ .../tableOps/delete/ReserveTablets.java | 2 +- .../manager/tableOps/merge/DeleteRows.java | 25 +++++++++++-------- 9 files changed, 43 insertions(+), 32 deletions(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java index 6496f500e9f..e29413e82aa 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java @@ -350,12 +350,10 @@ public void hostOndemand(Collection extents) { // Do not add any code here, it may interfere with the finally block removing extents from // hostingRequestInProgress try (var mutator = manager.getContext().getAmple().conditionallyMutateTablets()) { - inProgress.forEach(ke -> { - mutator.mutateTablet(ke).requireAbsentOperation() - .requireTabletAvailability(TabletAvailability.ONDEMAND).requireAbsentLocation() - .setHostingRequested().submit(TabletMetadata::getHostingRequested); - - }); + inProgress.forEach(ke -> mutator.mutateTablet(ke).requireAbsentOperation() + .requireTabletAvailability(TabletAvailability.ONDEMAND).requireAbsentLocation() + .setHostingRequested() + .submit(TabletMetadata::getHostingRequested, () -> "host ondemand")); List ranges = new ArrayList<>(); @@ -1094,7 +1092,7 @@ private void replaceVolumes(List volumeReplacemen "replaceVolume conditional mutation rejection check {} logsRemoved:{} filesRemoved:{}", tm.getExtent(), logsRemoved, filesRemoved); return logsRemoved && filesRemoved; - }); + }, () -> "replace volume"); } tabletsMutator.process().forEach((extent, result) -> { diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java index 6f2eca37565..fce920e4cba 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java @@ -635,8 +635,9 @@ public CompactionMetadata get() { } tabletMutator.putExternalCompaction(externalCompactionId, ecm); - tabletMutator - .submit(tm -> tm.getExternalCompactions().containsKey(externalCompactionId)); + tabletMutator.submit( + tm -> tm.getExternalCompactions().containsKey(externalCompactionId), + () -> "compaction reservation"); var result = tabletsMutator.process().get(extent); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/CommitCompaction.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/CommitCompaction.java index cb9492d7e30..81a777c5a6d 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/CommitCompaction.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/CommitCompaction.java @@ -128,8 +128,9 @@ private TabletMetadata commitCompaction(ServerContext ctx, ExternalCompactionId // make the needed updates to the tablet updateTabletForCompaction(commitData.stats, ecid, tablet, newDatafile, ecm, tabletMutator); - tabletMutator - .submit(tabletMetadata -> !tabletMetadata.getExternalCompactions().containsKey(ecid)); + tabletMutator.submit( + tabletMetadata -> !tabletMetadata.getExternalCompactions().containsKey(ecid), + () -> "commit compaction"); if (LOG.isDebugEnabled()) { LOG.debug("Compaction completed {} added {} removed {}", tablet.getExtent(), newDatafile, diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/availability/SetTabletAvailability.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/availability/SetTabletAvailability.java index bd94afd15b4..db4fcdbeaf2 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/availability/SetTabletAvailability.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/availability/SetTabletAvailability.java @@ -136,7 +136,8 @@ public long isReady(FateId fateId, Manager manager) throws Exception { tabletExtent); mutator.mutateTablet(tabletExtent).requireAbsentOperation() .putTabletAvailability(tabletAvailability) - .submit(tabletMeta -> tabletMeta.getTabletAvailability() == tabletAvailability); + .submit(tabletMeta -> tabletMeta.getTabletAvailability() == tabletAvailability, + () -> "set tablet availability"); } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java index 32003695251..2f18a6f3e07 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java @@ -111,7 +111,7 @@ private static void removeBulkLoadEntries(Ample ample, TableId tableId, FateId f tabletsMutator.mutateTablet(tablet.getExtent()).requireAbsentOperation(); tablet.getLoaded().entrySet().stream().filter(entry -> entry.getValue().equals(fateId)) .map(Map.Entry::getKey).forEach(tabletMutator::deleteBulkFile); - tabletMutator.submit(tm -> false); + tabletMutator.submit(tm -> false, () -> "remove bulk load entries"); } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java index 5ff2ca3ec39..0f9aeb7e56f 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java @@ -251,7 +251,7 @@ void load(List tablets, Files files) { Preconditions.checkState( loadingFiles.put(tablet.getExtent(), List.copyOf(filesToLoad.keySet())) == null); - tabletMutator.submit(tm -> false); + tabletMutator.submit(tm -> false, () -> "bulk load files"); } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java index 464f3651210..cbf55ef6b66 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java @@ -211,7 +211,8 @@ public int updateAndCheckTablets(Manager manager, FateId fateId) // this tablet has no files try to mark it as done tabletsMutator.mutateTablet(tablet.getExtent()).requireAbsentOperation() .requireSame(tablet, FILES, COMPACTED).putCompacted(fateId) - .submit(tabletMetadata -> tabletMetadata.getCompacted().contains(fateId)); + .submit(tabletMetadata -> tabletMetadata.getCompacted().contains(fateId), + () -> "no files, attempting to mark as compacted"); noFiles++; } else if (tablet.getSelectedFiles() == null && tablet.getExternalCompactions().isEmpty()) { // there are no selected files @@ -242,7 +243,8 @@ public int updateAndCheckTablets(Manager manager, FateId fateId) // no files were selected so mark the tablet as compacted tabletsMutator.mutateTablet(tablet.getExtent()).requireAbsentOperation() .requireSame(tablet, FILES, SELECTED, ECOMP, COMPACTED).putCompacted(fateId) - .submit(tabletMetadata -> tabletMetadata.getCompacted().contains(fateId)); + .submit(tabletMetadata -> tabletMetadata.getCompacted().contains(fateId), + () -> "no files, attempting to mark as compacted"); noneSelected++; } else { @@ -260,9 +262,11 @@ public int updateAndCheckTablets(Manager manager, FateId fateId) selectionsSubmitted.put(tablet.getExtent(), filesToCompact); - mutator.submit(tabletMetadata -> tabletMetadata.getSelectedFiles() != null - && tabletMetadata.getSelectedFiles().getFateId().equals(fateId) - || tabletMetadata.getCompacted().contains(fateId)); + mutator.submit( + tabletMetadata -> tabletMetadata.getSelectedFiles() != null + && tabletMetadata.getSelectedFiles().getFateId().equals(fateId) + || tabletMetadata.getCompacted().contains(fateId), + () -> "selecting files for compaction"); if (minSelected == null || tablet.getExtent().compareTo(minSelected) < 0) { minSelected = tablet.getExtent(); @@ -298,7 +302,8 @@ public int updateAndCheckTablets(Manager manager, FateId fateId) var mutator = tabletsMutator.mutateTablet(tablet.getExtent()).requireAbsentOperation() .requireSame(tablet, ECOMP, USER_COMPACTION_REQUESTED) .putUserCompactionRequested(fateId); - mutator.submit(tm -> tm.getUserCompactionsRequested().contains(fateId)); + mutator.submit(tm -> tm.getUserCompactionsRequested().contains(fateId), + () -> "marking as needing a user requested compaction"); userCompactionRequested++; } else { // Marker was already added and we are waiting @@ -400,7 +405,7 @@ private void cleanupTabletMetadata(FateId fateId, Manager manager) throws Except mutator.deleteUserCompactionRequested(fateId); } - mutator.submit(needsNoUpdate::test); + mutator.submit(needsNoUpdate::test, () -> "cleanup metadata for failed compaction"); } } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/ReserveTablets.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/ReserveTablets.java index f065206b2ff..1ce4c5e01b6 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/ReserveTablets.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/ReserveTablets.java @@ -96,7 +96,7 @@ public long isReady(FateId fateId, Manager manager) throws Exception { // must wait for the tablet to have no location before proceeding to actually delete. See // the documentation about the opid column in the MetadataSchema class for more details. conditionalMutator.mutateTablet(tabletMeta.getExtent()).requireAbsentOperation() - .putOperation(opid).submit(tm -> opid.equals(tm.getOperationId())); + .putOperation(opid).submit(tm -> opid.equals(tm.getOperationId()), () -> "put opid"); submitted++; } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java index 2c5221b7ce8..5d2407ea852 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java @@ -178,16 +178,21 @@ private Optional deleteTabletFiles(Manager manager, FateId fateId) { } } - filesToDelete.forEach(file -> log.debug("{} deleting file {} for {}", fateId, file, - tabletMetadata.getExtent())); - filesToAddMap.forEach((file, dfv) -> log.debug("{} adding file {} {} for {}", fateId, file, - dfv, tabletMetadata.getExtent())); - - filesToDelete.forEach(tabletMutator::deleteFile); - filesToAddMap.forEach(tabletMutator::putFile); - - tabletMutator.submit(tm -> tm.getFiles().containsAll(filesToAddMap.keySet()) - && Collections.disjoint(tm.getFiles(), filesToDelete)); + filesToDelete.forEach(file -> { + log.debug("{} deleting file {} for {}", fateId, file, tabletMetadata.getExtent()); + tabletMutator.deleteFile(file); + }); + + filesToAddMap.forEach((file, dfv) -> { + log.debug("{} adding file {} {} for {}", fateId, file, dfv, tabletMetadata.getExtent()); + tabletMutator.putFile(file, dfv); + }); + + tabletMutator + .submit( + tm -> tm.getFiles().containsAll(filesToAddMap.keySet()) + && Collections.disjoint(tm.getFiles(), filesToDelete), + () -> "delete tablet files"); } var results = tabletsMutator.process(); From 3929a0b3a3b178e260d3555228a0460a6865aa9b Mon Sep 17 00:00:00 2001 From: "Dom G." Date: Mon, 6 Jan 2025 12:26:44 -0500 Subject: [PATCH 3/5] Apply suggestions from code review - add ids to descriptions Co-authored-by: Keith Turner --- .../manager/compaction/coordinator/commit/CommitCompaction.java | 2 +- .../accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java | 2 +- .../apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java | 2 +- .../apache/accumulo/manager/tableOps/delete/ReserveTablets.java | 2 +- .../org/apache/accumulo/manager/tableOps/merge/DeleteRows.java | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/CommitCompaction.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/CommitCompaction.java index 81a777c5a6d..752dc604790 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/CommitCompaction.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/CommitCompaction.java @@ -130,7 +130,7 @@ private TabletMetadata commitCompaction(ServerContext ctx, ExternalCompactionId tabletMutator.submit( tabletMetadata -> !tabletMetadata.getExternalCompactions().containsKey(ecid), - () -> "commit compaction"); + () -> "commit compaction "+ecid); if (LOG.isDebugEnabled()) { LOG.debug("Compaction completed {} added {} removed {}", tablet.getExtent(), newDatafile, diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java index 2f18a6f3e07..85fe07b6f71 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java @@ -111,7 +111,7 @@ private static void removeBulkLoadEntries(Ample ample, TableId tableId, FateId f tabletsMutator.mutateTablet(tablet.getExtent()).requireAbsentOperation(); tablet.getLoaded().entrySet().stream().filter(entry -> entry.getValue().equals(fateId)) .map(Map.Entry::getKey).forEach(tabletMutator::deleteBulkFile); - tabletMutator.submit(tm -> false, () -> "remove bulk load entries"); + tabletMutator.submit(tm -> false, () -> "remove bulk load entries "+fateId); } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java index 0f9aeb7e56f..87e5546ac0c 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java @@ -251,7 +251,7 @@ void load(List tablets, Files files) { Preconditions.checkState( loadingFiles.put(tablet.getExtent(), List.copyOf(filesToLoad.keySet())) == null); - tabletMutator.submit(tm -> false, () -> "bulk load files"); + tabletMutator.submit(tm -> false, () -> "bulk load files "+fateId); } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/ReserveTablets.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/ReserveTablets.java index 1ce4c5e01b6..658134c55c4 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/ReserveTablets.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/ReserveTablets.java @@ -96,7 +96,7 @@ public long isReady(FateId fateId, Manager manager) throws Exception { // must wait for the tablet to have no location before proceeding to actually delete. See // the documentation about the opid column in the MetadataSchema class for more details. conditionalMutator.mutateTablet(tabletMeta.getExtent()).requireAbsentOperation() - .putOperation(opid).submit(tm -> opid.equals(tm.getOperationId()), () -> "put opid"); + .putOperation(opid).submit(tm -> opid.equals(tm.getOperationId()), () -> "put opid "+opid); submitted++; } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java index 5d2407ea852..42d5442314f 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java @@ -192,7 +192,7 @@ private Optional deleteTabletFiles(Manager manager, FateId fateId) { .submit( tm -> tm.getFiles().containsAll(filesToAddMap.keySet()) && Collections.disjoint(tm.getFiles(), filesToDelete), - () -> "delete tablet files"); + () -> "delete tablet files (as part of merge or deleterow operation) "+fateId); } var results = tabletsMutator.process(); From 22552e1a7d28605ff2b9a92726f0994495df50c2 Mon Sep 17 00:00:00 2001 From: Dom Garguilo Date: Mon, 6 Jan 2025 12:28:21 -0500 Subject: [PATCH 4/5] formatting --- .../compaction/coordinator/commit/CommitCompaction.java | 2 +- .../manager/tableOps/bulkVer2/CleanUpBulkImport.java | 2 +- .../accumulo/manager/tableOps/bulkVer2/LoadFiles.java | 2 +- .../accumulo/manager/tableOps/delete/ReserveTablets.java | 3 ++- .../accumulo/manager/tableOps/merge/DeleteRows.java | 9 ++++----- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/CommitCompaction.java b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/CommitCompaction.java index 752dc604790..29e626d9540 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/CommitCompaction.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/commit/CommitCompaction.java @@ -130,7 +130,7 @@ private TabletMetadata commitCompaction(ServerContext ctx, ExternalCompactionId tabletMutator.submit( tabletMetadata -> !tabletMetadata.getExternalCompactions().containsKey(ecid), - () -> "commit compaction "+ecid); + () -> "commit compaction " + ecid); if (LOG.isDebugEnabled()) { LOG.debug("Compaction completed {} added {} removed {}", tablet.getExtent(), newDatafile, diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java index 85fe07b6f71..e5cdfdb085a 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/CleanUpBulkImport.java @@ -111,7 +111,7 @@ private static void removeBulkLoadEntries(Ample ample, TableId tableId, FateId f tabletsMutator.mutateTablet(tablet.getExtent()).requireAbsentOperation(); tablet.getLoaded().entrySet().stream().filter(entry -> entry.getValue().equals(fateId)) .map(Map.Entry::getKey).forEach(tabletMutator::deleteBulkFile); - tabletMutator.submit(tm -> false, () -> "remove bulk load entries "+fateId); + tabletMutator.submit(tm -> false, () -> "remove bulk load entries " + fateId); } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java index 87e5546ac0c..c38fb54ce69 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java @@ -251,7 +251,7 @@ void load(List tablets, Files files) { Preconditions.checkState( loadingFiles.put(tablet.getExtent(), List.copyOf(filesToLoad.keySet())) == null); - tabletMutator.submit(tm -> false, () -> "bulk load files "+fateId); + tabletMutator.submit(tm -> false, () -> "bulk load files " + fateId); } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/ReserveTablets.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/ReserveTablets.java index 658134c55c4..6f9d65d0b87 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/ReserveTablets.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/ReserveTablets.java @@ -96,7 +96,8 @@ public long isReady(FateId fateId, Manager manager) throws Exception { // must wait for the tablet to have no location before proceeding to actually delete. See // the documentation about the opid column in the MetadataSchema class for more details. conditionalMutator.mutateTablet(tabletMeta.getExtent()).requireAbsentOperation() - .putOperation(opid).submit(tm -> opid.equals(tm.getOperationId()), () -> "put opid "+opid); + .putOperation(opid) + .submit(tm -> opid.equals(tm.getOperationId()), () -> "put opid " + opid); submitted++; } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java index 42d5442314f..3bf6bce841b 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java @@ -188,11 +188,10 @@ private Optional deleteTabletFiles(Manager manager, FateId fateId) { tabletMutator.putFile(file, dfv); }); - tabletMutator - .submit( - tm -> tm.getFiles().containsAll(filesToAddMap.keySet()) - && Collections.disjoint(tm.getFiles(), filesToDelete), - () -> "delete tablet files (as part of merge or deleterow operation) "+fateId); + tabletMutator.submit( + tm -> tm.getFiles().containsAll(filesToAddMap.keySet()) + && Collections.disjoint(tm.getFiles(), filesToDelete), + () -> "delete tablet files (as part of merge or deleterow operation) " + fateId); } var results = tabletsMutator.process(); From cdd9b9ce1189563987c085467587b7aa5f4151af Mon Sep 17 00:00:00 2001 From: Dom Garguilo Date: Mon, 6 Jan 2025 12:33:17 -0500 Subject: [PATCH 5/5] Add more IDs to descriptions --- .../manager/tableOps/compact/CompactionDriver.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java index cbf55ef6b66..51c4186bf9c 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/compact/CompactionDriver.java @@ -212,7 +212,7 @@ public int updateAndCheckTablets(Manager manager, FateId fateId) tabletsMutator.mutateTablet(tablet.getExtent()).requireAbsentOperation() .requireSame(tablet, FILES, COMPACTED).putCompacted(fateId) .submit(tabletMetadata -> tabletMetadata.getCompacted().contains(fateId), - () -> "no files, attempting to mark as compacted"); + () -> "no files, attempting to mark as compacted. " + fateId); noFiles++; } else if (tablet.getSelectedFiles() == null && tablet.getExternalCompactions().isEmpty()) { // there are no selected files @@ -244,7 +244,7 @@ public int updateAndCheckTablets(Manager manager, FateId fateId) tabletsMutator.mutateTablet(tablet.getExtent()).requireAbsentOperation() .requireSame(tablet, FILES, SELECTED, ECOMP, COMPACTED).putCompacted(fateId) .submit(tabletMetadata -> tabletMetadata.getCompacted().contains(fateId), - () -> "no files, attempting to mark as compacted"); + () -> "no files, attempting to mark as compacted. " + fateId); noneSelected++; } else { @@ -266,7 +266,7 @@ public int updateAndCheckTablets(Manager manager, FateId fateId) tabletMetadata -> tabletMetadata.getSelectedFiles() != null && tabletMetadata.getSelectedFiles().getFateId().equals(fateId) || tabletMetadata.getCompacted().contains(fateId), - () -> "selecting files for compaction"); + () -> "selecting files for compaction. " + fateId); if (minSelected == null || tablet.getExtent().compareTo(minSelected) < 0) { minSelected = tablet.getExtent(); @@ -303,7 +303,7 @@ public int updateAndCheckTablets(Manager manager, FateId fateId) .requireSame(tablet, ECOMP, USER_COMPACTION_REQUESTED) .putUserCompactionRequested(fateId); mutator.submit(tm -> tm.getUserCompactionsRequested().contains(fateId), - () -> "marking as needing a user requested compaction"); + () -> "marking as needing a user requested compaction. " + fateId); userCompactionRequested++; } else { // Marker was already added and we are waiting @@ -405,7 +405,8 @@ private void cleanupTabletMetadata(FateId fateId, Manager manager) throws Except mutator.deleteUserCompactionRequested(fateId); } - mutator.submit(needsNoUpdate::test, () -> "cleanup metadata for failed compaction"); + mutator.submit(needsNoUpdate::test, + () -> "cleanup metadata for failed compaction. " + fateId); } } }