diff --git a/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactableFile.java b/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactableFile.java index baf5f04949b..36b2d7b0ea9 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactableFile.java +++ b/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactableFile.java @@ -20,6 +20,7 @@ import java.net.URI; +import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.metadata.CompactableFileImpl; /** @@ -33,6 +34,16 @@ public interface CompactableFile { public URI getUri(); + /** + * @return A range associated with the file. If a file has an associated range then Accumulo will + * limit reads to within the range. Not all files have an associated range, it a file does + * not have a range then an infinite range is returned. The URI plus this range uniquely + * identify a file. + * + * @since 3.1.0 + */ + public Range getRange(); + public long getEstimatedSize(); public long getEstimatedEntries(); @@ -41,4 +52,12 @@ static CompactableFile create(URI uri, long estimatedSize, long estimatedEntries return new CompactableFileImpl(uri, estimatedSize, estimatedEntries); } + /** + * Creates a new CompactableFile object that implements this interface. + * + * @since 3.1.0 + */ + static CompactableFile create(URI uri, Range range, long estimatedSize, long estimatedEntries) { + return new CompactableFileImpl(uri, range, estimatedSize, estimatedEntries); + } } diff --git a/core/src/main/java/org/apache/accumulo/core/clientImpl/RootClientTabletCache.java b/core/src/main/java/org/apache/accumulo/core/clientImpl/RootClientTabletCache.java index f86d90b6f85..65090740171 100644 --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/RootClientTabletCache.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/RootClientTabletCache.java @@ -58,7 +58,7 @@ public class RootClientTabletCache extends ClientTabletCache { public void binMutations(ClientContext context, List mutations, Map> binnedMutations, List failures) { CachedTablet rootCachedTablet = getRootTabletLocation(context); - if (rootCachedTablet != null) { + if (rootCachedTablet != null && rootCachedTablet.getTserverLocation().isPresent()) { var tsm = new TabletServerMutations(rootCachedTablet.getTserverSession().orElseThrow()); for (T mutation : mutations) { tsm.addMutation(RootTable.EXTENT, mutation); diff --git a/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java b/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java index 8295b2c917b..da10ce2676b 100644 --- a/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java +++ b/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java @@ -29,6 +29,7 @@ import org.apache.accumulo.core.client.admin.CompactionConfig; import org.apache.accumulo.core.client.admin.compaction.CompactableFile; import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.metadata.CompactableFileImpl; import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.TabletFile; @@ -119,32 +120,33 @@ private static String getSize(Collection files) { * Lazily converts TableFile to file names. The lazy part is really important because when it is * not called with log.isDebugEnabled(). */ - private static Collection asFileNames(Collection files) { - return Collections2.transform(files, CompactableFile::getFileName); + private static Collection asMinimalString(Collection files) { + return Collections2.transform(files, + cf -> CompactableFileImpl.toStoredTabletFile(cf).toMinimalString()); } public static void selected(KeyExtent extent, CompactionKind kind, Collection inputs) { fileLog.trace("{} changed compaction selection set for {} new set {}", extent, kind, - Collections2.transform(inputs, StoredTabletFile::getFileName)); + Collections2.transform(inputs, StoredTabletFile::toMinimalString)); } public static void compacting(KeyExtent extent, CompactionJob job, CompactionConfig config) { if (fileLog.isDebugEnabled()) { if (config == null) { fileLog.debug("Compacting {} on {} for {} from {} size {}", extent, job.getExecutor(), - job.getKind(), asFileNames(job.getFiles()), getSize(job.getFiles())); + job.getKind(), asMinimalString(job.getFiles()), getSize(job.getFiles())); } else { fileLog.debug("Compacting {} on {} for {} from {} size {} config {}", extent, - job.getExecutor(), job.getKind(), asFileNames(job.getFiles()), getSize(job.getFiles()), - config); + job.getExecutor(), job.getKind(), asMinimalString(job.getFiles()), + getSize(job.getFiles()), config); } } } public static void compacted(KeyExtent extent, CompactionJob job, StoredTabletFile output) { fileLog.debug("Compacted {} for {} created {} from {}", extent, job.getKind(), output, - asFileNames(job.getFiles())); + asMinimalString(job.getFiles())); } public static void flushed(KeyExtent extent, Optional newDatafile) { diff --git a/core/src/main/java/org/apache/accumulo/core/manager/balancer/AssignmentParamsImpl.java b/core/src/main/java/org/apache/accumulo/core/manager/balancer/AssignmentParamsImpl.java index 7bdbf70fc1a..bf5e387facd 100644 --- a/core/src/main/java/org/apache/accumulo/core/manager/balancer/AssignmentParamsImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/manager/balancer/AssignmentParamsImpl.java @@ -34,8 +34,13 @@ import org.apache.accumulo.core.spi.balancer.TabletBalancer; import org.apache.accumulo.core.spi.balancer.data.TServerStatus; import org.apache.accumulo.core.spi.balancer.data.TabletServerId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class AssignmentParamsImpl implements TabletBalancer.AssignmentParameters { + + private static final Logger LOG = LoggerFactory.getLogger(AssignmentParamsImpl.class); + private final SortedMap currentStatus; private final Map unassigned; private final Map assignmentsOut; @@ -50,16 +55,26 @@ public static AssignmentParamsImpl fromThrift( Map unassigned, Map assignmentsOut) { SortedMap currentStatusNew = new TreeMap<>(); - currentStatus.forEach((tsi, status) -> currentStatusNew.put(new TabletServerIdImpl(tsi), - TServerStatusImpl.fromThrift(status))); - Map> tserverGroups = new HashMap<>(); - currentTServerGrouping.forEach((k, v) -> { + currentTServerGrouping.forEach((group, serversInGroup) -> { Set servers = new HashSet<>(); - v.forEach(tsi -> servers.add(TabletServerIdImpl.fromThrift(tsi))); - tserverGroups.put(k, servers); + serversInGroup.forEach(tsi -> { + TabletServerIdImpl id = TabletServerIdImpl.fromThrift(tsi); + if (currentStatus.containsKey(tsi)) { + currentStatusNew.put(id, TServerStatusImpl.fromThrift(currentStatus.get(tsi))); + servers.add(id); + } else { + LOG.debug("Dropping tserver {} from group {} as it's not in set of all servers", id, + group); + } + }); + if (!servers.isEmpty()) { + tserverGroups.put(group, servers); + } }); + LOG.debug("TServer groups for balancer assignment: {}", tserverGroups); + Map unassignedNew = new HashMap<>(); unassigned.forEach( (ke, tsi) -> unassignedNew.put(new TabletIdImpl(ke), TabletServerIdImpl.fromThrift(tsi))); diff --git a/core/src/main/java/org/apache/accumulo/core/manager/balancer/TabletServerIdImpl.java b/core/src/main/java/org/apache/accumulo/core/manager/balancer/TabletServerIdImpl.java index 2fefb466d33..013515a2530 100644 --- a/core/src/main/java/org/apache/accumulo/core/manager/balancer/TabletServerIdImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/manager/balancer/TabletServerIdImpl.java @@ -44,6 +44,10 @@ public TabletServerIdImpl(TServerInstance tServerInstance) { this.tServerInstance = requireNonNull(tServerInstance); } + public TServerInstance getTServerInstance() { + return tServerInstance; + } + @Override public String getHost() { return tServerInstance.getHostAndPort().getHost(); diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/AbstractTabletFile.java b/core/src/main/java/org/apache/accumulo/core/metadata/AbstractTabletFile.java index 0957c07a9d0..ae0d46fbf21 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/AbstractTabletFile.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/AbstractTabletFile.java @@ -20,6 +20,7 @@ import java.util.Objects; +import org.apache.accumulo.core.data.ByteSequence; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; import org.apache.hadoop.fs.Path; @@ -82,4 +83,24 @@ private static boolean isExclusiveKey(Key key) { return row.length() > 0 && row.byteAt(row.length() - 1) == (byte) 0x00; } + private static String stripZeroTail(ByteSequence row) { + if (row.byteAt(row.length() - 1) == (byte) 0x00) { + return row.subSequence(0, row.length() - 1).toString(); + } + return row.toString(); + } + + @Override + public String toMinimalString() { + if (hasRange()) { + String startRow = + range.isInfiniteStartKey() ? "-inf" : stripZeroTail(range.getStartKey().getRowData()); + String endRow = + range.isInfiniteStopKey() ? "+inf" : stripZeroTail(range.getEndKey().getRowData()); + return getFileName() + " (" + startRow + "," + endRow + "]"; + } else { + return getFileName(); + } + } + } diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/CompactableFileImpl.java b/core/src/main/java/org/apache/accumulo/core/metadata/CompactableFileImpl.java index 80e334ba688..f0d51f63b66 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/CompactableFileImpl.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/CompactableFileImpl.java @@ -22,6 +22,7 @@ import java.util.Objects; import org.apache.accumulo.core.client.admin.compaction.CompactableFile; +import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.metadata.schema.DataFileValue; public class CompactableFileImpl implements CompactableFile { @@ -36,6 +37,11 @@ public CompactableFileImpl(URI uri, long size, long entries) { this.dataFileValue = new DataFileValue(size, entries); } + public CompactableFileImpl(URI uri, Range range, long size, long entries) { + this.storedTabletFile = StoredTabletFile.of(uri, range); + this.dataFileValue = new DataFileValue(size, entries); + } + public CompactableFileImpl(StoredTabletFile storedTabletFile, DataFileValue dataFileValue) { this.storedTabletFile = Objects.requireNonNull(storedTabletFile); this.dataFileValue = Objects.requireNonNull(dataFileValue); @@ -46,6 +52,11 @@ public URI getUri() { return storedTabletFile.getPath().toUri(); } + @Override + public Range getRange() { + return storedTabletFile.getRange(); + } + @Override public String getFileName() { return storedTabletFile.getFileName(); @@ -91,6 +102,6 @@ public static StoredTabletFile toStoredTabletFile(CompactableFile cf) { @Override public String toString() { - return "[" + storedTabletFile.getFileName() + ", " + dataFileValue + "]"; + return "[" + storedTabletFile.toMinimalString() + ", " + dataFileValue + "]"; } } diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/TabletFile.java b/core/src/main/java/org/apache/accumulo/core/metadata/TabletFile.java index b31b6926d3e..8b90bd78803 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/TabletFile.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/TabletFile.java @@ -52,4 +52,9 @@ public interface TabletFile { * */ boolean hasRange(); + + /** + * @return a string with the filename and row range if there is one. + */ + String toMinimalString(); } diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/TabletState.java b/core/src/main/java/org/apache/accumulo/core/metadata/TabletState.java index 4be39cf906c..9fcf8add3f8 100644 --- a/core/src/main/java/org/apache/accumulo/core/metadata/TabletState.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/TabletState.java @@ -18,29 +18,18 @@ */ package org.apache.accumulo.core.metadata; -import java.util.Map; import java.util.Set; -import org.apache.accumulo.core.data.TabletId; -import org.apache.accumulo.core.dataImpl.TabletIdImpl; -import org.apache.accumulo.core.manager.balancer.TabletServerIdImpl; import org.apache.accumulo.core.metadata.schema.TabletMetadata; -import org.apache.accumulo.core.spi.balancer.TabletBalancer; -import org.apache.accumulo.core.spi.balancer.data.TabletServerId; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public enum TabletState { - UNASSIGNED, ASSIGNED, HOSTED, ASSIGNED_TO_DEAD_SERVER, SUSPENDED, NEEDS_REASSIGNMENT; + UNASSIGNED, ASSIGNED, HOSTED, ASSIGNED_TO_DEAD_SERVER, SUSPENDED; private static Logger log = LoggerFactory.getLogger(TabletState.class); public static TabletState compute(TabletMetadata tm, Set liveTServers) { - return compute(tm, liveTServers, null, null); - } - - public static TabletState compute(TabletMetadata tm, Set liveTServers, - TabletBalancer balancer, Map tserverGroups) { TabletMetadata.Location current = null; TabletMetadata.Location future = null; if (tm.hasCurrent()) { @@ -53,42 +42,6 @@ public static TabletState compute(TabletMetadata tm, Set liveTS : TabletState.ASSIGNED_TO_DEAD_SERVER; } else if (current != null) { if (liveTServers.contains(current.getServerInstance())) { - if (balancer != null) { - var tsii = new TabletServerIdImpl(current.getServerInstance()); - var resourceGroup = tserverGroups.get(tsii); - - if (resourceGroup != null) { - var reassign = balancer.needsReassignment(new TabletBalancer.CurrentAssignment() { - @Override - public TabletId getTablet() { - return new TabletIdImpl(tm.getExtent()); - } - - @Override - public TabletServerId getTabletServer() { - return tsii; - } - - @Override - public String getResourceGroup() { - return resourceGroup; - } - }); - - if (reassign) { - return TabletState.NEEDS_REASSIGNMENT; - } - } else { - // A tablet server should always have a resource group, however there is a race - // conditions where the resource group map was read before a tablet server came into - // existence. Another possible cause for an absent resource group is a bug in accumulo. - // In either case do not call the balancer for now with the assumption that the resource - // group will be available later. Log a message in case it is a bug. - log.trace( - "Could not find resource group for tserver {}, so did not consult balancer. Assuming this is a temporary race condition.", - current.getServerInstance()); - } - } return TabletState.HOSTED; } else { return TabletState.ASSIGNED_TO_DEAD_SERVER; diff --git a/core/src/main/java/org/apache/accumulo/core/spi/balancer/SimpleLoadBalancer.java b/core/src/main/java/org/apache/accumulo/core/spi/balancer/SimpleLoadBalancer.java index 50fcd8417fb..09189c0192a 100644 --- a/core/src/main/java/org/apache/accumulo/core/spi/balancer/SimpleLoadBalancer.java +++ b/core/src/main/java/org/apache/accumulo/core/spi/balancer/SimpleLoadBalancer.java @@ -356,6 +356,10 @@ private static TableId busiest(Map tables) { @Override public void getAssignments(AssignmentParameters params) { + if (params.currentStatus().isEmpty()) { + log.debug("No known TabletServers, skipping tablet assignment for now."); + return; + } params.unassignedTablets().forEach((tabletId, tserverId) -> params.addAssignment(tabletId, getAssignment(params.currentStatus(), tserverId))); } diff --git a/core/src/main/java/org/apache/accumulo/core/spi/balancer/TableLoadBalancer.java b/core/src/main/java/org/apache/accumulo/core/spi/balancer/TableLoadBalancer.java index 853608018f9..44cf29d1e87 100644 --- a/core/src/main/java/org/apache/accumulo/core/spi/balancer/TableLoadBalancer.java +++ b/core/src/main/java/org/apache/accumulo/core/spi/balancer/TableLoadBalancer.java @@ -149,10 +149,12 @@ private SortedMap getCurrentSetForTable( tserversInGroup.forEach(tsid -> { TServerStatus tss = allTServers.get(tsid); if (tss == null) { - throw new IllegalStateException("TabletServer " + tsid + " in " + groupNameInUse - + " TabletServer group, but not in set of all TabletServers"); + log.warn( + "Excluding TabletServer {} from group {} because TabletServerStatus is null, likely that Manager.StatusThread.updateStatus has not discovered it yet.", + tsid, groupNameInUse); + } else { + group.put(tsid, tss); } - group.put(tsid, tss); }); return group; } diff --git a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java index de16f12e541..6d3b36026ef 100644 --- a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java +++ b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java @@ -116,6 +116,8 @@ public void adminStopAll() throws IOException { if (p.exitValue() != 0) { throw new IOException("Failed to run `accumulo admin stopAll`"); } + stopAllServers(ServerType.COMPACTOR); + stopAllServers(ServerType.SCAN_SERVER); } @Override diff --git a/pom.xml b/pom.xml index 3eb4f441783..1a7b89de867 100644 --- a/pom.xml +++ b/pom.xml @@ -154,7 +154,7 @@ 1.27.0 2.0.7 0.17.0 - 3.8.2 + 3.8.3 @@ -600,10 +600,9 @@ 2.2.1.Final - org.jline jline - 3.21.0 + 3.24.0 org.latencyutils diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/LiveTServerSet.java b/server/base/src/main/java/org/apache/accumulo/server/manager/LiveTServerSet.java index 043ea072cfc..3d30e2623ca 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/LiveTServerSet.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/LiveTServerSet.java @@ -22,6 +22,7 @@ import static org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeMissingPolicy.SKIP; import java.nio.ByteBuffer; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -62,6 +63,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.annotations.VisibleForTesting; import com.google.common.net.HostAndPort; public class LiveTServerSet implements Watcher { @@ -217,12 +219,11 @@ static class TServerInfo { } } - // The set of active tservers with locks, indexed by their name in zookeeper + // The set of active tservers with locks, indexed by their name in zookeeper. When the contents of + // this map are modified, tServersSnapshot should be set to null. private final Map current = new HashMap<>(); - // as above, indexed by TServerInstance - private final Map currentInstances = new HashMap<>(); - // as above, grouped by resource group name - private final Map> currentGroups = new HashMap<>(); + + private LiveTServersSnapshot tServersSnapshot = null; // The set of entries in zookeeper without locks, and the first time each was noticed private final Map locklessServers = new HashMap<>(); @@ -283,6 +284,9 @@ private synchronized void checkServer(final Set updates, final Set doomed, final String path, final String zPath) throws InterruptedException, KeeperException { + // invalidate the snapshot forcing it to be recomputed the next time its requested + tServersSnapshot = null; + TServerInfo info = current.get(zPath); final var zLockPath = ServiceLock.path(path + "/" + zPath); @@ -293,8 +297,6 @@ private synchronized void checkServer(final Set updates, if (info != null) { doomed.add(info.instance); current.remove(zPath); - currentInstances.remove(info.instance); - currentGroups.get(info.resourceGroup).remove(info.instance); } Long firstSeen = locklessServers.get(zPath); @@ -315,18 +317,12 @@ private synchronized void checkServer(final Set updates, TServerInfo tServerInfo = new TServerInfo(instance, new TServerConnection(address), resourceGroup); current.put(zPath, tServerInfo); - currentInstances.put(instance, tServerInfo); - currentGroups.computeIfAbsent(resourceGroup, rg -> new HashSet<>()).add(instance); } else if (!info.instance.equals(instance)) { doomed.add(info.instance); updates.add(instance); TServerInfo tServerInfo = new TServerInfo(instance, new TServerConnection(address), resourceGroup); current.put(zPath, tServerInfo); - currentInstances.remove(info.instance); - currentGroups.getOrDefault(resourceGroup, new HashSet<>()).remove(instance); - currentInstances.put(instance, tServerInfo); - currentGroups.computeIfAbsent(resourceGroup, rg -> new HashSet<>()).add(instance); } } } @@ -369,21 +365,64 @@ public synchronized TServerConnection getConnection(TServerInstance server) { if (server == null) { return null; } - TServerInfo tServerInfo = currentInstances.get(server); + TServerInfo tServerInfo = getSnapshot().tserversInfo.get(server); if (tServerInfo == null) { return null; } return tServerInfo.connection; } - public synchronized Set getCurrentServers() { - return new HashSet<>(currentInstances.keySet()); + public static class LiveTServersSnapshot { + private final Set tservers; + private final Map> tserverGroups; + + // TServerInfo is only for internal use, so this field is private w/o a getter. + private final Map tserversInfo; + + @VisibleForTesting + public LiveTServersSnapshot(Set currentServers, + Map> serverGroups) { + this.tserversInfo = null; + this.tservers = Set.copyOf(currentServers); + Map> copy = new HashMap<>(); + serverGroups.forEach((k, v) -> copy.put(k, Set.copyOf(v))); + this.tserverGroups = Collections.unmodifiableMap(copy); + } + + public LiveTServersSnapshot(Map currentServers, + Map> serverGroups) { + this.tserversInfo = Map.copyOf(currentServers); + this.tservers = this.tserversInfo.keySet(); + Map> copy = new HashMap<>(); + serverGroups.forEach((k, v) -> copy.put(k, Set.copyOf(v))); + this.tserverGroups = Collections.unmodifiableMap(copy); + } + + public Set getTservers() { + return tservers; + } + + public Map> getTserverGroups() { + return tserverGroups; + } + } + + public synchronized LiveTServersSnapshot getSnapshot() { + if (tServersSnapshot == null) { + HashMap tServerInstances = new HashMap<>(); + Map> tserversGroups = new HashMap<>(); + current.values().forEach(tServerInfo -> { + tServerInstances.put(tServerInfo.instance, tServerInfo); + tserversGroups.computeIfAbsent(tServerInfo.resourceGroup, rg -> new HashSet<>()) + .add(tServerInfo.instance); + }); + tServersSnapshot = new LiveTServersSnapshot(tServerInstances, tserversGroups); + } + return tServersSnapshot; } - public synchronized Map> getCurrentServersGroups() { - Map> copy = new HashMap<>(); - currentGroups.forEach((k, v) -> copy.put(k, new HashSet<>(v))); - return copy; + public synchronized Set getCurrentServers() { + return getSnapshot().getTservers(); } public synchronized int size() { @@ -420,6 +459,10 @@ TServerInstance find(Map servers, String tabletServer) { } public synchronized void remove(TServerInstance server) { + + // invalidate the snapshot forcing it to be recomputed the next time its requested + tServersSnapshot = null; + String zPath = null; for (Entry entry : current.entrySet()) { if (entry.getValue().instance.equals(server)) { @@ -431,7 +474,6 @@ public synchronized void remove(TServerInstance server) { return; } current.remove(zPath); - currentInstances.remove(server); log.info("Removing zookeeper lock for {}", server); String fullpath = context.getZooKeeperRoot() + Constants.ZTSERVERS + "/" + zPath; diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/CurrentState.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/CurrentState.java deleted file mode 100644 index 90745603d28..00000000000 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/CurrentState.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.accumulo.server.manager.state; - -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.accumulo.core.data.TableId; -import org.apache.accumulo.core.dataImpl.KeyExtent; -import org.apache.accumulo.core.manager.thrift.ManagerState; -import org.apache.accumulo.core.metadata.TServerInstance; -import org.apache.accumulo.core.util.Pair; -import org.apache.hadoop.fs.Path; - -public interface CurrentState { - - Set onlineTables(); - - Set onlineTabletServers(); - - Map> tServerResourceGroups(); - - Set shutdownServers(); - - /** - * Provide an immutable snapshot view of migrating tablets. Objects contained in the set may still - * be mutable. - */ - Set migrationsSnapshot(); - - ManagerState getManagerState(); - - List> getVolumeReplacements(); - - // ELASTICITIY_TODO it would be nice if this method could take DataLevel as an argument and only - // retrieve information about compactions in that data level. Attempted this and a lot of - // refactoring was needed to get that small bit of information to this method. Would be best to - // address this after issue. May be best to attempt this after #3576. - Map> getCompactionHints(); -} diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java index ce2b3ede023..8545ecd7c75 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java @@ -55,8 +55,9 @@ public String name() { } @Override - public ClosableIterator iterator(List ranges) { - return wrapped.iterator(ranges); + public ClosableIterator iterator(List ranges, + TabletManagementParameters parameters) { + return wrapped.iterator(ranges, parameters); } @Override diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java index 2a2ee11c095..a182745da5d 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java @@ -30,26 +30,25 @@ import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.metadata.schema.TabletMetadata; +import com.google.common.base.Preconditions; + class MetaDataStateStore extends AbstractTabletStateStore implements TabletStateStore { protected final ClientContext context; - protected final CurrentState state; private final String targetTableName; private final Ample ample; private final DataLevel level; - protected MetaDataStateStore(DataLevel level, ClientContext context, CurrentState state, - String targetTableName) { + protected MetaDataStateStore(DataLevel level, ClientContext context, String targetTableName) { super(context); this.level = level; this.context = context; - this.state = state; this.ample = context.getAmple(); this.targetTableName = targetTableName; } - MetaDataStateStore(DataLevel level, ClientContext context, CurrentState state) { - this(level, context, state, MetadataTable.NAME); + MetaDataStateStore(DataLevel level, ClientContext context) { + this(level, context, MetadataTable.NAME); } @Override @@ -58,8 +57,10 @@ public DataLevel getLevel() { } @Override - public ClosableIterator iterator(List ranges) { - return new TabletManagementScanner(context, ranges, state, targetTableName); + public ClosableIterator iterator(List ranges, + TabletManagementParameters parameters) { + Preconditions.checkArgument(parameters.getLevel() == getLevel()); + return new TabletManagementScanner(context, ranges, parameters, targetTableName); } @Override diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/RootTabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/RootTabletStateStore.java index 4b574ade13a..97c9f7ec32d 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/RootTabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/RootTabletStateStore.java @@ -26,15 +26,19 @@ import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; +import com.google.common.base.Preconditions; + class RootTabletStateStore extends MetaDataStateStore { - RootTabletStateStore(DataLevel level, ClientContext context, CurrentState state) { - super(level, context, state, RootTable.NAME); + RootTabletStateStore(DataLevel level, ClientContext context) { + super(level, context, RootTable.NAME); } @Override - public ClosableIterator iterator(List ranges) { - return new TabletManagementScanner(context, ranges, state, RootTable.NAME); + public ClosableIterator iterator(List ranges, + TabletManagementParameters parameters) { + Preconditions.checkArgument(parameters.getLevel() == getLevel()); + return new TabletManagementScanner(context, ranges, parameters, RootTable.NAME); } @Override diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletGoalState.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletGoalState.java new file mode 100644 index 00000000000..4dedbe0bf81 --- /dev/null +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletGoalState.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.manager.state; + +import org.apache.accumulo.core.data.TabletId; +import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.dataImpl.TabletIdImpl; +import org.apache.accumulo.core.manager.balancer.TabletServerIdImpl; +import org.apache.accumulo.core.metadata.TServerInstance; +import org.apache.accumulo.core.metadata.TabletState; +import org.apache.accumulo.core.metadata.schema.Ample; +import org.apache.accumulo.core.metadata.schema.TabletMetadata; +import org.apache.accumulo.core.metadata.schema.TabletOperationType; +import org.apache.accumulo.core.spi.balancer.TabletBalancer; +import org.apache.accumulo.core.spi.balancer.data.TabletServerId; +import org.apache.accumulo.core.tablet.thrift.TUnloadTabletGoal; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Preconditions; + +public enum TabletGoalState { + + HOSTED(TUnloadTabletGoal.UNKNOWN), + UNASSIGNED(TUnloadTabletGoal.UNASSIGNED), + SUSPENDED(TUnloadTabletGoal.SUSPENDED); + + private final TUnloadTabletGoal unloadGoal; + + TabletGoalState(TUnloadTabletGoal unloadGoal) { + this.unloadGoal = unloadGoal; + } + + /** The purpose of unloading this tablet. */ + public TUnloadTabletGoal howUnload() { + return unloadGoal; + } + + private static final Logger log = LoggerFactory.getLogger(TabletGoalState.class); + + public static TabletGoalState compute(TabletMetadata tm, TabletState currentState, + TabletBalancer balancer, TabletManagementParameters params) { + Preconditions.checkArgument(Ample.DataLevel.of(tm.getTableId()) == params.getLevel(), + "Tablet %s not in expected level %s", tm.getExtent(), params.getLevel()); + + // Always follow through with assignments + if (currentState == TabletState.ASSIGNED) { + return HOSTED; + } + + KeyExtent extent = tm.getExtent(); + // Shutting down? + TabletGoalState systemGoalState = getSystemGoalState(tm, params); + + if (systemGoalState == TabletGoalState.HOSTED) { + if (!params.isParentLevelUpgraded()) { + // The place where this tablet stores its metadata was not upgraded, so do not assign this + // tablet yet. + return UNASSIGNED; + } + + // When an operation id is set tablets need to be unassigned unless there are still wals. When + // there are wals the tablet needs to be hosted to recover data in them. However, deleting + // tablets do not need to recover wals. + if (tm.getOperationId() != null && (tm.getLogs().isEmpty() + || tm.getOperationId().getType() == TabletOperationType.DELETING)) { + return TabletGoalState.UNASSIGNED; + } + + if (!params.isTableOnline(tm.getTableId())) { + return UNASSIGNED; + } + + switch (tm.getHostingGoal()) { + case NEVER: + return UNASSIGNED; + case ONDEMAND: + if (!tm.getHostingRequested()) { + return UNASSIGNED; + } + } + + TServerInstance dest = params.getMigrations().get(extent); + if (dest != null && tm.hasCurrent() && !dest.equals(tm.getLocation().getServerInstance())) { + return UNASSIGNED; + } + + if (currentState == TabletState.HOSTED && balancer != null) { + // see if the balancer thinks this tablet needs to be unassigned + + Preconditions.checkArgument( + tm.getLocation().getType() == TabletMetadata.LocationType.CURRENT, + "Expected current tablet location %s %s", tm.getExtent(), tm.getLocation()); + var tsii = new TabletServerIdImpl(tm.getLocation().getServerInstance()); + + var resourceGroup = params.getResourceGroup(tm.getLocation().getServerInstance()); + + if (resourceGroup != null) { + var reassign = balancer.needsReassignment(new TabletBalancer.CurrentAssignment() { + @Override + public TabletId getTablet() { + return new TabletIdImpl(tm.getExtent()); + } + + @Override + public TabletServerId getTabletServer() { + return tsii; + } + + @Override + public String getResourceGroup() { + return resourceGroup; + } + }); + + if (reassign) { + return UNASSIGNED; + } + } else { + // ELASTICITY_TODO this log level was set to error so that this case can be examined for + // bugs. A tablet server should always have a resource group. If there are unavoidable + // race conditions for getting tablet servers and their RGs, that that should be handled + // in the TabletManagementParameters data acquisition phase so that not all code has to + // deal with it. Eventually this log level should possibly be adjusted or converted to an + // exception. + log.error( + "Could not find resource group for tserver {}, so did not consult balancer. Need to determine the cause of this.", + tm.getLocation().getServerInstance()); + } + } + + if (tm.hasCurrent() + && params.getServersToShutdown().contains(tm.getLocation().getServerInstance())) { + if (params.canSuspendTablets()) { + return SUSPENDED; + } else { + return UNASSIGNED; + } + } + } + return systemGoalState; + } + + private static TabletGoalState getSystemGoalState(TabletMetadata tm, + TabletManagementParameters params) { + switch (params.getManagerState()) { + case NORMAL: + return HOSTED; + case HAVE_LOCK: // fall-through intended + case INITIAL: // fall-through intended + case SAFE_MODE: + if (tm.getExtent().isMeta()) { + return HOSTED; + } + return TabletGoalState.UNASSIGNED; + case UNLOAD_METADATA_TABLETS: + if (tm.getExtent().isRootTablet()) { + return HOSTED; + } + return UNASSIGNED; + case UNLOAD_ROOT_TABLET: + case STOP: + return UNASSIGNED; + default: + throw new IllegalStateException("Unknown Manager State"); + } + } +} diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java index f25c141236d..99ef16de483 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementIterator.java @@ -18,17 +18,10 @@ */ package org.apache.accumulo.server.manager.state; -import static org.apache.accumulo.core.util.LazySingletons.GSON; - import java.io.IOException; -import java.lang.reflect.Type; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Base64; -import java.util.Collection; import java.util.Collections; import java.util.EnumSet; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -39,26 +32,21 @@ import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.ScannerBase; -import org.apache.accumulo.core.client.admin.TabletHostingGoal; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.conf.ConfigurationTypeHelper; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.iterators.IteratorEnvironment; import org.apache.accumulo.core.iterators.SkippingIterator; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iterators.user.WholeRowIterator; -import org.apache.accumulo.core.manager.balancer.TabletServerIdImpl; import org.apache.accumulo.core.manager.state.TabletManagement; import org.apache.accumulo.core.manager.state.TabletManagement.ManagementAction; import org.apache.accumulo.core.manager.thrift.ManagerState; import org.apache.accumulo.core.metadata.ReferencedTabletFile; import org.apache.accumulo.core.metadata.StoredTabletFile; -import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.TabletState; import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CurrentLocationColumnFamily; @@ -74,24 +62,15 @@ import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.spi.balancer.SimpleLoadBalancer; import org.apache.accumulo.core.spi.balancer.TabletBalancer; -import org.apache.accumulo.core.spi.balancer.data.TabletServerId; import org.apache.accumulo.core.spi.compaction.CompactionKind; import org.apache.accumulo.core.tabletserver.log.LogEntry; -import org.apache.accumulo.core.util.AddressUtil; -import org.apache.accumulo.core.util.Pair; import org.apache.accumulo.server.compaction.CompactionJobGenerator; import org.apache.accumulo.server.fs.VolumeUtil; import org.apache.accumulo.server.iterators.TabletIteratorEnvironment; import org.apache.accumulo.server.manager.balancer.BalancerEnvironmentImpl; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.DataInputBuffer; -import org.apache.hadoop.io.DataOutputBuffer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.base.Joiner; -import com.google.gson.reflect.TypeToken; - /** * Iterator used by the TabletGroupWatcher threads in the Manager. This iterator returns * TabletManagement objects for each Tablet that needs some type of action performed on it by the @@ -99,163 +78,10 @@ */ public class TabletManagementIterator extends SkippingIterator { private static final Logger LOG = LoggerFactory.getLogger(TabletManagementIterator.class); - - private static final String SERVERS_OPTION = "servers"; - private static final String TABLES_OPTION = "tables"; - private static final String MIGRATIONS_OPTION = "migrations"; - private static final String MANAGER_STATE_OPTION = "managerState"; - private static final String SHUTTING_DOWN_OPTION = "shuttingDown"; - private static final String RESOURCE_GROUPS = "resourceGroups"; - private static final String TSERVER_GROUP_PREFIX = "serverGroups_"; - private static final String COMPACTION_HINTS_OPTIONS = "compactionHints"; - private static final String VOLUME_REPLACEMENTS_OPTION = "replacements"; + private static final String TABLET_GOAL_STATE_PARAMS_OPTION = "tgsParams"; private CompactionJobGenerator compactionGenerator; private TabletBalancer balancer; - private static void setCurrentServers(final IteratorSetting cfg, - final Set goodServers) { - if (goodServers != null) { - List servers = new ArrayList<>(); - for (TServerInstance server : goodServers) { - servers.add(server.getHostPortSession()); - } - cfg.addOption(SERVERS_OPTION, Joiner.on(",").join(servers)); - } - } - - private static void setOnlineTables(final IteratorSetting cfg, final Set onlineTables) { - if (onlineTables != null) { - cfg.addOption(TABLES_OPTION, Joiner.on(",").join(onlineTables)); - } - } - - private static void setMigrations(final IteratorSetting cfg, - final Collection migrations) { - DataOutputBuffer buffer = new DataOutputBuffer(); - try { - for (KeyExtent extent : migrations) { - extent.writeTo(buffer); - } - } catch (Exception ex) { - throw new RuntimeException(ex); - } - String encoded = - Base64.getEncoder().encodeToString(Arrays.copyOf(buffer.getData(), buffer.getLength())); - cfg.addOption(MIGRATIONS_OPTION, encoded); - } - - private static void setManagerState(final IteratorSetting cfg, final ManagerState state) { - cfg.addOption(MANAGER_STATE_OPTION, state.toString()); - } - - private static void setShuttingDown(final IteratorSetting cfg, - final Set servers) { - if (servers != null) { - cfg.addOption(SHUTTING_DOWN_OPTION, Joiner.on(",").join(servers)); - } - } - - private static void setCompactionHints(final IteratorSetting cfg, - Map> allHints) { - cfg.addOption(COMPACTION_HINTS_OPTIONS, GSON.get().toJson(allHints)); - } - - private static void setTServerResourceGroups(final IteratorSetting cfg, - Map> tServerResourceGroups) { - if (tServerResourceGroups == null) { - return; - } - cfg.addOption(RESOURCE_GROUPS, Joiner.on(",").join(tServerResourceGroups.keySet())); - for (Entry> entry : tServerResourceGroups.entrySet()) { - cfg.addOption(TSERVER_GROUP_PREFIX + entry.getKey(), Joiner.on(",").join(entry.getValue())); - } - } - - private static void setVolumeReplacements(final IteratorSetting cfg, - List> replacements) { - if (replacements == null || replacements.isEmpty()) { - return; - } - cfg.addOption(VOLUME_REPLACEMENTS_OPTION, GSON.get().toJson(replacements)); - } - - private static Map parseTServerResourceGroups(Map options) { - Map resourceGroups = new HashMap<>(); - String groups = options.get(RESOURCE_GROUPS); - if (groups != null) { - for (String groupName : groups.split(",")) { - String groupServers = options.get(TSERVER_GROUP_PREFIX + groupName); - if (groupServers != null) { - Set servers = parseServers(groupServers); - servers.forEach(server -> resourceGroups.put(new TabletServerIdImpl(server), groupName)); - } - } - } - return resourceGroups; - } - - private static Set parseMigrations(final String migrations) { - Set result = new HashSet<>(); - if (migrations != null) { - try { - DataInputBuffer buffer = new DataInputBuffer(); - byte[] data = Base64.getDecoder().decode(migrations); - buffer.reset(data, data.length); - while (buffer.available() > 0) { - result.add(KeyExtent.readFrom(buffer)); - } - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } - return result; - } - - private static Set parseTableIDs(final String tableIDs) { - Set result = new HashSet<>(); - if (tableIDs != null) { - for (String tableID : tableIDs.split(",")) { - result.add(TableId.of(tableID)); - } - } - return result; - } - - private static Set parseServers(final String servers) { - Set result = new HashSet<>(); - if (servers != null) { - // parse "host:port[INSTANCE]" - if (!servers.isEmpty()) { - for (String part : servers.split(",")) { - String[] parts = part.split("\\[", 2); - String hostport = parts[0]; - String instance = parts[1]; - if (instance != null && instance.endsWith("]")) { - instance = instance.substring(0, instance.length() - 1); - } - result.add(new TServerInstance(AddressUtil.parseAddress(hostport, false), instance)); - } - } - } - return result; - } - - private static Map> parseCompactionHints(String json) { - if (json == null) { - return Map.of(); - } - Type tt = new TypeToken>>() {}.getType(); - return GSON.get().fromJson(json, tt); - } - - private static List> parseVolumeReplacements(final String json) { - if (json == null) { - return List.of(); - } - Type tt = new TypeToken>>() {}.getType(); - return GSON.get().fromJson(json, tt); - } - private static boolean shouldReturnDueToSplit(final TabletMetadata tm, final long splitThreshold) { final long sumOfFileSizes = @@ -266,51 +92,38 @@ private static boolean shouldReturnDueToSplit(final TabletMetadata tm, return shouldSplit; } - private boolean shouldReturnDueToLocation(final TabletMetadata tm, - final Set onlineTables, final Set current) { + private boolean shouldReturnDueToLocation(final TabletMetadata tm) { - if (migrations.contains(tm.getExtent())) { + if (tabletMgmtParams.getMigrations().containsKey(tm.getExtent())) { + // Ideally only the state and goalState would need to be used to determine if a tablet should + // be returned. However, the Manager/TGW currently needs everything in the migrating set + // returned so it can update in memory maps it has. If this were improved then this case would + // not be needed. return true; } - // is the table supposed to be online or offline? - final boolean shouldBeOnline = - onlineTables.contains(tm.getTableId()) && tm.getOperationId() == null; - - TabletState state = TabletState.compute(tm, current, balancer, tserverResourceGroups); + TabletState state = TabletState.compute(tm, tabletMgmtParams.getOnlineTsevers()); + TabletGoalState goalState = TabletGoalState.compute(tm, state, balancer, tabletMgmtParams); if (LOG.isTraceEnabled()) { - LOG.trace( - "{} is {}. Table is {}line. Tablet hosting goal is {}, hostingRequested: {}, opId: {}", - tm.getExtent(), state, (shouldBeOnline ? "on" : "off"), tm.getHostingGoal(), - tm.getHostingRequested(), tm.getOperationId()); + LOG.trace("extent:{} state:{} goalState:{} hostingGoal:{}, hostingRequested: {}, opId: {}", + tm.getExtent(), state, goalState, tm.getHostingGoal(), tm.getHostingRequested(), + tm.getOperationId()); } - switch (state) { - case ASSIGNED: - // we always want data about assigned tablets - return true; + + switch (goalState) { case HOSTED: - if (!shouldBeOnline || tm.getHostingGoal() == TabletHostingGoal.NEVER - || (tm.getHostingGoal() == TabletHostingGoal.ONDEMAND && !tm.getHostingRequested())) { - return true; - } - break; - case ASSIGNED_TO_DEAD_SERVER: - case NEEDS_REASSIGNMENT: - return true; + return state != TabletState.HOSTED; case SUSPENDED: + return state != TabletState.SUSPENDED; case UNASSIGNED: - if (shouldBeOnline && (tm.getHostingGoal() == TabletHostingGoal.ALWAYS - || (tm.getHostingGoal() == TabletHostingGoal.ONDEMAND && tm.getHostingRequested()))) { - return true; - } - break; + return state != TabletState.UNASSIGNED; default: - throw new AssertionError("Inconceivable! The tablet is an unrecognized state: " + state); + throw new IllegalStateException("unknown goal state " + goalState); } - return false; } - public static void configureScanner(final ScannerBase scanner, final CurrentState state) { + public static void configureScanner(final ScannerBase scanner, + final TabletManagementParameters tabletMgmtParams) { // TODO so many columns are being fetch it may not make sense to fetch columns TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner); ServerColumnFamily.DIRECTORY_COLUMN.fetch(scanner); @@ -327,17 +140,7 @@ public static void configureScanner(final ScannerBase scanner, final CurrentStat scanner.addScanIterator(new IteratorSetting(1000, "wholeRows", WholeRowIterator.class)); IteratorSetting tabletChange = new IteratorSetting(1001, "ManagerTabletInfoIterator", TabletManagementIterator.class); - if (state != null) { - TabletManagementIterator.setCurrentServers(tabletChange, state.onlineTabletServers()); - TabletManagementIterator.setOnlineTables(tabletChange, state.onlineTables()); - TabletManagementIterator.setMigrations(tabletChange, state.migrationsSnapshot()); - TabletManagementIterator.setManagerState(tabletChange, state.getManagerState()); - TabletManagementIterator.setShuttingDown(tabletChange, state.shutdownServers()); - TabletManagementIterator.setTServerResourceGroups(tabletChange, - state.tServerResourceGroups()); - TabletManagementIterator.setCompactionHints(tabletChange, state.getCompactionHints()); - TabletManagementIterator.setVolumeReplacements(tabletChange, state.getVolumeReplacements()); - } + tabletChange.addOption(TABLET_GOAL_STATE_PARAMS_OPTION, tabletMgmtParams.serialize()); scanner.addScanIterator(tabletChange); } @@ -345,40 +148,20 @@ public static TabletManagement decode(Entry e) throws IOException { return new TabletManagement(e.getKey(), e.getValue()); } - private final Set current = new HashSet<>(); - private final Set onlineTables = new HashSet<>(); - private final Map tserverResourceGroups = new HashMap<>(); - private final Set migrations = new HashSet<>(); - private final List> volumeReplacements = new ArrayList<>(); - private ManagerState managerState = ManagerState.NORMAL; private IteratorEnvironment env; private Key topKey = null; private Value topValue = null; + private TabletManagementParameters tabletMgmtParams = null; @Override public void init(SortedKeyValueIterator source, Map options, IteratorEnvironment env) throws IOException { super.init(source, options, env); this.env = env; - current.addAll(parseServers(options.get(SERVERS_OPTION))); - onlineTables.addAll(parseTableIDs(options.get(TABLES_OPTION))); - tserverResourceGroups.putAll(parseTServerResourceGroups(options)); - migrations.addAll(parseMigrations(options.get(MIGRATIONS_OPTION))); - volumeReplacements.addAll(parseVolumeReplacements(options.get(VOLUME_REPLACEMENTS_OPTION))); - String managerStateOptionValue = options.get(MANAGER_STATE_OPTION); - try { - managerState = ManagerState.valueOf(managerStateOptionValue); - } catch (RuntimeException ex) { - if (managerStateOptionValue != null) { - LOG.error("Unable to decode managerState {}", managerStateOptionValue); - } - } - Set shuttingDown = parseServers(options.get(SHUTTING_DOWN_OPTION)); - if (shuttingDown != null) { - current.removeAll(shuttingDown); - } - compactionGenerator = new CompactionJobGenerator(env.getPluginEnv(), - parseCompactionHints(options.get(COMPACTION_HINTS_OPTIONS))); + tabletMgmtParams = + TabletManagementParameters.deserialize(options.get(TABLET_GOAL_STATE_PARAMS_OPTION)); + compactionGenerator = + new CompactionJobGenerator(env.getPluginEnv(), tabletMgmtParams.getCompactionHints()); final AccumuloConfiguration conf = new ConfigurationCopy(env.getPluginEnv().getConfiguration()); BalancerEnvironmentImpl benv = new BalancerEnvironmentImpl(((TabletIteratorEnvironment) env).getServerContext()); @@ -419,7 +202,9 @@ protected void consume() throws IOException { actions.clear(); Exception error = null; try { - if (managerState != ManagerState.NORMAL || current.isEmpty() || onlineTables.isEmpty()) { + if (tabletMgmtParams.getManagerState() != ManagerState.NORMAL + || tabletMgmtParams.getOnlineTsevers().isEmpty() + || tabletMgmtParams.getOnlineTables().isEmpty()) { // when manager is in the process of starting up or shutting down return everything. actions.add(ManagementAction.NEEDS_LOCATION_UPDATE); } else { @@ -461,8 +246,8 @@ private boolean needsVolumeReplacement(final TabletMetadata tm) { final List filesToRemove = new ArrayList<>(); final SortedMap filesToAdd = new TreeMap<>(); - VolumeUtil.volumeReplacementEvaluation(volumeReplacements, tm, logsToRemove, logsToAdd, - filesToRemove, filesToAdd); + VolumeUtil.volumeReplacementEvaluation(tabletMgmtParams.getVolumeReplacements(), tm, + logsToRemove, logsToAdd, filesToRemove, filesToAdd); if (logsToRemove.size() + filesToRemove.size() > 0) { return true; @@ -483,11 +268,11 @@ private void computeTabletManagementActions(final TabletMetadata tm, return; } - if (!volumeReplacements.isEmpty() && needsVolumeReplacement(tm)) { + if (!tabletMgmtParams.getVolumeReplacements().isEmpty() && needsVolumeReplacement(tm)) { reasonsToReturnThisTablet.add(ManagementAction.NEEDS_VOLUME_REPLACEMENT); } - if (shouldReturnDueToLocation(tm, onlineTables, current)) { + if (shouldReturnDueToLocation(tm)) { reasonsToReturnThisTablet.add(ManagementAction.NEEDS_LOCATION_UPDATE); } diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementParameters.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementParameters.java new file mode 100644 index 00000000000..4d183cd6c6f --- /dev/null +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementParameters.java @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.manager.state; + +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; +import static java.util.stream.Collectors.toSet; +import static java.util.stream.Collectors.toUnmodifiableMap; +import static java.util.stream.Collectors.toUnmodifiableSet; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; + +import org.apache.accumulo.core.data.AbstractId; +import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.manager.thrift.ManagerState; +import org.apache.accumulo.core.metadata.TServerInstance; +import org.apache.accumulo.core.metadata.schema.Ample; +import org.apache.accumulo.core.util.Pair; +import org.apache.accumulo.server.manager.LiveTServerSet; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.DataInputBuffer; +import org.apache.hadoop.io.DataOutputBuffer; + +import com.google.common.base.Suppliers; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +/** + * An immutable snapshot of the information needed by the TabletGroupWatcher and the + * {@link TabletManagementIterator} to make decisions about tablets. + */ +public class TabletManagementParameters { + // ELASTICITY_TODO need to unit test serialization and deserialization of this class. + private final ManagerState managerState; + private final Map parentUpgradeMap; + private final Set onlineTables; + private final Set serversToShutdown; + private final Map migrations; + + private final Ample.DataLevel level; + + private final Supplier> resourceGroups; + private final Map> tserverGroups; + private final Map> compactionHints; + private final Set onlineTservers; + private final boolean canSuspendTablets; + private final List> volumeReplacements; + + public TabletManagementParameters(ManagerState managerState, + Map parentUpgradeMap, Set onlineTables, + LiveTServerSet.LiveTServersSnapshot liveTServersSnapshot, + Set serversToShutdown, Map migrations, + Ample.DataLevel level, Map> compactionHints, + boolean canSuspendTablets, List> volumeReplacements) { + this.managerState = managerState; + this.parentUpgradeMap = Map.copyOf(parentUpgradeMap); + // TODO could filter by level + this.onlineTables = Set.copyOf(onlineTables); + // This is already immutable, so no need to copy + this.onlineTservers = liveTServersSnapshot.getTservers(); + this.serversToShutdown = Set.copyOf(serversToShutdown); + // TODO could filter by level + this.migrations = Map.copyOf(migrations); + this.level = level; + // This is already immutable, so no need to copy + this.tserverGroups = liveTServersSnapshot.getTserverGroups(); + this.compactionHints = makeImmutable(compactionHints); + this.resourceGroups = Suppliers.memoize(() -> { + Map resourceGroups = new HashMap<>(); + TabletManagementParameters.this.tserverGroups.forEach((resourceGroup, tservers) -> tservers + .forEach(tserver -> resourceGroups.put(tserver, resourceGroup))); + return Map.copyOf(resourceGroups); + }); + this.canSuspendTablets = canSuspendTablets; + this.volumeReplacements = volumeReplacements; + } + + private TabletManagementParameters(JsonData jdata) { + this.managerState = jdata.managerState; + this.parentUpgradeMap = Map.copyOf(jdata.parentUpgradeMap); + this.onlineTables = jdata.onlineTables.stream().map(TableId::of).collect(toUnmodifiableSet()); + this.onlineTservers = + jdata.onlineTservers.stream().map(TServerInstance::new).collect(toUnmodifiableSet()); + this.serversToShutdown = + jdata.serversToShutdown.stream().map(TServerInstance::new).collect(toUnmodifiableSet()); + this.migrations = jdata.migrations.entrySet().stream() + .collect(toUnmodifiableMap(entry -> JsonData.strToExtent(entry.getKey()), + entry -> new TServerInstance(entry.getValue()))); + this.level = jdata.level; + this.compactionHints = makeImmutable(jdata.compactionHints); + this.tserverGroups = jdata.tserverGroups.entrySet().stream().collect(toUnmodifiableMap( + Map.Entry::getKey, + entry -> entry.getValue().stream().map(TServerInstance::new).collect(toUnmodifiableSet()))); + this.resourceGroups = Suppliers.memoize(() -> { + Map resourceGroups = new HashMap<>(); + TabletManagementParameters.this.tserverGroups.forEach((resourceGroup, tservers) -> tservers + .forEach(tserver -> resourceGroups.put(tserver, resourceGroup))); + return Map.copyOf(resourceGroups); + }); + this.canSuspendTablets = jdata.canSuspendTablets; + this.volumeReplacements = jdata.volumeReplacements; + } + + public ManagerState getManagerState() { + return managerState; + } + + public boolean isParentLevelUpgraded() { + return parentUpgradeMap.get(level); + } + + public Set getOnlineTsevers() { + return onlineTservers; + } + + public Set getServersToShutdown() { + return serversToShutdown; + } + + public boolean isTableOnline(TableId tableId) { + return onlineTables.contains(tableId); + } + + public Map getMigrations() { + return migrations; + } + + public Ample.DataLevel getLevel() { + return level; + } + + public String getResourceGroup(TServerInstance tserver) { + return resourceGroups.get().get(tserver); + } + + public Map> getGroupedTServers() { + return tserverGroups; + } + + public Set getOnlineTables() { + return onlineTables; + } + + public Map> getCompactionHints() { + return compactionHints; + } + + public boolean canSuspendTablets() { + return canSuspendTablets; + } + + public List> getVolumeReplacements() { + return volumeReplacements; + } + + private static Map> + makeImmutable(Map> compactionHints) { + var copy = new HashMap>(); + compactionHints.forEach((ftxid, hints) -> copy.put(ftxid, Map.copyOf(hints))); + return Collections.unmodifiableMap(copy); + } + + private static class JsonData { + final ManagerState managerState; + final Map parentUpgradeMap; + final Collection onlineTables; + final Collection onlineTservers; + final Collection serversToShutdown; + final Map migrations; + + final Ample.DataLevel level; + + final Map> tserverGroups; + + final Map> compactionHints; + + final boolean canSuspendTablets; + final List> volumeReplacements; + + private static String toString(KeyExtent extent) { + DataOutputBuffer buffer = new DataOutputBuffer(); + try { + extent.writeTo(buffer); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + return Base64.getEncoder() + .encodeToString(Arrays.copyOf(buffer.getData(), buffer.getLength())); + + } + + private static KeyExtent strToExtent(String kes) { + byte[] data = Base64.getDecoder().decode(kes); + DataInputBuffer buffer = new DataInputBuffer(); + buffer.reset(data, data.length); + try { + return KeyExtent.readFrom(buffer); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + JsonData(TabletManagementParameters params) { + managerState = params.managerState; + parentUpgradeMap = params.parentUpgradeMap; + onlineTables = params.onlineTables.stream().map(AbstractId::canonical).collect(toList()); + onlineTservers = params.getOnlineTsevers().stream().map(TServerInstance::getHostPortSession) + .collect(toList()); + serversToShutdown = params.serversToShutdown.stream().map(TServerInstance::getHostPortSession) + .collect(toList()); + migrations = params.migrations.entrySet().stream().collect( + toMap(entry -> toString(entry.getKey()), entry -> entry.getValue().getHostPortSession())); + level = params.level; + tserverGroups = params.getGroupedTServers().entrySet().stream() + .collect(toMap(Map.Entry::getKey, entry -> entry.getValue().stream() + .map(TServerInstance::getHostPortSession).collect(toSet()))); + compactionHints = params.compactionHints; + canSuspendTablets = params.canSuspendTablets; + volumeReplacements = params.volumeReplacements; + } + + } + + public String serialize() { + Gson gson = new GsonBuilder().disableHtmlEscaping().create(); + return gson.toJson(new JsonData(this)); + } + + public static TabletManagementParameters deserialize(String json) { + Gson gson = new GsonBuilder().disableHtmlEscaping().create(); + JsonData jdata = gson.fromJson(json, JsonData.class); + return new TabletManagementParameters(jdata); + } + +} diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementScanner.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementScanner.java index e34d16304c8..c5a02ef93aa 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementScanner.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementScanner.java @@ -59,8 +59,8 @@ public class TabletManagementScanner implements ClosableIterator ranges, CurrentState state, - String tableName) { + public TabletManagementScanner(ClientContext context, List ranges, + TabletManagementParameters tmgmtParams, String tableName) { // scan over metadata table, looking for tablets in the wrong state based on the live servers // and online tables try { @@ -90,16 +90,11 @@ public TabletManagementScanner(ClientContext context, List ranges, Curren throw new RuntimeException("Error obtaining locations for table: " + tableName); } cleanable = CleanerUtil.unclosed(this, TabletManagementScanner.class, closed, log, mdScanner); - TabletManagementIterator.configureScanner(mdScanner, state); + TabletManagementIterator.configureScanner(mdScanner, tmgmtParams); mdScanner.setRanges(ranges); iter = mdScanner.iterator(); } - // This constructor is called from utilities and tests - public TabletManagementScanner(ClientContext context, Range range, String tableName) { - this(context, List.of(range), null, tableName); - } - @Override public void close() { if (closed.compareAndSet(false, true)) { diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java index 19cc5b5832b..ee81950fe71 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java @@ -36,7 +36,7 @@ /** * Interface for storing information about tablet assignments. There are three implementations: */ -public interface TabletStateStore extends ClosableIterable { +public interface TabletStateStore { /** * Get the level for this state store @@ -52,14 +52,14 @@ public interface TabletStateStore extends ClosableIterable { * Scan the information about the tablets covered by this store that have end row in the specified * ranges. */ - ClosableIterator iterator(List ranges); + ClosableIterator iterator(List ranges, + TabletManagementParameters parameters); /** * Scan the information about all tablets covered by this store.. */ - @Override - default ClosableIterator iterator() { - return iterator(List.of(MetadataSchema.TabletsSection.getRange())); + default ClosableIterator iterator(TabletManagementParameters parameters) { + return iterator(List.of(MetadataSchema.TabletsSection.getRange()), parameters); } /** @@ -118,11 +118,6 @@ static TabletStateStore getStoreForTablet(KeyExtent extent, ServerContext contex } public static TabletStateStore getStoreForLevel(DataLevel level, ServerContext context) { - return getStoreForLevel(level, context, null); - } - - public static TabletStateStore getStoreForLevel(DataLevel level, ServerContext context, - CurrentState state) { TabletStateStore tss; switch (level) { @@ -130,10 +125,10 @@ public static TabletStateStore getStoreForLevel(DataLevel level, ServerContext c tss = new ZooTabletStateStore(level, context); break; case METADATA: - tss = new RootTabletStateStore(level, context, state); + tss = new RootTabletStateStore(level, context); break; case USER: - tss = new MetaDataStateStore(level, context, state); + tss = new MetaDataStateStore(level, context); break; default: throw new IllegalArgumentException("Unknown level " + level); diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java index c88847c5360..05f5648497e 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java @@ -40,6 +40,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Preconditions; + class ZooTabletStateStore extends AbstractTabletStateStore implements TabletStateStore { private static final Logger log = LoggerFactory.getLogger(ZooTabletStateStore.class); @@ -60,7 +62,9 @@ public DataLevel getLevel() { } @Override - public ClosableIterator iterator(List ranges) { + public ClosableIterator iterator(List ranges, + TabletManagementParameters parameters) { + Preconditions.checkArgument(parameters.getLevel() == getLevel()); return new ClosableIterator<>() { boolean finished = false; diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java b/server/base/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java index 2cd208a629b..fb28ed13e8d 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/FindOfflineTablets.java @@ -23,25 +23,18 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.accumulo.core.client.TableNotFoundException; -import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.data.TableId; -import org.apache.accumulo.core.dataImpl.KeyExtent; -import org.apache.accumulo.core.manager.state.TabletManagement; import org.apache.accumulo.core.manager.state.tables.TableState; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.TabletState; import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; -import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.trace.TraceUtil; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.cli.ServerUtilOpts; import org.apache.accumulo.server.manager.LiveTServerSet; import org.apache.accumulo.server.manager.LiveTServerSet.Listener; -import org.apache.accumulo.server.manager.state.TabletManagementScanner; -import org.apache.accumulo.server.manager.state.TabletStateStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -82,8 +75,8 @@ public void update(LiveTServerSet current, Set deleted, tservers.startListeningForTabletServerChanges(); scanning.set(true); - Iterator zooScanner = - TabletStateStore.getStoreForLevel(DataLevel.ROOT, context).iterator(); + Iterator zooScanner = + context.getAmple().readTablets().forLevel(DataLevel.ROOT).build().iterator(); int offline = 0; @@ -97,8 +90,8 @@ public void update(LiveTServerSet current, Set deleted, } System.out.println("Scanning " + RootTable.NAME); - Iterator rootScanner = - new TabletManagementScanner(context, TabletsSection.getRange(), RootTable.NAME); + Iterator rootScanner = + context.getAmple().readTablets().forLevel(DataLevel.METADATA).build().iterator(); if ((offline = checkTablets(context, rootScanner, tservers)) > 0) { return offline; } @@ -109,32 +102,24 @@ public void update(LiveTServerSet current, Set deleted, System.out.println("Scanning " + MetadataTable.NAME); - Range range = TabletsSection.getRange(); - if (tableName != null) { - TableId tableId = context.getTableId(tableName); - range = new KeyExtent(tableId, null, null).toMetaRange(); - } - - try (TabletManagementScanner metaScanner = - new TabletManagementScanner(context, range, MetadataTable.NAME)) { - return checkTablets(context, metaScanner, tservers); + try (var metaScanner = context.getAmple().readTablets().forLevel(DataLevel.USER).build()) { + return checkTablets(context, metaScanner.iterator(), tservers); } } - private static int checkTablets(ServerContext context, Iterator scanner, + private static int checkTablets(ServerContext context, Iterator scanner, LiveTServerSet tservers) { int offline = 0; while (scanner.hasNext() && !System.out.checkError()) { - TabletManagement mti = scanner.next(); - TabletMetadata tabletMetadata = mti.getTabletMetadata(); + TabletMetadata tabletMetadata = scanner.next(); Set liveTServers = tservers.getCurrentServers(); TabletState state = TabletState.compute(tabletMetadata, liveTServers); if (state != null && state != TabletState.HOSTED - && context.getTableManager().getTableState(mti.getTabletMetadata().getTableId()) + && context.getTableManager().getTableState(tabletMetadata.getTableId()) != TableState.OFFLINE) { - System.out.println( - mti + " is " + state + " #walogs:" + mti.getTabletMetadata().getLogs().size()); + System.out.println(tabletMetadata.getExtent() + " is " + state + " #walogs:" + + tabletMetadata.getLogs().size()); offline++; } } diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ListOnlineOnDemandTablets.java b/server/base/src/main/java/org/apache/accumulo/server/util/ListOnlineOnDemandTablets.java index 8bf64157b87..70e02e5ebc1 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/ListOnlineOnDemandTablets.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/ListOnlineOnDemandTablets.java @@ -24,19 +24,17 @@ import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.admin.TabletHostingGoal; -import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.manager.state.TabletManagement; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.TabletState; -import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; +import org.apache.accumulo.core.metadata.schema.Ample; import org.apache.accumulo.core.metadata.schema.TabletMetadata; +import org.apache.accumulo.core.metadata.schema.TabletsMetadata; import org.apache.accumulo.core.trace.TraceUtil; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.cli.ServerUtilOpts; import org.apache.accumulo.server.manager.LiveTServerSet; import org.apache.accumulo.server.manager.LiveTServerSet.Listener; -import org.apache.accumulo.server.manager.state.TabletManagementScanner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,27 +77,24 @@ public void update(LiveTServerSet current, Set deleted, System.out.println("Scanning " + MetadataTable.NAME); - Range range = TabletsSection.getRange(); - - try (TabletManagementScanner metaScanner = - new TabletManagementScanner(context, range, MetadataTable.NAME)) { - return checkTablets(context, metaScanner, tservers); + try (TabletsMetadata metaScanner = + context.getAmple().readTablets().forLevel(Ample.DataLevel.USER).build()) { + return checkTablets(context, metaScanner.iterator(), tservers); } } - private static int checkTablets(ServerContext context, Iterator scanner, + private static int checkTablets(ServerContext context, Iterator scanner, LiveTServerSet tservers) { int online = 0; while (scanner.hasNext() && !System.out.checkError()) { - final TabletManagement mti = scanner.next(); - TabletMetadata tabletMetadata = mti.getTabletMetadata(); + TabletMetadata tabletMetadata = scanner.next(); Set liveTServers = tservers.getCurrentServers(); TabletState state = TabletState.compute(tabletMetadata, liveTServers); if (state == TabletState.HOSTED - && mti.getTabletMetadata().getHostingGoal() == TabletHostingGoal.ONDEMAND) { - System.out.println( - mti + " is " + state + " #walogs:" + mti.getTabletMetadata().getLogs().size()); + && tabletMetadata.getHostingGoal() == TabletHostingGoal.ONDEMAND) { + System.out.println(tabletMetadata.getExtent() + " is " + state + " #walogs:" + + tabletMetadata.getLogs().size()); online++; } } diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java index f783d77ac5f..fba1867b475 100644 --- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java +++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java @@ -18,6 +18,12 @@ */ package org.apache.accumulo.gc; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LAST; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOGS; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SUSPEND; + import java.io.FileNotFoundException; import java.io.IOException; import java.util.Collection; @@ -31,7 +37,6 @@ import org.apache.accumulo.core.gc.thrift.GCStatus; import org.apache.accumulo.core.gc.thrift.GcCycleStats; -import org.apache.accumulo.core.manager.state.TabletManagement; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.TabletState; import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; @@ -45,7 +50,6 @@ import org.apache.accumulo.server.log.WalStateManager.WalMarkerException; import org.apache.accumulo.server.log.WalStateManager.WalState; import org.apache.accumulo.server.manager.LiveTServerSet; -import org.apache.accumulo.server.manager.state.TabletStateStore; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; @@ -64,7 +68,7 @@ public class GarbageCollectWriteAheadLogs { private final VolumeManager fs; private final LiveTServerSet liveServers; private final WalStateManager walMarker; - private final Iterable store; + private final Iterable store; /** * Creates a new GC WAL object. @@ -79,9 +83,12 @@ public class GarbageCollectWriteAheadLogs { this.liveServers = liveServers; this.walMarker = new WalStateManager(context); this.store = () -> Iterators.concat( - TabletStateStore.getStoreForLevel(DataLevel.ROOT, context).iterator(), - TabletStateStore.getStoreForLevel(DataLevel.METADATA, context).iterator(), - TabletStateStore.getStoreForLevel(DataLevel.USER, context).iterator()); + context.getAmple().readTablets().forLevel(DataLevel.ROOT) + .fetch(LOCATION, LAST, LOGS, PREV_ROW, SUSPEND).checkConsistency().build().iterator(), + context.getAmple().readTablets().forLevel(DataLevel.METADATA) + .fetch(LOCATION, LAST, LOGS, PREV_ROW, SUSPEND).checkConsistency().build().iterator(), + context.getAmple().readTablets().forLevel(DataLevel.USER) + .fetch(LOCATION, LAST, LOGS, PREV_ROW, SUSPEND).checkConsistency().build().iterator()); } /** @@ -93,7 +100,7 @@ public class GarbageCollectWriteAheadLogs { */ @VisibleForTesting GarbageCollectWriteAheadLogs(ServerContext context, VolumeManager fs, - LiveTServerSet liveTServerSet, WalStateManager walMarker, Iterable store) { + LiveTServerSet liveTServerSet, WalStateManager walMarker, Iterable store) { this.context = context; this.fs = fs; this.liveServers = liveTServerSet; @@ -275,13 +282,11 @@ private Map removeEntriesInUse(Map idsToIgnore = - candidates.remove(mti.getTabletMetadata().getLocation().getServerInstance()); + Set idsToIgnore = candidates.remove(tabletMetadata.getLocation().getServerInstance()); if (idsToIgnore != null) { result.keySet().removeAll(idsToIgnore); recoveryLogs.keySet().removeAll(idsToIgnore); @@ -289,7 +294,7 @@ private Map removeEntriesInUse(Map walogs = Collections.emptyList(); - private final TabletManagement tabletAssignedToServer1; - private final TabletManagement tabletAssignedToServer2; + private final TabletMetadata tabletAssignedToServer1; + private final TabletMetadata tabletAssignedToServer2; { try { - tabletAssignedToServer1 = new TabletManagement(Set.of(), + tabletAssignedToServer1 = TabletMetadata.builder(extent).putLocation(Location.current(server1)) - .putHostingGoal(TabletHostingGoal.ALWAYS).build(LAST, SUSPEND, LOGS), - ""); - tabletAssignedToServer2 = new TabletManagement(Set.of(), + .putHostingGoal(TabletHostingGoal.ALWAYS).build(LAST, SUSPEND, LOGS); + tabletAssignedToServer2 = TabletMetadata.builder(extent).putLocation(Location.current(server2)) - .putHostingGoal(TabletHostingGoal.NEVER).build(LAST, SUSPEND, LOGS), - ""); + .putHostingGoal(TabletHostingGoal.NEVER).build(LAST, SUSPEND, LOGS); } catch (Exception ex) { throw new RuntimeException(ex); } } - private final Iterable tabletOnServer1List = + private final Iterable tabletOnServer1List = Collections.singletonList(tabletAssignedToServer1); - private final Iterable tabletOnServer2List = + private final Iterable tabletOnServer2List = Collections.singletonList(tabletAssignedToServer2); @Test diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java index cb5446f9310..6608b566381 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java @@ -99,7 +99,6 @@ import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily; -import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metrics.MetricsUtil; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.spi.balancer.BalancerEnvironment; @@ -108,10 +107,8 @@ import org.apache.accumulo.core.spi.balancer.data.TServerStatus; import org.apache.accumulo.core.spi.balancer.data.TabletMigration; import org.apache.accumulo.core.spi.balancer.data.TabletServerId; -import org.apache.accumulo.core.tablet.thrift.TUnloadTabletGoal; import org.apache.accumulo.core.trace.TraceUtil; import org.apache.accumulo.core.util.Halt; -import org.apache.accumulo.core.util.Pair; import org.apache.accumulo.core.util.Retry; import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.core.util.threads.Threads; @@ -130,9 +127,9 @@ import org.apache.accumulo.server.compaction.CompactionConfigStorage; import org.apache.accumulo.server.fs.VolumeManager; import org.apache.accumulo.server.manager.LiveTServerSet; +import org.apache.accumulo.server.manager.LiveTServerSet.LiveTServersSnapshot; import org.apache.accumulo.server.manager.LiveTServerSet.TServerConnection; import org.apache.accumulo.server.manager.balancer.BalancerEnvironmentImpl; -import org.apache.accumulo.server.manager.state.CurrentState; import org.apache.accumulo.server.manager.state.DeadServerList; import org.apache.accumulo.server.manager.state.TabletServerState; import org.apache.accumulo.server.manager.state.TabletStateStore; @@ -149,7 +146,6 @@ import org.apache.accumulo.server.util.ScanServerMetadataEntries; import org.apache.accumulo.server.util.ServerBulkImportStatus; import org.apache.accumulo.server.util.TableInfoUtil; -import org.apache.hadoop.fs.Path; import org.apache.thrift.TException; import org.apache.thrift.server.TServer; import org.apache.thrift.transport.TTransportException; @@ -175,7 +171,7 @@ * The manager will also coordinate log recoveries and reports general status. */ public class Manager extends AbstractServer - implements LiveTServerSet.Listener, TableObserver, CurrentState, HighlyAvailableService { + implements LiveTServerSet.Listener, TableObserver, HighlyAvailableService { static final Logger log = LoggerFactory.getLogger(Manager.class); @@ -237,12 +233,14 @@ public class Manager extends AbstractServer private final TabletStateStore metadataTabletStore; private final TabletStateStore userTabletStore; - @Override public synchronized ManagerState getManagerState() { return state; } - @Override + // ELASTICITIY_TODO it would be nice if this method could take DataLevel as an argument and only + // retrieve information about compactions in that data level. Attempted this and a lot of + // refactoring was needed to get that small bit of information to this method. Would be best to + // address this after issue. May be best to attempt this after #3576. public Map> getCompactionHints() { Map allConfig = null; try { @@ -456,9 +454,9 @@ public static void main(String[] args) throws Exception { final long tokenLifetime = aconf.getTimeInMillis(Property.GENERAL_DELEGATION_TOKEN_LIFETIME); - this.rootTabletStore = TabletStateStore.getStoreForLevel(DataLevel.ROOT, context, this); - this.metadataTabletStore = TabletStateStore.getStoreForLevel(DataLevel.METADATA, context, this); - this.userTabletStore = TabletStateStore.getStoreForLevel(DataLevel.USER, context, this); + this.rootTabletStore = TabletStateStore.getStoreForLevel(DataLevel.ROOT, context); + this.metadataTabletStore = TabletStateStore.getStoreForLevel(DataLevel.METADATA, context); + this.userTabletStore = TabletStateStore.getStoreForLevel(DataLevel.USER, context); authenticationTokenKeyManager = null; keyDistributor = null; @@ -544,110 +542,8 @@ public CompactionJobQueues getCompactionQueues() { return compactionJobQueues; } - enum TabletGoalState { - HOSTED(TUnloadTabletGoal.UNKNOWN), - UNASSIGNED(TUnloadTabletGoal.UNASSIGNED), - DELETED(TUnloadTabletGoal.DELETED), - SUSPENDED(TUnloadTabletGoal.SUSPENDED); - - private final TUnloadTabletGoal unloadGoal; - - TabletGoalState(TUnloadTabletGoal unloadGoal) { - this.unloadGoal = unloadGoal; - } - - /** The purpose of unloading this tablet. */ - public TUnloadTabletGoal howUnload() { - return unloadGoal; - } - } - - TabletGoalState getSystemGoalState(TabletMetadata tm) { - switch (getManagerState()) { - case NORMAL: - return TabletGoalState.HOSTED; - case HAVE_LOCK: // fall-through intended - case INITIAL: // fall-through intended - case SAFE_MODE: - if (tm.getExtent().isMeta()) { - return TabletGoalState.HOSTED; - } - return TabletGoalState.UNASSIGNED; - case UNLOAD_METADATA_TABLETS: - if (tm.getExtent().isRootTablet()) { - return TabletGoalState.HOSTED; - } - return TabletGoalState.UNASSIGNED; - case UNLOAD_ROOT_TABLET: - return TabletGoalState.UNASSIGNED; - case STOP: - return TabletGoalState.UNASSIGNED; - default: - throw new IllegalStateException("Unknown Manager State"); - } - } - - TabletGoalState getTableGoalState(TabletMetadata tm) { - TableState tableState = getContext().getTableManager().getTableState(tm.getTableId()); - if (tableState == null) { - return TabletGoalState.DELETED; - } - switch (tableState) { - case DELETING: - return TabletGoalState.DELETED; - case OFFLINE: - case NEW: - return TabletGoalState.UNASSIGNED; - default: - switch (tm.getHostingGoal()) { - case ALWAYS: - return TabletGoalState.HOSTED; - case NEVER: - return TabletGoalState.UNASSIGNED; - case ONDEMAND: - if (tm.getHostingRequested()) { - return TabletGoalState.HOSTED; - } else { - return TabletGoalState.UNASSIGNED; - } - default: - throw new IllegalStateException( - "Tablet Hosting Goal is unhandled: " + tm.getHostingGoal()); - } - } - } - - TabletGoalState getGoalState(TabletMetadata tm) { - KeyExtent extent = tm.getExtent(); - // Shutting down? - TabletGoalState state = getSystemGoalState(tm); - - if (state == TabletGoalState.HOSTED) { - if (!upgradeCoordinator.getStatus().isParentLevelUpgraded(extent)) { - // The place where this tablet stores its metadata was not upgraded, so do not assign this - // tablet yet. - return TabletGoalState.UNASSIGNED; - } - - if (tm.getOperationId() != null) { - return TabletGoalState.UNASSIGNED; - } - - if (tm.hasCurrent() && serversToShutdown.contains(tm.getLocation().getServerInstance())) { - return TabletGoalState.SUSPENDED; - } - - // taking table offline? - state = getTableGoalState(tm); - if (state == TabletGoalState.HOSTED) { - // Maybe this tablet needs to be migrated - TServerInstance dest = migrations.get(extent); - if (dest != null && tm.hasCurrent() && !dest.equals(tm.getLocation().getServerInstance())) { - return TabletGoalState.UNASSIGNED; - } - } - } - return state; + public UpgradeCoordinator.UpgradeStatus getUpgradeStatus() { + return upgradeCoordinator.getStatus(); } private class MigrationCleanupThread implements Runnable { @@ -821,12 +717,11 @@ public void run() { } private long updateStatus() { - Set currentServers = tserverSet.getCurrentServers(); + var tseversSnapshot = tserverSet.getSnapshot(); TreeMap temp = new TreeMap<>(); - tserverStatus = gatherTableInformation(currentServers, temp); + tserverStatus = gatherTableInformation(tseversSnapshot.getTservers(), temp); tserverStatusForBalancer = Collections.unmodifiableSortedMap(temp); - tServerGroupingForBalancer = - Collections.unmodifiableMap(tserverSet.getCurrentServersGroups()); + tServerGroupingForBalancer = tseversSnapshot.getTserverGroups(); checkForHeldServer(tserverStatus); if (!badServers.isEmpty()) { @@ -840,7 +735,7 @@ private long updateStatus() { log.debug("not balancing while shutting down servers {}", serversToShutdown); } else { for (TabletGroupWatcher tgw : watchers) { - if (!tgw.isSameTserversAsLastScan(currentServers)) { + if (!tgw.isSameTserversAsLastScan(tseversSnapshot.getTservers())) { log.debug("not balancing just yet, as collection of live tservers is in flux"); return DEFAULT_WAIT_FOR_WATCHER; } @@ -880,7 +775,7 @@ private void checkForHeldServer(SortedMap ts private long balanceTablets() { BalanceParamsImpl params = BalanceParamsImpl.fromThrift(tserverStatusForBalancer, - tServerGroupingForBalancer, tserverStatus, migrationsSnapshot()); + tServerGroupingForBalancer, tserverStatus, migrationsSnapshot().keySet()); long wait = tabletBalancer.balance(params); for (TabletMigration m : checkMigrationSanity(tserverStatusForBalancer.keySet(), @@ -1582,7 +1477,6 @@ public void initialize() {} @Override public void sessionExpired() {} - @Override public Set onlineTables() { Set result = new HashSet<>(); if (getManagerState() != ManagerState.NORMAL) { @@ -1606,14 +1500,12 @@ public Set onlineTables() { return result; } - @Override public Set onlineTabletServers() { - return tserverSet.getCurrentServers(); + return tserverSet.getSnapshot().getTservers(); } - @Override - public Map> tServerResourceGroups() { - return tserverSet.getCurrentServersGroups(); + public LiveTServersSnapshot tserversSnapshot() { + return tserverSet.getSnapshot(); } // recovers state from the persistent transaction to shutdown a server @@ -1697,19 +1589,15 @@ public boolean delegationTokensAvailable() { return delegationTokensAvailable; } - @Override - public Set migrationsSnapshot() { - Set migrationKeys; + public Map migrationsSnapshot() { synchronized (migrations) { - migrationKeys = new HashSet<>(migrations.keySet()); + return Map.copyOf(migrations); } - return Collections.unmodifiableSet(migrationKeys); } - @Override public Set shutdownServers() { synchronized (serversToShutdown) { - return new HashSet<>(serversToShutdown); + return Set.copyOf(serversToShutdown); } } @@ -1774,10 +1662,4 @@ public TabletStateStore getTabletStateStore(DataLevel level) { throw new IllegalStateException("Unhandled DataLevel value: " + level); } } - - @Override - public List> getVolumeReplacements() { - return getContext().getVolumeReplacements(); - } - } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java index bc36460f116..d82a354949d 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -51,7 +52,6 @@ import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.dataImpl.thrift.TKeyExtent; import org.apache.accumulo.core.logging.TabletLogger; -import org.apache.accumulo.core.manager.balancer.TabletServerIdImpl; import org.apache.accumulo.core.manager.state.TabletManagement; import org.apache.accumulo.core.manager.state.TabletManagement.ManagementAction; import org.apache.accumulo.core.manager.state.tables.TableState; @@ -64,7 +64,7 @@ import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.TabletState; -import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; +import org.apache.accumulo.core.metadata.schema.Ample; import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CurrentLocationColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.FutureLocationColumnFamily; @@ -72,16 +72,15 @@ import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType; import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location; import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.spi.balancer.data.TabletServerId; import org.apache.accumulo.core.tabletserver.log.LogEntry; import org.apache.accumulo.core.util.TextUtil; import org.apache.accumulo.core.util.threads.Threads; import org.apache.accumulo.core.util.threads.Threads.AccumuloDaemonThread; -import org.apache.accumulo.manager.Manager.TabletGoalState; import org.apache.accumulo.manager.metrics.ManagerMetrics; import org.apache.accumulo.manager.split.SplitTask; import org.apache.accumulo.manager.state.TableCounts; import org.apache.accumulo.manager.state.TableStats; +import org.apache.accumulo.manager.upgrade.UpgradeCoordinator; import org.apache.accumulo.server.ServiceEnvironmentImpl; import org.apache.accumulo.server.compaction.CompactionJobGenerator; import org.apache.accumulo.server.conf.TableConfiguration; @@ -92,7 +91,9 @@ import org.apache.accumulo.server.manager.state.Assignment; import org.apache.accumulo.server.manager.state.ClosableIterator; import org.apache.accumulo.server.manager.state.DistributedStoreException; +import org.apache.accumulo.server.manager.state.TabletGoalState; import org.apache.accumulo.server.manager.state.TabletManagementIterator; +import org.apache.accumulo.server.manager.state.TabletManagementParameters; import org.apache.accumulo.server.manager.state.TabletStateStore; import org.apache.accumulo.server.manager.state.UnassignedTablet; import org.apache.accumulo.server.util.MetadataTableUtil; @@ -131,8 +132,8 @@ public Text getEncodedEndRow() { private SortedSet lastScanServers = Collections.emptySortedSet(); private final EventHandler eventHandler; private final ManagerMetrics metrics; - private WalStateManager walStateManager; + private volatile Set filteredServersToShutdown = Set.of(); TabletGroupWatcher(Manager manager, TabletStateStore store, TabletGroupWatcher dependentWatcher, ManagerMetrics metrics) { @@ -179,12 +180,31 @@ private static class TabletLists { private final SortedMap destinations; private final Map> currentTServerGrouping; - public TabletLists(Manager m, SortedMap curTServers, - Map> grouping) { + public TabletLists(SortedMap curTServers, + Map> grouping, Set serversToShutdown) { + var destinationsMod = new TreeMap<>(curTServers); - destinationsMod.keySet().removeAll(m.serversToShutdown); + if (!serversToShutdown.isEmpty()) { + // Remove servers that are in the process of shutting down from the lists of tablet + // servers. + destinationsMod.keySet().removeAll(serversToShutdown); + HashMap> groupingCopy = new HashMap<>(); + grouping.forEach((group, groupsServers) -> { + if (Collections.disjoint(groupsServers, serversToShutdown)) { + groupingCopy.put(group, groupsServers); + } else { + var serversCopy = new HashSet<>(groupsServers); + serversCopy.removeAll(serversToShutdown); + groupingCopy.put(group, Collections.unmodifiableSet(serversCopy)); + } + }); + + this.currentTServerGrouping = Collections.unmodifiableMap(groupingCopy); + } else { + this.currentTServerGrouping = grouping; + } + this.destinations = Collections.unmodifiableSortedMap(destinationsMod); - this.currentTServerGrouping = grouping; } public void reset() { @@ -226,15 +246,17 @@ public void run() { continue; } - var currentTservers = getCurrentTservers(); + TabletManagementParameters tabletMgmtParams = createTabletManagementParameters(false); + + var currentTservers = getCurrentTservers(tabletMgmtParams.getOnlineTsevers()); if (currentTservers.isEmpty()) { setNeedsFullScan(); continue; } - try (var iter = store.iterator(ranges)) { + try (var iter = store.iterator(ranges, tabletMgmtParams)) { long t1 = System.currentTimeMillis(); - manageTablets(iter, currentTservers, false); + manageTablets(iter, tabletMgmtParams, currentTservers, false); long t2 = System.currentTimeMillis(); Manager.log.debug(String.format("[%s]: partial scan time %.2f seconds for %,d ranges", store.name(), (t2 - t1) / 1000., ranges.size())); @@ -298,19 +320,50 @@ synchronized void waitForFullScan(long millis) { } } + private TabletManagementParameters createTabletManagementParameters(boolean firstTime) { + + HashMap parentLevelUpgrade = new HashMap<>(); + UpgradeCoordinator.UpgradeStatus upgradeStatus = manager.getUpgradeStatus(); + for (var level : Ample.DataLevel.values()) { + parentLevelUpgrade.put(level, upgradeStatus.isParentLevelUpgraded(level)); + } + + Set shutdownServers; + if (store.getLevel() == Ample.DataLevel.USER) { + shutdownServers = manager.shutdownServers(); + } else { + // Use the servers to shutdown filtered by the dependent watcher. These are servers to + // shutdown that the dependent watcher has determined it has no tablets hosted on or assigned + // to. + shutdownServers = dependentWatcher.getFilteredServersToShutdown(); + } + + var tServersSnapshot = manager.tserversSnapshot(); + + return new TabletManagementParameters(manager.getManagerState(), parentLevelUpgrade, + manager.onlineTables(), tServersSnapshot, shutdownServers, manager.migrationsSnapshot(), + store.getLevel(), manager.getCompactionHints(), canSuspendTablets(), + firstTime ? manager.getContext().getVolumeReplacements() : List.of()); + } + + private Set getFilteredServersToShutdown() { + return filteredServersToShutdown; + } + private static class TableMgmtStats { int[] counts = new int[TabletState.values().length]; private int totalUnloaded; } private TableMgmtStats manageTablets(Iterator iter, + TabletManagementParameters tableMgmtParams, SortedMap currentTServers, boolean isFullScan) throws BadLocationStateException, TException, DistributedStoreException, WalMarkerException, IOException { TableMgmtStats tableMgmtStats = new TableMgmtStats(); final boolean shuttingDownAllTabletServers = - manager.serversToShutdown.equals(currentTServers.keySet()); + tableMgmtParams.getServersToShutdown().equals(currentTServers.keySet()); if (shuttingDownAllTabletServers && !isFullScan) { // If we are shutting down all of the TabletServers, then don't process any events // from the EventCoordinator. @@ -320,19 +373,14 @@ private TableMgmtStats manageTablets(Iterator iter, int unloaded = 0; - final Map> currentTServerGrouping = - manager.tserverSet.getCurrentServersGroups(); - - TabletLists tLists = new TabletLists(manager, currentTServers, currentTServerGrouping); + TabletLists tLists = new TabletLists(currentTServers, tableMgmtParams.getGroupedTServers(), + tableMgmtParams.getServersToShutdown()); CompactionJobGenerator compactionGenerator = new CompactionJobGenerator( - new ServiceEnvironmentImpl(manager.getContext()), manager.getCompactionHints()); + new ServiceEnvironmentImpl(manager.getContext()), tableMgmtParams.getCompactionHints()); - final Map resourceGroups = new HashMap<>(); - manager.tServerResourceGroups().forEach((group, tservers) -> { - tservers.stream().map(TabletServerIdImpl::new) - .forEach(tabletServerId -> resourceGroups.put(tabletServerId, group)); - }); + Set filteredServersToShutdown = + new HashSet<>(tableMgmtParams.getServersToShutdown()); while (iter.hasNext()) { final TabletManagement mti = iter.next(); @@ -353,6 +401,7 @@ private TableMgmtStats manageTablets(Iterator iter, continue; } + final Set actions = mti.getActions(); if (tabletMeta.isFutureAndCurrentLocationSet()) { throw new BadLocationStateException(tabletMeta.getExtent() + " is both assigned and hosted, which should never happen: " + this, @@ -375,11 +424,13 @@ private TableMgmtStats manageTablets(Iterator iter, final TableConfiguration tableConf = manager.getContext().getTableConfiguration(tableId); - TabletGoalState goal = manager.getGoalState(tabletMeta); - TabletState state = TabletState.compute(tabletMeta, currentTServers.keySet(), - manager.tabletBalancer, resourceGroups); + final TabletState state = TabletState.compute(tabletMeta, currentTServers.keySet()); + // This is final because nothing in this method should change the goal. All computation of the + // goal should be done in TabletGoalState.compute() so that all parts of the Accumulo code + // will compute a consistent goal. + TabletGoalState goal = + TabletGoalState.compute(tabletMeta, state, manager.tabletBalancer, tableMgmtParams); - final Set actions = mti.getActions(); final Location location = tabletMeta.getLocation(); if (actions.contains(ManagementAction.NEEDS_VOLUME_REPLACEMENT)) { @@ -389,7 +440,7 @@ private TableMgmtStats manageTablets(Iterator iter, final List filesToRemove = new ArrayList<>(); final SortedMap filesToAdd = new TreeMap<>(); - VolumeUtil.volumeReplacementEvaluation(manager.getVolumeReplacements(), tabletMeta, + VolumeUtil.volumeReplacementEvaluation(tableMgmtParams.getVolumeReplacements(), tabletMeta, logsToRemove, logsToAdd, filesToRemove, filesToAdd); if (logsToRemove.size() + filesToRemove.size() > 0) { @@ -450,61 +501,14 @@ private TableMgmtStats manageTablets(Iterator iter, stats.update(tableId, state); } - // Always follow through with assignments - if (state == TabletState.ASSIGNED) { - goal = TabletGoalState.HOSTED; - } else if (state == TabletState.NEEDS_REASSIGNMENT) { - goal = TabletGoalState.UNASSIGNED; - } - - if (tm.getOperationId() != null) { - goal = TabletGoalState.UNASSIGNED; - } - if (Manager.log.isTraceEnabled()) { Manager.log.trace( "[{}] Shutting down all Tservers: {}, dependentCount: {} Extent: {}, state: {}, goal: {} actions:{}", - store.name(), manager.serversToShutdown.equals(currentTServers.keySet()), + store.name(), tableMgmtParams.getServersToShutdown().equals(currentTServers.keySet()), dependentWatcher == null ? "null" : dependentWatcher.assignedOrHosted(), tm.getExtent(), state, goal, actions); } - // if we are shutting down all the tabletservers, we have to do it in order - if (shuttingDownAllTabletServers - && (goal == TabletGoalState.SUSPENDED && state == TabletState.HOSTED)) { - if (dependentWatcher != null) { - // If the dependentWatcher is for the user tables, check to see - // that user tables exist. - DataLevel dependentLevel = dependentWatcher.store.getLevel(); - boolean userTablesExist = true; - switch (dependentLevel) { - case USER: - Set onlineTables = manager.onlineTables(); - onlineTables.remove(RootTable.ID); - onlineTables.remove(MetadataTable.ID); - userTablesExist = !onlineTables.isEmpty(); - break; - case METADATA: - case ROOT: - default: - break; - } - // If the stats object in the dependentWatcher is empty, then it - // currently does not have data about what is hosted or not. In - // that case host these tablets until the dependent watcher can - // gather some data. - final Map stats = dependentWatcher.getStats(); - if (dependentLevel == DataLevel.USER) { - if (userTablesExist - && (stats == null || stats.isEmpty() || assignedOrHosted(stats) > 0)) { - goal = TabletGoalState.HOSTED; - } - } else if (stats == null || stats.isEmpty() || assignedOrHosted(stats) > 0) { - goal = TabletGoalState.HOSTED; - } - } - } - if (actions.contains(ManagementAction.NEEDS_SPLITTING)) { LOG.debug("{} may need splitting.", tm.getExtent()); if (manager.getSplitter().isSplittable(tm)) { @@ -534,6 +538,11 @@ private TableMgmtStats manageTablets(Iterator iter, // metadata scan could remove any tablets that were not updated during the scan. if (actions.contains(ManagementAction.NEEDS_LOCATION_UPDATE)) { + + if (tm.getLocation() != null) { + filteredServersToShutdown.remove(tm.getLocation().getServerInstance()); + } + if (goal == TabletGoalState.HOSTED) { if ((state != TabletState.HOSTED && !tm.getLogs().isEmpty()) && manager.recoveryManager.recoverLogs(tm.getExtent(), tm.getLogs())) { @@ -576,7 +585,6 @@ private TableMgmtStats manageTablets(Iterator iter, case ASSIGNED_TO_DEAD_SERVER: unassignDeadTablet(tLists, tm); break; - case NEEDS_REASSIGNMENT: case HOSTED: TServerConnection client = manager.tserverSet.getConnection(location.getServerInstance()); @@ -600,13 +608,19 @@ private TableMgmtStats manageTablets(Iterator iter, } flushChanges(tLists); + + if (isFullScan) { + this.filteredServersToShutdown = Set.copyOf(filteredServersToShutdown); + } + return tableMgmtStats; } - private SortedMap getCurrentTservers() { + private SortedMap + getCurrentTservers(Set onlineTservers) { // Get the current status for the current list of tservers final SortedMap currentTServers = new TreeMap<>(); - for (TServerInstance entry : manager.tserverSet.getCurrentServers()) { + for (TServerInstance entry : onlineTservers) { currentTServers.put(entry, manager.tserverStatus.get(entry)); } return currentTServers; @@ -615,6 +629,7 @@ private SortedMap getCurrentTservers() { @Override public void run() { int[] oldCounts = new int[TabletState.values().length]; + boolean firstCall = true; while (manager.stillManager()) { // slow things down a little, otherwise we spam the logs when there are many wake-up events @@ -625,7 +640,9 @@ public void run() { final long waitTimeBetweenScans = manager.getConfiguration() .getTimeInMillis(Property.MANAGER_TABLET_GROUP_WATCHER_INTERVAL); - var currentTServers = getCurrentTservers(); + TabletManagementParameters tableMgmtParams = createTabletManagementParameters(firstCall); + firstCall = false; + var currentTServers = getCurrentTservers(tableMgmtParams.getOnlineTsevers()); ClosableIterator iter = null; try { @@ -639,14 +656,14 @@ public void run() { stats.begin(); - ManagerState managerState = manager.getManagerState(); + ManagerState managerState = tableMgmtParams.getManagerState(); // Clear the need for a full scan before starting a full scan inorder to detect events that // happen during the full scan. eventHandler.clearNeedsFullScan(); - iter = store.iterator(); - var tabletMgmtStats = manageTablets(iter, currentTServers, true); + iter = store.iterator(tableMgmtParams); + var tabletMgmtStats = manageTablets(iter, tableMgmtParams, currentTServers, true); // provide stats after flushing changes to avoid race conditions w/ delete table stats.end(managerState); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/CleanUp.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/CleanUp.java index c0d60c82346..0b0512b0722 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/CleanUp.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/delete/CleanUp.java @@ -18,6 +18,10 @@ */ package org.apache.accumulo.manager.tableOps.delete; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SUSPEND; + import java.io.IOException; import java.net.UnknownHostException; import java.util.Arrays; @@ -27,17 +31,14 @@ import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.IteratorSetting; -import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.NamespaceId; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.fate.Repo; import org.apache.accumulo.core.iterators.user.GrepIterator; -import org.apache.accumulo.core.manager.state.TabletManagement; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.TabletState; @@ -49,7 +50,6 @@ import org.apache.accumulo.manager.tableOps.ManagerRepo; import org.apache.accumulo.manager.tableOps.Utils; import org.apache.accumulo.server.fs.VolumeManager; -import org.apache.accumulo.server.manager.state.TabletManagementIterator; import org.apache.accumulo.server.problems.ProblemReports; import org.apache.accumulo.server.util.MetadataTableUtil; import org.apache.hadoop.fs.Path; @@ -92,23 +92,20 @@ public long isReady(long tid, Manager manager) throws Exception { } boolean done = true; - Range tableRange = new KeyExtent(tableId, null, null).toMetaRange(); - Scanner scanner = manager.getContext().createScanner(MetadataTable.NAME, Authorizations.EMPTY); - TabletManagementIterator.configureScanner(scanner, manager); - scanner.setRange(tableRange); - - for (Entry entry : scanner) { - final TabletManagement mti = TabletManagementIterator.decode(entry); - final TabletMetadata tm = mti.getTabletMetadata(); + + try (var tablets = manager.getContext().getAmple().readTablets().forTable(tableId) + .fetch(LOCATION, PREV_ROW, SUSPEND).checkConsistency().build()) { Set liveTServers = manager.onlineTabletServers(); - TabletState state = TabletState.compute(tm, liveTServers); - if (!state.equals(TabletState.UNASSIGNED)) { - // This code will even wait on tablets that are assigned to dead tablets servers. This is - // intentional because the manager may make metadata writes for these tablets. See #587 - log.debug("Still waiting for table({}) to be deleted; Target tablet state: UNASSIGNED, " - + "Current tablet state: {}, locationState: {}", tableId, state, tm); - done = false; - break; + for (TabletMetadata tm : tablets) { + TabletState state = TabletState.compute(tm, liveTServers); + if (!state.equals(TabletState.UNASSIGNED)) { + // This code will even wait on tablets that are assigned to dead tablets servers. This is + // intentional because the manager may make metadata writes for these tablets. See #587 + log.debug("Still waiting for table({}) to be deleted; Target tablet state: UNASSIGNED, " + + "Current tablet state: {}, locationState: {}", tableId, state, tm); + done = false; + break; + } } } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java index b67417f3216..0cc31fcaa90 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/DeleteRows.java @@ -20,6 +20,7 @@ import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOGS; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.OPID; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW; import static org.apache.accumulo.manager.tableOps.merge.MergeTablets.validateTablet; @@ -90,7 +91,7 @@ private Optional deleteTabletFiles(Manager manager, long tid) { try ( var tabletsMetadata = manager.getContext().getAmple().readTablets() .forTable(range.tableId()).overlapping(range.prevEndRow(), range.endRow()) - .fetch(OPID, LOCATION, FILES, PREV_ROW).checkConsistency().build(); + .fetch(OPID, LOCATION, FILES, PREV_ROW, LOGS).checkConsistency().build(); var tabletsMutator = manager.getContext().getAmple().conditionallyMutateTablets()) { KeyExtent firstCompleteContained = null; @@ -98,7 +99,6 @@ private Optional deleteTabletFiles(Manager manager, long tid) { for (var tabletMetadata : tabletsMetadata) { validateTablet(tabletMetadata, fateStr, opid, data.tableId); - var tabletMutator = tabletsMutator.mutateTablet(tabletMetadata.getExtent()) .requireOperation(opid).requireAbsentLocation(); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java index 8d5dbc8d7c7..ddb1616f806 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/MergeTablets.java @@ -23,6 +23,7 @@ import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.HOSTING_GOAL; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOGS; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.OPID; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.TIME; @@ -95,7 +96,7 @@ private void mergeMetadataRecords(Manager manager, long tid) throws AccumuloExce try (var tabletsMetadata = manager.getContext().getAmple().readTablets() .forTable(range.tableId()).overlapping(range.prevEndRow(), range.endRow()) - .fetch(OPID, LOCATION, HOSTING_GOAL, FILES, TIME, DIR, ECOMP, PREV_ROW).build()) { + .fetch(OPID, LOCATION, HOSTING_GOAL, FILES, TIME, DIR, ECOMP, PREV_ROW, LOGS).build()) { int tabletsSeen = 0; @@ -233,6 +234,9 @@ static void validateTablet(TabletMetadata tabletMeta, String fateStr, TabletOper Preconditions.checkState(expectedTableId.equals(tabletMeta.getTableId()), "%s tablet %s has unexpected table id %s expected %s", fateStr, tabletMeta.getExtent(), tabletMeta.getTableId(), expectedTableId); + Preconditions.checkState(tabletMeta.getLogs().isEmpty(), + "%s merging tablet %s has unexpected walogs %s", fateStr, tabletMeta.getExtent(), + tabletMeta.getLogs().size()); } /** diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/ReserveTablets.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/ReserveTablets.java index 04a5511b689..87f319f6337 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/ReserveTablets.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/merge/ReserveTablets.java @@ -19,6 +19,7 @@ package org.apache.accumulo.manager.tableOps.merge; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION; +import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOGS; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.OPID; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW; @@ -54,7 +55,7 @@ public long isReady(long tid, Manager env) throws Exception { try ( var tablets = env.getContext().getAmple().readTablets().forTable(data.tableId) - .overlapping(range.prevEndRow(), range.endRow()).fetch(PREV_ROW, LOCATION, OPID) + .overlapping(range.prevEndRow(), range.endRow()).fetch(PREV_ROW, LOCATION, LOGS, OPID) .checkConsistency().build(); var tabletsMutator = env.getContext().getAmple().conditionallyMutateTablets();) { @@ -62,6 +63,7 @@ public long isReady(long tid, Manager env) throws Exception { int otherOps = 0; int opsSet = 0; int locations = 0; + int wals = 0; for (var tabletMeta : tablets) { @@ -77,6 +79,8 @@ public long isReady(long tid, Manager env) throws Exception { locations++; } + wals += tabletMeta.getLogs().size(); + count++; } @@ -84,8 +88,8 @@ public long isReady(long tid, Manager env) throws Exception { .filter(conditionalResult -> conditionalResult.getStatus() == Status.ACCEPTED).count(); log.debug( - "{} reserve tablets op:{} count:{} other opids:{} opids set:{} locations:{} accepted:{}", - FateTxId.formatTid(tid), data.op, count, otherOps, opsSet, locations, opsAccepted); + "{} reserve tablets op:{} count:{} other opids:{} opids set:{} locations:{} accepted:{} wals:{}", + FateTxId.formatTid(tid), data.op, count, otherOps, opsSet, locations, opsAccepted, wals); // while there are table lock a tablet can be concurrently deleted, so should always see // tablets @@ -98,7 +102,7 @@ public long isReady(long tid, Manager env) throws Exception { FateTxId.formatTid(tid)); } - if (locations > 0 || otherOps > 0) { + if (locations > 0 || otherOps > 0 || wals > 0) { // need to wait on these tablets return Math.max(1000, count); } diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java index 32ce0c29341..dedda83872b 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java @@ -32,9 +32,9 @@ import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.client.AccumuloException; -import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.fate.ReadOnlyTStore; import org.apache.accumulo.core.fate.ZooStore; +import org.apache.accumulo.core.metadata.schema.Ample; import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.core.volume.Volume; import org.apache.accumulo.manager.EventCoordinator; @@ -59,7 +59,7 @@ public enum UpgradeStatus { */ INITIAL { @Override - public boolean isParentLevelUpgraded(KeyExtent extent) { + public boolean isParentLevelUpgraded(Ample.DataLevel level) { return false; } }, @@ -68,8 +68,8 @@ public boolean isParentLevelUpgraded(KeyExtent extent) { */ UPGRADED_ZOOKEEPER { @Override - public boolean isParentLevelUpgraded(KeyExtent extent) { - return extent.isRootTablet(); + public boolean isParentLevelUpgraded(Ample.DataLevel level) { + return level == Ample.DataLevel.ROOT; } }, /** @@ -77,8 +77,8 @@ public boolean isParentLevelUpgraded(KeyExtent extent) { */ UPGRADED_ROOT { @Override - public boolean isParentLevelUpgraded(KeyExtent extent) { - return extent.isMeta(); + public boolean isParentLevelUpgraded(Ample.DataLevel level) { + return level == Ample.DataLevel.METADATA || level == Ample.DataLevel.ROOT; } }, /** @@ -86,7 +86,7 @@ public boolean isParentLevelUpgraded(KeyExtent extent) { */ COMPLETE { @Override - public boolean isParentLevelUpgraded(KeyExtent extent) { + public boolean isParentLevelUpgraded(Ample.DataLevel level) { return true; } }, @@ -95,7 +95,7 @@ public boolean isParentLevelUpgraded(KeyExtent extent) { */ FAILED { @Override - public boolean isParentLevelUpgraded(KeyExtent extent) { + public boolean isParentLevelUpgraded(Ample.DataLevel level) { return false; } }; @@ -104,7 +104,7 @@ public boolean isParentLevelUpgraded(KeyExtent extent) { * Determines if the place where this extent stores its metadata was upgraded for a given * upgrade status. */ - public abstract boolean isParentLevelUpgraded(KeyExtent extent); + public abstract boolean isParentLevelUpgraded(Ample.DataLevel level); } private static final Logger log = LoggerFactory.getLogger(UpgradeCoordinator.class); diff --git a/server/monitor/pom.xml b/server/monitor/pom.xml index c52fe1fb624..49534f0d1e4 100644 --- a/server/monitor/pom.xml +++ b/server/monitor/pom.xml @@ -80,10 +80,6 @@ org.apache.accumulo accumulo-start - - org.apache.hadoop - hadoop-client-api - org.apache.logging.log4j log4j-api diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tables/TablesResource.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tables/TablesResource.java index 39a6b51cf7b..ccc5eb7ca55 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tables/TablesResource.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/tables/TablesResource.java @@ -36,23 +36,20 @@ import jakarta.ws.rs.Produces; import jakarta.ws.rs.core.MediaType; -import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.manager.state.tables.TableState; import org.apache.accumulo.core.manager.thrift.ManagerMonitorInfo; import org.apache.accumulo.core.manager.thrift.TableInfo; import org.apache.accumulo.core.manager.thrift.TabletServerStatus; -import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; -import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; +import org.apache.accumulo.core.metadata.schema.Ample; import org.apache.accumulo.core.metadata.schema.TabletMetadata; +import org.apache.accumulo.core.metadata.schema.TabletsMetadata; import org.apache.accumulo.monitor.Monitor; import org.apache.accumulo.monitor.rest.tservers.TabletServer; import org.apache.accumulo.monitor.rest.tservers.TabletServers; -import org.apache.accumulo.server.manager.state.TabletManagementScanner; import org.apache.accumulo.server.tables.TableManager; import org.apache.accumulo.server.util.TableInfoUtil; -import org.apache.hadoop.io.Text; /** * Generates a tables list from the Monitor as a JSON object @@ -145,25 +142,20 @@ public TabletServers getParticipatingTabletServers(@PathParam("tableId") @NotNul if (RootTable.ID.equals(tableId)) { locs.add(rootTabletLocation); } else { - String systemTableName = - MetadataTable.ID.equals(tableId) ? RootTable.NAME : MetadataTable.NAME; - TabletManagementScanner scanner = new TabletManagementScanner(monitor.getContext(), - new Range(TabletsSection.encodeRow(tableId, new Text()), - TabletsSection.encodeRow(tableId, null)), - systemTableName); - - while (scanner.hasNext()) { - final TabletMetadata tm = scanner.next().getTabletMetadata(); - if (tm.hasCurrent()) { - try { - locs.add(tm.getLocation().getHostPort()); - } catch (Exception ex) { - scanner.close(); - return tabletServers; + var level = Ample.DataLevel.of(tableId); + try (TabletsMetadata tablets = + monitor.getContext().getAmple().readTablets().forLevel(level).build()) { + + for (TabletMetadata tm : tablets) { + if (tm.hasCurrent()) { + try { + locs.add(tm.getLocation().getHostPort()); + } catch (Exception ex) { + return tabletServers; + } } } } - scanner.close(); } List tservers = new ArrayList<>(); diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletClientHandler.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletClientHandler.java index 5a0b53d5995..4b53ac58b99 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletClientHandler.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletClientHandler.java @@ -210,7 +210,6 @@ private void setUpdateTablet(UpdateSession us, KeyExtent keyExtent) { us.currentTablet = null; us.authFailures.put(keyExtent, SecurityErrorCode.TABLE_DOESNT_EXIST); server.updateMetrics.addUnknownTabletErrors(0); - return; } catch (ThriftSecurityException e) { log.error("Denying permission to check user " + us.getUser() + " with user " + e.getUser(), e); @@ -219,7 +218,6 @@ private void setUpdateTablet(UpdateSession us, KeyExtent keyExtent) { us.currentTablet = null; us.authFailures.put(keyExtent, e.getCode()); server.updateMetrics.addPermissionErrors(0); - return; } } @@ -486,7 +484,8 @@ public UpdateErrors closeUpdate(TInfo tinfo, long updateID) throws NoSuchScanIDE us.authFailures.entrySet().stream() .collect(Collectors.toMap(e -> e.getKey().toThrift(), Entry::getValue))); } finally { - // Atomically unreserve and delete the session. If there any write stragglers, they will fail + // Atomically unreserve and delete the session. If there are any write stragglers, they will + // fail // after this point. server.sessionManager.removeSession(updateID, true); } @@ -708,7 +707,7 @@ public TConditionalSession startConditionalUpdate(TInfo tinfo, TCredentials cred String classLoaderContext) throws ThriftSecurityException, TException { TableId tableId = TableId.of(tableIdStr); - Authorizations userauths = null; + Authorizations userauths; NamespaceId namespaceId = getNamespaceId(credentials, tableId); if (!security.canConditionallyUpdate(credentials, tableId, namespaceId)) { throw new ThriftSecurityException(credentials.getPrincipal(), @@ -781,7 +780,7 @@ public List conditionalUpdate(TInfo tinfo, long sessID, } catch (IOException ioe) { throw new TException(ioe); } catch (Exception e) { - log.warn("Exception returned for conditionalUpdate {}", e); + log.warn("Exception returned for conditionalUpdate. tableId: {}, opid: {}", tid, opid, e); throw e; } finally { writeTracker.finishWrite(opid); @@ -1179,7 +1178,7 @@ public List refreshTablets(TInfo tinfo, TCredentials credentials, @Override public List getActiveLogs(TInfo tinfo, TCredentials credentials) { String log = server.logger.getLogFile(); - // Might be null if there no active logger + // Might be null if there is no active logger if (log == null) { return Collections.emptyList(); } diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java index a8a2cc50dcc..7ff5b70e351 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java @@ -70,12 +70,11 @@ public class SessionManager { private final long maxUpdateIdle; private final BlockingQueue deferredCleanupQueue = new ArrayBlockingQueue<>(5000); private final Long expiredSessionMarker = (long) -1; - private final AccumuloConfiguration aconf; private final ServerContext ctx; public SessionManager(ServerContext context) { this.ctx = context; - this.aconf = context.getConfiguration(); + final AccumuloConfiguration aconf = context.getConfiguration(); maxUpdateIdle = aconf.getTimeInMillis(Property.TSERV_UPDATE_SESSION_MAXIDLE); maxIdle = aconf.getTimeInMillis(Property.TSERV_SESSION_MAXIDLE); @@ -230,22 +229,16 @@ public boolean removeIfNotReserved(long sessionId) { return true; } - boolean removed = false; - synchronized (session) { if (session.state == State.RESERVED) { return false; } - session.state = State.REMOVED; - removed = true; } - if (removed) { - sessions.remove(sessionId); - } + sessions.remove(sessionId); - return removed; + return true; } static void cleanup(BlockingQueue deferredCleanupQueue, Session session) { @@ -351,7 +344,7 @@ public Map> getActiveScansPerTable() { Set> copiedIdleSessions = new HashSet<>(); - /** + /* * Add sessions so that get the list returned in the active scans call */ for (Session session : deferredCleanupQueue) { @@ -391,7 +384,7 @@ public List getActiveScans() { final long ct = System.currentTimeMillis(); final Set> copiedIdleSessions = new HashSet<>(); - /** + /* * Add sessions so that get the list returned in the active scans call */ for (Session session : deferredCleanupQueue) { diff --git a/test/src/main/java/org/apache/accumulo/test/LocatorIT.java b/test/src/main/java/org/apache/accumulo/test/LocatorIT.java index fa301cb5ae4..3bad4dbd45d 100644 --- a/test/src/main/java/org/apache/accumulo/test/LocatorIT.java +++ b/test/src/main/java/org/apache/accumulo/test/LocatorIT.java @@ -52,10 +52,8 @@ import org.apache.accumulo.test.functional.ManagerAssignmentIT; import org.apache.accumulo.test.util.Wait; import org.apache.hadoop.io.Text; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -@Disabled // ELASTICITY_TODO public class LocatorIT extends AccumuloClusterHarness { @Override @@ -129,10 +127,8 @@ public void testBasic() throws Exception { ranges.clear(); tableOps.setTabletHostingGoal(tableName, new Range(), TabletHostingGoal.ALWAYS); - Wait.waitFor( - () -> alwaysHostedAndCurrentNotNull.test( - ManagerAssignmentIT.getManagerTabletInfo(client, tableId, null).getTabletMetadata()), - 60000, 250); + Wait.waitFor(() -> alwaysHostedAndCurrentNotNull + .test(ManagerAssignmentIT.getTabletMetadata(client, tableId, null)), 60000, 250); ranges.add(r1); Locations ret = tableOps.locate(tableName, ranges); @@ -147,13 +143,8 @@ public void testBasic() throws Exception { splits.add(new Text("r")); tableOps.addSplits(tableName, splits); - // ELASTICITY_TODO split does not set hosting goal, so this throws exception - assertThrows(AccumuloException.class, () -> tableOps.locate(tableName, ranges)); - tableOps.setTabletHostingGoal(tableName, new Range(), TabletHostingGoal.ALWAYS); - Wait.waitFor( - () -> alwaysHostedAndCurrentNotNull.test( - ManagerAssignmentIT.getManagerTabletInfo(client, tableId, null).getTabletMetadata()), - 60000, 250); + Wait.waitFor(() -> alwaysHostedAndCurrentNotNull + .test(ManagerAssignmentIT.getTabletMetadata(client, tableId, null)), 60000, 250); ret = tableOps.locate(tableName, ranges); assertContains(ret, tservers, Map.of(r1, Set.of(t2), r2, Set.of(t2, t3)), @@ -165,15 +156,9 @@ public void testBasic() throws Exception { tableOps.online(tableName, true); - // ELASTICITY_TODO Split does not set hosting goal - tableOps.setTabletHostingGoal(tableName, new Range(), TabletHostingGoal.ALWAYS); - - // TabletGroupWatcher interval set to 5s - Thread.sleep(7000); - Wait.waitFor( - () -> alwaysHostedAndCurrentNotNull.test(ManagerAssignmentIT - .getManagerTabletInfo(client, tableId, new Text("r")).getTabletMetadata()), + () -> alwaysHostedAndCurrentNotNull + .test(ManagerAssignmentIT.getTabletMetadata(client, tableId, new Text("r"))), 60000, 250); ArrayList ranges2 = new ArrayList<>(); diff --git a/test/src/main/java/org/apache/accumulo/test/ManagerRepairsDualAssignmentIT.java b/test/src/main/java/org/apache/accumulo/test/ManagerRepairsDualAssignmentIT.java index c4094cfe08a..502921e9751 100644 --- a/test/src/main/java/org/apache/accumulo/test/ManagerRepairsDualAssignmentIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ManagerRepairsDualAssignmentIT.java @@ -26,15 +26,14 @@ import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.admin.NewTableConfiguration; import org.apache.accumulo.core.client.admin.TabletHostingGoal; -import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.dataImpl.KeyExtent; -import org.apache.accumulo.core.manager.state.TabletManagement; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; @@ -45,8 +44,6 @@ import org.apache.accumulo.minicluster.ServerType; import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; import org.apache.accumulo.server.ServerContext; -import org.apache.accumulo.server.manager.state.ClosableIterator; -import org.apache.accumulo.server.manager.state.TabletStateStore; import org.apache.accumulo.test.functional.ConfigurableMacBase; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.RawLocalFileSystem; @@ -74,7 +71,6 @@ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) public void test() throws Exception { // make some tablets, spread 'em around try (AccumuloClient c = Accumulo.newClient().from(getClientProperties()).build()) { - ClientContext context = (ClientContext) c; ServerContext serverContext = cluster.getServerContext(); String table = this.getUniqueNames(1)[0]; c.securityOperations().grantTablePermission("root", MetadataTable.NAME, @@ -90,16 +86,19 @@ public void test() throws Exception { // scan the metadata table and get the two table location states Set states = new HashSet<>(); Set oldLocations = new HashSet<>(); - TabletStateStore store = TabletStateStore.getStoreForLevel(DataLevel.USER, serverContext); while (states.size() < 2) { UtilWaitThread.sleep(250); oldLocations.clear(); - for (TabletManagement mti : store) { - if (mti.getTabletMetadata().hasCurrent()) { - states.add(mti.getTabletMetadata().getLocation()); - oldLocations.add(mti.getTabletMetadata()); - } + try ( + var tablets = serverContext.getAmple().readTablets().forLevel(DataLevel.USER).build()) { + tablets.iterator().forEachRemaining(tm -> { + if (tm.hasCurrent()) { + states.add(tm.getLocation()); + oldLocations.add(tm); + } + }); } + } assertEquals(2, states.size()); // Kill a tablet server... we don't care which one... wait for everything to be reassigned @@ -109,16 +108,19 @@ public void test() throws Exception { while (true) { UtilWaitThread.sleep(1000); states.clear(); - boolean allAssigned = true; - for (TabletManagement mti : store) { - if (mti.getTabletMetadata().hasCurrent()) { - states.add(mti.getTabletMetadata().getLocation()); - } else { - allAssigned = false; - } + AtomicBoolean allAssigned = new AtomicBoolean(true); + try ( + var tablets = serverContext.getAmple().readTablets().forLevel(DataLevel.USER).build()) { + tablets.iterator().forEachRemaining(tm -> { + if (tm.hasCurrent()) { + states.add(tm.getLocation()); + } else { + allAssigned.set(false); + } + }); } System.out.println(states + " size " + states.size() + " allAssigned " + allAssigned); - if (states.size() != 2 && allAssigned) { + if (states.size() != 2 && allAssigned.get()) { break; } } @@ -136,20 +138,20 @@ public void test() throws Exception { tabletMutator.putLocation(moved.getLocation()); tabletMutator.mutate(); // wait for the manager to fix the problem - waitForCleanStore(store); + waitForCleanStore(serverContext, DataLevel.USER); // now jam up the metadata table tabletMutator = serverContext.getAmple().mutateTablet(new KeyExtent(MetadataTable.ID, null, null)); tabletMutator.putLocation(moved.getLocation()); tabletMutator.mutate(); - waitForCleanStore(TabletStateStore.getStoreForLevel(DataLevel.METADATA, serverContext)); + waitForCleanStore(serverContext, DataLevel.METADATA); } } - private void waitForCleanStore(TabletStateStore store) { + private void waitForCleanStore(ServerContext serverContext, DataLevel level) { while (true) { - try (ClosableIterator iter = store.iterator()) { - iter.forEachRemaining(t -> {}); + try (var tablets = serverContext.getAmple().readTablets().forLevel(level).build()) { + tablets.iterator().forEachRemaining(t -> {}); } catch (Exception ex) { System.out.println(ex); UtilWaitThread.sleep(250); diff --git a/test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java b/test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java index 1c0b33ce378..a528d967c49 100644 --- a/test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java +++ b/test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java @@ -31,6 +31,8 @@ import org.apache.accumulo.core.client.Accumulo; import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.admin.NewTableConfiguration; +import org.apache.accumulo.core.client.admin.TabletHostingGoal; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.metadata.MetadataTable; @@ -40,11 +42,10 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily; import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.test.functional.ConfigurableMacBase; +import org.apache.accumulo.test.util.Wait; import org.apache.hadoop.io.Text; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -@Disabled // ELASTICITY_TODO public class WaitForBalanceIT extends ConfigurableMacBase { private static final int NUM_SPLITS = 50; @@ -64,7 +65,9 @@ public void test() throws Exception { c.instanceOperations().waitForBalance(); assertTrue(isBalanced(c)); final String tableName = getUniqueNames(1)[0]; - c.tableOperations().create(tableName); + NewTableConfiguration ntc = new NewTableConfiguration(); + ntc.withInitialHostingGoal(TabletHostingGoal.ALWAYS); + c.tableOperations().create(tableName, ntc); c.instanceOperations().waitForBalance(); final SortedSet partitionKeys = new TreeSet<>(); for (int i = 0; i < NUM_SPLITS; i++) { @@ -73,7 +76,7 @@ public void test() throws Exception { c.tableOperations().addSplits(tableName, partitionKeys); assertFalse(isBalanced(c)); c.instanceOperations().waitForBalance(); - assertTrue(isBalanced(c)); + Wait.waitFor(() -> isBalanced(c)); } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/AssignLocationModeIT.java b/test/src/main/java/org/apache/accumulo/test/functional/AssignLocationModeIT.java index e742cbbde43..92805e572a0 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/AssignLocationModeIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/AssignLocationModeIT.java @@ -66,7 +66,7 @@ public void test() throws Exception { TabletMetadata newTablet; do { UtilWaitThread.sleep(250); - newTablet = ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); + newTablet = ManagerAssignmentIT.getTabletMetadata(c, tableId, null); } while (!newTablet.hasCurrent()); // this would be null if the mode was not "assign" assertEquals(newTablet.getLocation().getHostPort(), newTablet.getLast().getHostPort()); @@ -82,24 +82,21 @@ public void test() throws Exception { .get(Property.TSERV_LAST_LOCATION_MODE.getKey())); // last location should not be set yet - TabletMetadata unflushed = - ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); + TabletMetadata unflushed = ManagerAssignmentIT.getTabletMetadata(c, tableId, null); assertEquals(newTablet.getLocation().getHostPort(), unflushed.getLocation().getHostPort()); assertEquals(newTablet.getLocation().getHostPort(), unflushed.getLast().getHostPort()); assertTrue(newTablet.hasCurrent()); // take the tablet offline c.tableOperations().offline(tableName, true); - TabletMetadata offline = - ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); + TabletMetadata offline = ManagerAssignmentIT.getTabletMetadata(c, tableId, null); assertNull(offline.getLocation()); assertFalse(offline.hasCurrent()); assertEquals(newTablet.getLocation().getHostPort(), offline.getLast().getHostPort()); // put it back online, should have the same last location c.tableOperations().online(tableName, true); - TabletMetadata online = - ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); + TabletMetadata online = ManagerAssignmentIT.getTabletMetadata(c, tableId, null); assertTrue(online.hasCurrent()); assertNotNull(online.getLocation()); assertEquals(newTablet.getLast().getHostPort(), online.getLast().getHostPort()); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/CompactLocationModeIT.java b/test/src/main/java/org/apache/accumulo/test/functional/CompactLocationModeIT.java index 471aedf566d..eb672574f07 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/CompactLocationModeIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/CompactLocationModeIT.java @@ -63,7 +63,7 @@ public void test() throws Exception { TabletMetadata newTablet; do { UtilWaitThread.sleep(250); - newTablet = ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); + newTablet = ManagerAssignmentIT.getTabletMetadata(c, tableId, null); } while (!newTablet.hasCurrent()); assertNull(newTablet.getLast()); assertNotNull(newTablet.getLocation()); @@ -79,8 +79,7 @@ public void test() throws Exception { .get(Property.TSERV_LAST_LOCATION_MODE.getKey())); // no last location should be set yet - TabletMetadata unflushed = - ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); + TabletMetadata unflushed = ManagerAssignmentIT.getTabletMetadata(c, tableId, null); assertEquals(newTablet.getLocation().getHostPort(), unflushed.getLocation().getHostPort()); assertNull(unflushed.getLast()); assertTrue(newTablet.hasCurrent()); @@ -88,23 +87,20 @@ public void test() throws Exception { // This should give it a last location if the mode is being used correctly c.tableOperations().flush(tableName, null, null, true); - TabletMetadata flushed = - ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); + TabletMetadata flushed = ManagerAssignmentIT.getTabletMetadata(c, tableId, null); assertEquals(newTablet.getLocation().getHostPort(), flushed.getLocation().getHostPort()); assertEquals(flushed.getLocation().getHostPort(), flushed.getLast().getHostPort()); assertTrue(newTablet.hasCurrent()); // take the tablet offline c.tableOperations().offline(tableName, true); - TabletMetadata offline = - ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); + TabletMetadata offline = ManagerAssignmentIT.getTabletMetadata(c, tableId, null); assertNull(offline.getLocation()); assertEquals(flushed.getLocation().getHostPort(), offline.getLast().getHostPort()); // put it back online, should have the same last location c.tableOperations().online(tableName, true); - TabletMetadata online = - ManagerAssignmentIT.getManagerTabletInfo(c, tableId, null).getTabletMetadata(); + TabletMetadata online = ManagerAssignmentIT.getTabletMetadata(c, tableId, null); assertTrue(online.hasCurrent()); assertNotNull(online.getLocation()); assertEquals(offline.getLast().getHostPort(), online.getLast().getHostPort()); diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java index 0f82311d7a5..739add9599c 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java @@ -63,7 +63,6 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.lock.ServiceLock; -import org.apache.accumulo.core.manager.state.TabletManagement; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.schema.Ample; @@ -79,7 +78,6 @@ import org.apache.accumulo.core.trace.TraceUtil; import org.apache.accumulo.harness.SharedMiniClusterBase; import org.apache.accumulo.minicluster.ServerType; -import org.apache.accumulo.server.manager.state.TabletManagementScanner; import org.apache.accumulo.test.util.Wait; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.BeforeAll; @@ -133,10 +131,8 @@ public void test() throws Exception { // wait for the tablet to exist in the metadata table. The tablet // will not be hosted so the current location will be empty. - Wait.waitFor( - () -> getManagerTabletInfo(c, tableId, null).getTabletMetadata().getExtent() != null, - 10000, 250); - TabletMetadata newTablet = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); + Wait.waitFor(() -> getTabletMetadata(c, tableId, null) != null, 10000, 250); + TabletMetadata newTablet = getTabletMetadata(c, tableId, null); assertNotNull(newTablet.getExtent()); assertFalse(newTablet.hasCurrent()); assertNull(newTablet.getLast()); @@ -152,7 +148,7 @@ public void test() throws Exception { // give it a last location c.tableOperations().flush(tableName, null, null, true); - TabletMetadata flushed = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); + TabletMetadata flushed = getTabletMetadata(c, tableId, null); assertTrue(flushed.hasCurrent()); assertNotNull(flushed.getLocation()); assertEquals(flushed.getLocation().getHostPort(), flushed.getLast().getHostPort()); @@ -161,7 +157,7 @@ public void test() throws Exception { // take the tablet offline c.tableOperations().offline(tableName, true); - TabletMetadata offline = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); + TabletMetadata offline = getTabletMetadata(c, tableId, null); assertFalse(offline.hasCurrent()); assertNull(offline.getLocation()); assertEquals(flushed.getLocation().getHostPort(), offline.getLast().getHostPort()); @@ -169,7 +165,7 @@ public void test() throws Exception { // put it back online c.tableOperations().online(tableName, true); - TabletMetadata online = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); + TabletMetadata online = getTabletMetadata(c, tableId, null); assertTrue(online.hasCurrent()); assertNotNull(online.getLocation()); assertEquals(online.getLocation().getHostPort(), online.getLast().getHostPort()); @@ -181,10 +177,10 @@ public void test() throws Exception { Predicate alwaysHostedOrCurrentNotNull = t -> (t.getHostingGoal() == TabletHostingGoal.ALWAYS && t.hasCurrent()); - Wait.waitFor(() -> alwaysHostedOrCurrentNotNull - .test(getManagerTabletInfo(c, tableId, null).getTabletMetadata()), 60000, 250); + Wait.waitFor(() -> alwaysHostedOrCurrentNotNull.test(getTabletMetadata(c, tableId, null)), + 60000, 250); - final TabletMetadata always = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); + final TabletMetadata always = getTabletMetadata(c, tableId, null); assertTrue(alwaysHostedOrCurrentNotNull.test(always)); assertTrue(always.hasCurrent()); assertEquals(flushed.getLocation().getHostPort(), always.getLast().getHostPort()); @@ -194,10 +190,10 @@ public void test() throws Exception { c.tableOperations().setTabletHostingGoal(tableName, new Range(), TabletHostingGoal.NEVER); Predicate neverHostedOrCurrentNull = t -> (t.getHostingGoal() == TabletHostingGoal.NEVER && !t.hasCurrent()); - Wait.waitFor(() -> neverHostedOrCurrentNull - .test(getManagerTabletInfo(c, tableId, null).getTabletMetadata()), 60000, 250); + Wait.waitFor(() -> neverHostedOrCurrentNull.test(getTabletMetadata(c, tableId, null)), 60000, + 250); - final TabletMetadata never = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); + final TabletMetadata never = getTabletMetadata(c, tableId, null); assertTrue(neverHostedOrCurrentNull.test(never)); assertNull(never.getLocation()); assertEquals(flushed.getLocation().getHostPort(), never.getLast().getHostPort()); @@ -207,10 +203,8 @@ public void test() throws Exception { c.tableOperations().setTabletHostingGoal(tableName, new Range(), TabletHostingGoal.ONDEMAND); Predicate ondemandHosted = t -> t.getHostingGoal() == TabletHostingGoal.ONDEMAND; - Wait.waitFor( - () -> ondemandHosted.test(getManagerTabletInfo(c, tableId, null).getTabletMetadata()), - 60000, 250); - final TabletMetadata ondemand = getManagerTabletInfo(c, tableId, null).getTabletMetadata(); + Wait.waitFor(() -> ondemandHosted.test(getTabletMetadata(c, tableId, null)), 60000, 250); + final TabletMetadata ondemand = getTabletMetadata(c, tableId, null); assertTrue(ondemandHosted.test(ondemand)); assertNull(ondemand.getLocation()); assertEquals(flushed.getLocation().getHostPort(), ondemand.getLast().getHostPort()); @@ -621,11 +615,16 @@ public void testShutdownOnlyTServerWithoutUserTable() throws Exception { } } - public static TabletManagement getManagerTabletInfo(AccumuloClient c, String tableId, - Text endRow) { - try (TabletManagementScanner s = new TabletManagementScanner((ClientContext) c, - new Range(TabletsSection.encodeRow(TableId.of(tableId), endRow)), MetadataTable.NAME)) { - return s.next(); + public static TabletMetadata getTabletMetadata(AccumuloClient c, String tableId, Text endRow) { + var ctx = (ClientContext) c; + try (var tablets = ctx.getAmple().readTablets().forTable(TableId.of(tableId)) + .overlapping(endRow, null).build()) { + var iter = tablets.iterator(); + if (iter.hasNext()) { + return iter.next(); + } } + + return null; } } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/RestartIT.java b/test/src/main/java/org/apache/accumulo/test/functional/RestartIT.java index 3ab2b968593..5e952629945 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/RestartIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/RestartIT.java @@ -51,7 +51,6 @@ import org.apache.hadoop.fs.RawLocalFileSystem; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -232,7 +231,6 @@ public void killedTabletServer2() throws Exception { } @Test - @Disabled // ELASTICITY_TODO public void killedTabletServerDuringShutdown() throws Exception { try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) { String tableName = getUniqueNames(1)[0]; diff --git a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java index 26242d7912b..e5911e5773f 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/TabletManagementIteratorIT.java @@ -24,7 +24,6 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -64,6 +63,7 @@ import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.TServerInstance; +import org.apache.accumulo.core.metadata.schema.Ample; import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CurrentLocationColumnFamily; @@ -75,8 +75,9 @@ import org.apache.accumulo.core.util.Pair; import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.harness.AccumuloClusterHarness; -import org.apache.accumulo.server.manager.state.CurrentState; +import org.apache.accumulo.server.manager.LiveTServerSet; import org.apache.accumulo.server.manager.state.TabletManagementIterator; +import org.apache.accumulo.server.manager.state.TabletManagementParameters; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.junit.jupiter.api.Test; @@ -130,15 +131,15 @@ public void test() throws AccumuloException, AccumuloSecurityException, TableExi // examine a clone of the metadata table, so we can manipulate it copyTable(client, MetadataTable.NAME, metaCopy1); - State state = new State(client); - int tabletsInFlux = findTabletsNeedingAttention(client, metaCopy1, state); + TabletManagementParameters tabletMgmtParams = createParameters(client); + int tabletsInFlux = findTabletsNeedingAttention(client, metaCopy1, tabletMgmtParams); while (tabletsInFlux > 0) { log.debug("Waiting for {} tablets for {}", tabletsInFlux, metaCopy1); UtilWaitThread.sleep(500); copyTable(client, MetadataTable.NAME, metaCopy1); - tabletsInFlux = findTabletsNeedingAttention(client, metaCopy1, state); + tabletsInFlux = findTabletsNeedingAttention(client, metaCopy1, tabletMgmtParams); } - assertEquals(0, findTabletsNeedingAttention(client, metaCopy1, state), + assertEquals(0, findTabletsNeedingAttention(client, metaCopy1, tabletMgmtParams), "No tables should need attention"); // The metadata table stabilized and metaCopy1 contains a copy suitable for testing. Before @@ -151,46 +152,41 @@ public void test() throws AccumuloException, AccumuloSecurityException, TableExi setTabletHostingGoal(client, metaCopy1, t1, TabletHostingGoal.ALWAYS.name()); // t3 is hosted, setting to never will generate a change to unhost tablets setTabletHostingGoal(client, metaCopy1, t3, TabletHostingGoal.NEVER.name()); - state = new State(client); - assertEquals(4, findTabletsNeedingAttention(client, metaCopy1, state), + tabletMgmtParams = createParameters(client); + assertEquals(4, findTabletsNeedingAttention(client, metaCopy1, tabletMgmtParams), "Should have four tablets with hosting goal changes"); // test the assigned case (no location) removeLocation(client, metaCopy1, t3); - assertEquals(2, findTabletsNeedingAttention(client, metaCopy1, state), + assertEquals(2, findTabletsNeedingAttention(client, metaCopy1, tabletMgmtParams), "Should have two tablets without a loc"); // Test setting the operation id on one of the tablets in table t1. Table t1 has two tablets // w/o a location. Only one should need attention because of the operation id. setOperationId(client, metaCopy1, t1); - assertEquals(1, findTabletsNeedingAttention(client, metaCopy1, state), + assertEquals(1, findTabletsNeedingAttention(client, metaCopy1, tabletMgmtParams), "Should have tablets needing attention because of operation id"); // test the cases where the assignment is to a dead tserver reassignLocation(client, metaCopy2, t3); - assertEquals(1, findTabletsNeedingAttention(client, metaCopy2, state), + assertEquals(1, findTabletsNeedingAttention(client, metaCopy2, tabletMgmtParams), "Only 1 of 2 tablets in table t1 should be returned"); // test the bad tablet location state case (inconsistent metadata) - state = new State(client); + tabletMgmtParams = createParameters(client); addDuplicateLocation(client, metaCopy3, t3); - assertEquals(1, findTabletsNeedingAttention(client, metaCopy3, state), + assertEquals(1, findTabletsNeedingAttention(client, metaCopy3, tabletMgmtParams), "Should have 1 tablet that needs a metadata repair"); // test the volume replacements case. Need to insert some files into // the metadata for t4, then run the TabletManagementIterator with // volume replacements addFiles(client, metaCopy4, t4); - state = new State(client) { - @Override - public List> getVolumeReplacements() { - List> replacements = new ArrayList<>(); - replacements.add(new Pair(new Path("file:/vol1/accumulo/inst_id"), - new Path("file:/vol2/accumulo/inst_id"))); - return replacements; - } - }; - assertEquals(1, findTabletsNeedingAttention(client, metaCopy4, state), + List> replacements = new ArrayList<>(); + replacements.add(new Pair(new Path("file:/vol1/accumulo/inst_id"), + new Path("file:/vol2/accumulo/inst_id"))); + tabletMgmtParams = createParameters(client, replacements); + assertEquals(1, findTabletsNeedingAttention(client, metaCopy4, tabletMgmtParams), "Should have one tablet that needs a volume replacement"); // clean up @@ -303,13 +299,12 @@ private void removeLocation(AccumuloClient client, String table, String tableNam deleter.close(); } - private int findTabletsNeedingAttention(AccumuloClient client, String table, State state) - throws TableNotFoundException, IOException { + private int findTabletsNeedingAttention(AccumuloClient client, String table, + TabletManagementParameters tabletMgmtParams) throws TableNotFoundException, IOException { int results = 0; List resultList = new ArrayList<>(); try (Scanner scanner = client.createScanner(table, Authorizations.EMPTY)) { - TabletManagementIterator.configureScanner(scanner, state); - log.debug("Current state = {}", state); + TabletManagementIterator.configureScanner(scanner, tabletMgmtParams); scanner.updateScanIteratorOption("tabletChange", "debug", "1"); for (Entry e : scanner) { if (e != null) { @@ -397,76 +392,34 @@ private void dropTables(AccumuloClient client, String... tables) } } - private static class State implements CurrentState { - - final ClientContext context; - - State(AccumuloClient client) { - this.context = (ClientContext) client; - } + private static TabletManagementParameters createParameters(AccumuloClient client) { + return createParameters(client, List.of()); + } - private Set tservers; - private Set onlineTables; - - @Override - public Set onlineTabletServers() { - HashSet tservers = new HashSet<>(); - for (String tserver : context.instanceOperations().getTabletServers()) { - try { - var zPath = ServiceLock.path(ZooUtil.getRoot(context.instanceOperations().getInstanceId()) - + Constants.ZTSERVERS + "/" + tserver); - long sessionId = ServiceLock.getSessionId(context.getZooCache(), zPath); - tservers.add(new TServerInstance(tserver, sessionId)); - } catch (Exception e) { - throw new RuntimeException(e); - } + private static TabletManagementParameters createParameters(AccumuloClient client, + List> replacements) { + var context = (ClientContext) client; + Set onlineTables = Sets.filter(context.getTableIdToNameMap().keySet(), + tableId -> context.getTableState(tableId) == TableState.ONLINE); + + HashSet tservers = new HashSet<>(); + for (String tserver : context.instanceOperations().getTabletServers()) { + try { + var zPath = ServiceLock.path(ZooUtil.getRoot(context.instanceOperations().getInstanceId()) + + Constants.ZTSERVERS + "/" + tserver); + long sessionId = ServiceLock.getSessionId(context.getZooCache(), zPath); + tservers.add(new TServerInstance(tserver, sessionId)); + } catch (Exception e) { + throw new RuntimeException(e); } - this.tservers = Collections.unmodifiableSet(tservers); - return tservers; - } - - @Override - public Set onlineTables() { - Set onlineTables = context.getTableIdToNameMap().keySet(); - this.onlineTables = - Sets.filter(onlineTables, tableId -> context.getTableState(tableId) == TableState.ONLINE); - return this.onlineTables; - } - - @Override - public Map> tServerResourceGroups() { - return new HashMap<>(); - } - - @Override - public Set migrationsSnapshot() { - return Collections.emptySet(); - } - - @Override - public Set shutdownServers() { - return Collections.emptySet(); } - @Override - public ManagerState getManagerState() { - return ManagerState.NORMAL; - } - - @Override - public Map> getCompactionHints() { - return Map.of(); - } - - @Override - public List> getVolumeReplacements() { - return List.of(); - } - - @Override - public String toString() { - return "tservers: " + tservers + " onlineTables: " + onlineTables; - } + return new TabletManagementParameters(ManagerState.NORMAL, + Map.of( + Ample.DataLevel.ROOT, true, Ample.DataLevel.USER, true, Ample.DataLevel.METADATA, true), + onlineTables, + new LiveTServerSet.LiveTServersSnapshot(tservers, + Map.of(Constants.DEFAULT_RESOURCE_GROUP_NAME, tservers)), + Set.of(), Map.of(), Ample.DataLevel.USER, Map.of(), true, replacements); } - } diff --git a/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java b/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java index cd183ef364c..e8c38a5bc46 100644 --- a/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java @@ -52,11 +52,11 @@ import org.apache.accumulo.core.clientImpl.ClientTabletCache; import org.apache.accumulo.core.conf.ClientProperty; import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.dataImpl.KeyExtent; import org.apache.accumulo.core.metadata.MetadataTable; -import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.TServerInstance; +import org.apache.accumulo.core.metadata.schema.Ample; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletMetadata.LocationType; import org.apache.accumulo.core.rpc.clients.ThriftClientTypes; @@ -65,7 +65,6 @@ import org.apache.accumulo.minicluster.ServerType; import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; import org.apache.accumulo.miniclusterImpl.ProcessReference; -import org.apache.accumulo.server.manager.state.TabletManagementScanner; import org.apache.accumulo.test.functional.ConfigurableMacBase; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; @@ -135,11 +134,11 @@ public void setUp() throws Exception { // Wait for the balancer to assign all metadata tablets to the chosen server. ClientContext ctx = (ClientContext) client; - TabletLocations tl = TabletLocations.retrieve(ctx, MetadataTable.NAME, RootTable.NAME); + TabletLocations tl = TabletLocations.retrieve(ctx, MetadataTable.NAME); while (tl.hosted.keySet().size() != 1 || !tl.hosted.containsKey(metadataServer)) { log.info("Metadata tablets are not hosted on the correct server. Waiting for balancer..."); Thread.sleep(1000L); - tl = TabletLocations.retrieve(ctx, MetadataTable.NAME, RootTable.NAME); + tl = TabletLocations.retrieve(ctx, MetadataTable.NAME); } log.info("Metadata tablets are now hosted on {}", metadataServer); } @@ -403,11 +402,6 @@ private static class TabletLocations { public static TabletLocations retrieve(final ClientContext ctx, final String tableName) throws Exception { - return retrieve(ctx, tableName, MetadataTable.NAME); - } - - public static TabletLocations retrieve(final ClientContext ctx, final String tableName, - final String metaName) throws Exception { int sleepTime = 200; int remainingAttempts = 30; @@ -415,7 +409,7 @@ public static TabletLocations retrieve(final ClientContext ctx, final String tab try { FutureTask tlsFuture = new FutureTask<>(() -> { TabletLocations answer = new TabletLocations(); - answer.scan(ctx, tableName, metaName); + answer.scan(ctx, tableName); return answer; }); THREAD_POOL.execute(tlsFuture); @@ -434,12 +428,14 @@ public static TabletLocations retrieve(final ClientContext ctx, final String tab } } - private void scan(ClientContext ctx, String tableName, String metaName) { + private void scan(ClientContext ctx, String tableName) { Map idMap = ctx.tableOperations().tableIdMap(); String tableId = Objects.requireNonNull(idMap.get(tableName)); - try (var scanner = new TabletManagementScanner(ctx, new Range(), metaName)) { + var level = Ample.DataLevel.of(TableId.of(tableId)); + try (var tablets = ctx.getAmple().readTablets().forLevel(level).build()) { + var scanner = tablets.iterator(); while (scanner.hasNext()) { - final TabletMetadata tm = scanner.next().getTabletMetadata(); + final TabletMetadata tm = scanner.next(); final KeyExtent ke = tm.getExtent(); if (!tm.getTableId().canonical().equals(tableId)) { diff --git a/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java b/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java index 63175fb26c6..af8f9ddd02e 100644 --- a/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java +++ b/test/src/main/java/org/apache/accumulo/test/performance/NullTserver.java @@ -53,7 +53,6 @@ import org.apache.accumulo.core.dataImpl.thrift.TSummaryRequest; import org.apache.accumulo.core.dataImpl.thrift.UpdateErrors; import org.apache.accumulo.core.manager.thrift.TabletServerStatus; -import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.metadata.schema.TabletMetadata; @@ -73,7 +72,6 @@ import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.client.ClientServiceHandler; import org.apache.accumulo.server.manager.state.Assignment; -import org.apache.accumulo.server.manager.state.TabletManagementScanner; import org.apache.accumulo.server.manager.state.TabletStateStore; import org.apache.accumulo.server.rpc.TServerUtils; import org.apache.accumulo.server.rpc.ThriftProcessorTypes; @@ -311,12 +309,13 @@ public static void main(String[] args) throws Exception { // read the locations for the table Range tableRange = new KeyExtent(tableId, null, null).toMetaRange(); List assignments = new ArrayList<>(); - try (var s = new TabletManagementScanner(context, tableRange, MetadataTable.NAME)) { + try (var tablets = context.getAmple().readTablets().forLevel(DataLevel.USER).build()) { long randomSessionID = opts.port; TServerInstance instance = new TServerInstance(addr, randomSessionID); + var s = tablets.iterator(); while (s.hasNext()) { - TabletMetadata next = s.next().getTabletMetadata(); + TabletMetadata next = s.next(); assignments.add(new Assignment(next.getExtent(), instance, next.getLast())); } }