Move the tracking of unsplittable tablets to metadata table

core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java

@@ -419,6 +419,10 @@ interface TabletUpdates<T> {
     T putUserCompactionRequested(FateId fateId);
 
     T deleteUserCompactionRequested(FateId fateId);
+
+    T setUnSplittable(UnSplittableMetadata unSplittableMeta);
+
+    T deleteUnSplittable();
   }
 
   interface TabletMutator extends TabletUpdates<TabletMutator> {

core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java

@@ -430,6 +430,19 @@ public static class UserCompactionRequestedColumnFamily {
       public static final Text NAME = new Text(STR_NAME);
     }
 
+    /**
+     * This family is used to track information needed for splits. Currently, the only thing stored
+     * is if the tablets are un-splittable based on the files the tablet and configuration related
+     * to splits.
+     */
+    public static class SplitColumnFamily {
+      public static final String STR_NAME = "split";
+      public static final Text NAME = new Text(STR_NAME);
+      public static final String UNSPLITTABLE_QUAL = "unsplittable";
+      public static final ColumnFQ UNSPLITTABLE_COLUMN =
+          new ColumnFQ(NAME, new Text(UNSPLITTABLE_QUAL));
+    }
+
     // TODO when removing the Upgrader12to13 class in the upgrade package, also remove this class.
     public static class Upgrade12to13 {
 

core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java

@@ -70,6 +70,7 @@
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.MergedColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SplitColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SuspendLocationColumn;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.UserCompactionRequestedColumnFamily;
@@ -82,6 +83,8 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
@@ -120,6 +123,8 @@ public class TabletMetadata {
   private boolean futureAndCurrentLocationSet = false;
   private Set<FateId> compacted;
   private Set<FateId> userCompactionsRequested;
+  private UnSplittableMetadata unSplittableMetadata;
+  private Supplier<Long> fileSize;
 
   public static TabletMetadataBuilder builder(KeyExtent extent) {
     return new TabletMetadataBuilder(extent);
@@ -150,7 +155,8 @@ public enum ColumnType {
     OPID,
     SELECTED,
     COMPACTED,
-    USER_COMPACTION_REQUESTED
+    USER_COMPACTION_REQUESTED,
+    UNSPLITTABLE
   }
 
   public static class Location {
@@ -316,6 +322,14 @@ public Map<StoredTabletFile,DataFileValue> getFilesMap() {
     return files;
   }
 
+  /**
+   * @return the sum of the tablets files sizes
+   */
+  public long getFileSize() {
+    ensureFetched(ColumnType.FILES);
+    return fileSize.get();
+  }
+
   public SelectedFiles getSelectedFiles() {
     ensureFetched(ColumnType.SELECTED);
     return selectedFiles;
@@ -381,6 +395,11 @@ public boolean getHostingRequested() {
     return onDemandHostingRequested;
   }
 
+  public UnSplittableMetadata getUnSplittable() {
+    ensureFetched(ColumnType.UNSPLITTABLE);
+    return unSplittableMetadata;
+  }
+
   @Override
   public String toString() {
     return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE).append("tableId", tableId)
@@ -394,7 +413,8 @@ public String toString() {
         .append("onDemandHostingRequested", onDemandHostingRequested)
         .append("operationId", operationId).append("selectedFiles", selectedFiles)
         .append("futureAndCurrentLocationSet", futureAndCurrentLocationSet)
-        .append("userCompactionsRequested", userCompactionsRequested).toString();
+        .append("userCompactionsRequested", userCompactionsRequested)
+        .append("unSplittableMetadata", unSplittableMetadata).toString();
   }
 
   public SortedMap<Key,Value> getKeyValues() {
@@ -545,6 +565,13 @@ public static <E extends Entry<Key,Value>> TabletMetadata convertRow(Iterator<E>
         case UserCompactionRequestedColumnFamily.STR_NAME:
           userCompactionsRequestedBuilder.add(FateId.from(qual));
           break;
+        case SplitColumnFamily.STR_NAME:
+          if (qual.equals(SplitColumnFamily.UNSPLITTABLE_QUAL)) {
+            te.unSplittableMetadata = UnSplittableMetadata.toUnSplittable(val);
+          } else {
+            throw new IllegalStateException("Unexpected SplitColumnFamily qualifier: " + qual);
+          }
+          break;
         default:
           throw new IllegalStateException("Unexpected family " + fam);
 
@@ -557,7 +584,10 @@ public static <E extends Entry<Key,Value>> TabletMetadata convertRow(Iterator<E>
       te.availability = TabletAvailability.HOSTED;
     }
 
-    te.files = filesBuilder.build();
+    var files = filesBuilder.build();
+    te.files = files;
+    te.fileSize =
+        Suppliers.memoize(() -> files.values().stream().mapToLong(DataFileValue::getSize).sum());
     te.loadedFiles = loadedFilesBuilder.build();
     te.fetchedCols = fetchedColumns;
     te.scans = scansBuilder.build();

core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadataBuilder.java

@@ -36,6 +36,7 @@
 import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SELECTED;
 import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SUSPEND;
 import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.TIME;
+import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.UNSPLITTABLE;
 import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.USER_COMPACTION_REQUESTED;
 
 import java.util.Arrays;
@@ -295,6 +296,18 @@ public TabletMetadataBuilder deleteUserCompactionRequested(FateId fateId) {
     throw new UnsupportedOperationException();
   }
 
+  @Override
+  public TabletMetadataBuilder setUnSplittable(UnSplittableMetadata unSplittableMeta) {
+    fetched.add(UNSPLITTABLE);
+    internalBuilder.setUnSplittable(unSplittableMeta);
+    return this;
+  }
+
+  @Override
+  public TabletMetadataBuilder deleteUnSplittable() {
+    throw new UnsupportedOperationException();
+  }
+
   /**
    * @param extraFetched Anything that was put on the builder will automatically be added to the
    *        fetched set. However, for the case where something was not put and it needs to be

core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMutatorBase.java

@@ -44,6 +44,7 @@
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.MergedColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SplitColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SuspendLocationColumn;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.UserCompactionRequestedColumnFamily;
@@ -347,6 +348,18 @@ public T deleteUserCompactionRequested(FateId fateId) {
     return getThis();
   }
 
+  @Override
+  public T setUnSplittable(UnSplittableMetadata unSplittableMeta) {
+    SplitColumnFamily.UNSPLITTABLE_COLUMN.put(mutation, new Value(unSplittableMeta.toBase64()));
+    return getThis();
+  }
+
+  @Override
+  public T deleteUnSplittable() {
+    SplitColumnFamily.UNSPLITTABLE_COLUMN.putDelete(mutation);
+    return getThis();
+  }
+
   public void setCloseAfterMutate(AutoCloseable closeable) {
     this.closeAfterMutate = closeable;
   }

core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java

@@ -81,6 +81,7 @@
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.MergedColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SplitColumnFamily;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.SuspendLocationColumn;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.UserCompactionRequestedColumnFamily;
 import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
@@ -396,6 +397,9 @@ public Options fetch(ColumnType... colsToFetch) {
           case USER_COMPACTION_REQUESTED:
             families.add(UserCompactionRequestedColumnFamily.NAME);
             break;
+          case UNSPLITTABLE:
+            qualifiers.add(SplitColumnFamily.UNSPLITTABLE_COLUMN);
+            break;
           default:
             throw new IllegalArgumentException("Unknown col type " + colToFetch);
         }

core/src/main/java/org/apache/accumulo/core/metadata/schema/UnSplittableMetadata.java

@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.metadata.schema;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Base64;
+import java.util.Objects;
+import java.util.Set;
+
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
+
+import com.google.common.base.Preconditions;
+import com.google.common.hash.HashCode;
+import com.google.common.hash.Hashing;
+
+public class UnSplittableMetadata {
+
+  private final HashCode hashOfSplitParameters;
+
+  private UnSplittableMetadata(KeyExtent keyExtent, long splitThreshold, long maxEndRowSize,
+      int maxFilesToOpen, Set<StoredTabletFile> files) {
+    this(calculateSplitParamsHash(keyExtent, splitThreshold, maxEndRowSize, maxFilesToOpen, files));
+  }
+
+  private UnSplittableMetadata(HashCode hashOfSplitParameters) {
+    this.hashOfSplitParameters = Objects.requireNonNull(hashOfSplitParameters);
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    UnSplittableMetadata that = (UnSplittableMetadata) o;
+    return Objects.equals(hashOfSplitParameters, that.hashOfSplitParameters);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(hashOfSplitParameters);
+  }
+
+  @Override
+  public String toString() {
+    return toBase64();
+  }
+
+  public String toBase64() {
+    return Base64.getEncoder().encodeToString(hashOfSplitParameters.asBytes());
+  }
+
+  @SuppressWarnings("UnstableApiUsage")
+  private static HashCode calculateSplitParamsHash(KeyExtent keyExtent, long splitThreshold,
+      long maxEndRowSize, int maxFilesToOpen, Set<StoredTabletFile> files) {
+    Preconditions.checkArgument(splitThreshold > 0, "splitThreshold must be greater than 0");
+    Preconditions.checkArgument(maxEndRowSize > 0, "maxEndRowSize must be greater than 0");
+    Preconditions.checkArgument(maxFilesToOpen > 0, "maxFilesToOpen must be greater than 0");
+
+    // Use static call to murmur3_128() so the seed is always the same
+    // Hashing.goodFastHash will seed with the current time, and we need the seed to be
+    // the same across restarts and instances
+    var hasher = Hashing.murmur3_128().newHasher();
+    hasher.putBytes(serializeKeyExtent(keyExtent)).putLong(splitThreshold).putLong(maxEndRowSize)
+        .putInt(maxFilesToOpen);
+    files.stream().map(StoredTabletFile::getNormalizedPathStr).sorted()
+        .forEach(path -> hasher.putString(path, UTF_8));
+    return hasher.hash();
+  }
+
+  public static UnSplittableMetadata toUnSplittable(KeyExtent keyExtent, long splitThreshold,
+      long maxEndRowSize, int maxFilesToOpen, Set<StoredTabletFile> files) {
+    return new UnSplittableMetadata(keyExtent, splitThreshold, maxEndRowSize, maxFilesToOpen,
+        files);
+  }
+
+  public static UnSplittableMetadata toUnSplittable(String base64HashOfSplitParameters) {
+    return new UnSplittableMetadata(
+        HashCode.fromBytes(Base64.getDecoder().decode(base64HashOfSplitParameters)));
+  }
+
+  private static byte[] serializeKeyExtent(KeyExtent keyExtent) {
+    try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        DataOutputStream dos = new DataOutputStream(baos)) {
+      keyExtent.writeTo(dos);
+      dos.close();
+      return baos.toByteArray();
+    } catch (IOException e) {
+      throw new UncheckedIOException(e);
+    }
+  }
+}

core/src/main/java/org/apache/accumulo/core/util/Merge.java

@@ -37,7 +37,6 @@
 import org.apache.accumulo.core.data.TableId;
 import org.apache.accumulo.core.dataImpl.KeyExtent;
 import org.apache.accumulo.core.metadata.AccumuloTable;
-import org.apache.accumulo.core.metadata.schema.DataFileValue;
 import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
 import org.apache.accumulo.core.trace.TraceUtil;
 import org.apache.hadoop.io.Text;
@@ -162,10 +161,8 @@ public void mergomatic(AccumuloClient client, String table, Text start, Text end
           .overRange(new KeyExtent(tableId, end, start).toMetaRange()).fetch(FILES, PREV_ROW)
           .build()) {
 
-        Iterator<Size> sizeIterator = tablets.stream().map(tm -> {
-          long size = tm.getFilesMap().values().stream().mapToLong(DataFileValue::getSize).sum();
-          return new Size(tm.getExtent(), size);
-        }).iterator();
+        Iterator<Size> sizeIterator =
+            tablets.stream().map(tm -> new Size(tm.getExtent(), tm.getFileSize())).iterator();
 
         while (sizeIterator.hasNext()) {
           Size next = sizeIterator.next();