-
Notifications
You must be signed in to change notification settings - Fork 448
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add Mergeability column to support automatic merges (#5187)
Add Mergeability column to support automatic merges This adds a new Mergeability column for marking if/when tablets are eligible to me merged by the system based on a threshold. The column stores two values, a duration which is a delay for when a tablet can be merged that is relative to the time the Manager uses, Steady time. It also stores the current Steady time value when inserted so that later we can add the delay plus the original time when inserted to see if enough time has passed. The Steady Time value will only be stored if the delay is >= 0, otherwise it will be null as it won't be used. The Steady Time value isn't technically needed for a delay of 0 because 0 means it's eligible to merge always, but it could be useful if we wanted to change the value for some reason and it would allow logging when it was stored. There are 2 possible states for the delay value: 1) null : This means a tablet will never automatically merge 2) duration >= 0 : Tablet is eligible to merge after the given delay (stored as a duration), relative to the current system Steady time. Ie. the tablet can be merged if the current manager time is later than the delay value + the steady time value when inserted. If the duration is 0 then it means it can merge always (now). This change only adds the new column itself and populates it. The default is to never merge automatically for all cases except for when the system automatically splits tablets. In that case the newly split tablets are marked as being eligible to merge always (duration of 0). Future updates will add API enhancements to allow setting/viewing the mergeability setting as well as to enable automatic merging by the system that is based on this new column value. When automatic merging is enabled, if a user wants to make a tablet eligible to be merged in the future they would do so by setting a delay that is positive. For example, to make a tablet eligible to be merged 3 days from now the user set a duration of 3 days in the API (future PR) and when the system inserts the value into metadata it will also include the current SteadyTime on creation. Later when the the current steady time passes that set delay + original stored steady time value the tablet would be eligible to be merged. To enable merging immediately they can set the duration to 0 or use the TabletMergeability.always() helper which is just a shortcut to a duration of 0.
- Loading branch information
Showing
30 changed files
with
525 additions
and
38 deletions.
There are no files selected for viewing
136 changes: 136 additions & 0 deletions
136
core/src/main/java/org/apache/accumulo/core/client/admin/TabletMergeability.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,136 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * https://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
| package org.apache.accumulo.core.client.admin; | ||
|
|
||
| import java.io.Serializable; | ||
| import java.time.Duration; | ||
| import java.util.Objects; | ||
| import java.util.Optional; | ||
|
|
||
| import com.google.common.base.Preconditions; | ||
|
|
||
| /** | ||
| * @since 4.0.0 | ||
| */ | ||
| public class TabletMergeability implements Serializable { | ||
| private static final long serialVersionUID = 1L; | ||
|
|
||
| private static final TabletMergeability NEVER = new TabletMergeability(); | ||
| private static final TabletMergeability ALWAYS = new TabletMergeability(Duration.ZERO); | ||
|
|
||
| private final Duration delay; | ||
|
|
||
| private TabletMergeability(Duration delay) { | ||
| this.delay = Objects.requireNonNull(delay); | ||
| } | ||
|
|
||
| // Edge case for NEVER | ||
| private TabletMergeability() { | ||
| this.delay = null; | ||
| } | ||
|
|
||
| /** | ||
| * Determines if the configured delay signals a tablet is never eligible to be automatically | ||
| * merged. | ||
| * | ||
| * @return true if never mergeable, else false | ||
| */ | ||
| public boolean isNever() { | ||
| return this.delay == null; | ||
| } | ||
|
|
||
| /** | ||
| * Determines if the configured delay signals a tablet is always eligible to be automatically | ||
| * merged now. (Has a delay of 0) | ||
| * | ||
| * @return true if always mergeable now, else false | ||
| */ | ||
| public boolean isAlways() { | ||
| return delay != null && this.delay.isZero(); | ||
| } | ||
|
|
||
| /** | ||
| * Returns an Optional duration of the delay which is one of: | ||
| * | ||
| * <ul> | ||
| * <li>empty (never)</li> | ||
| * <li>0 (now)</li> | ||
| * <li>positive delay</li> | ||
| * </ul> | ||
| * | ||
| * @return the configured mergeability delay | ||
| */ | ||
| public Optional<Duration> getDelay() { | ||
| return Optional.ofNullable(delay); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean equals(Object o) { | ||
| if (o == null || getClass() != o.getClass()) { | ||
| return false; | ||
| } | ||
| TabletMergeability that = (TabletMergeability) o; | ||
| return Objects.equals(delay, that.delay); | ||
| } | ||
|
|
||
| @Override | ||
| public int hashCode() { | ||
| return Objects.hashCode(delay); | ||
| } | ||
|
|
||
| @Override | ||
| public String toString() { | ||
| if (delay == null) { | ||
| return "TabletMergeability=NEVER"; | ||
| } | ||
| return "TabletMergeability=AFTER:" + delay.toMillis() + "ms"; | ||
| } | ||
|
|
||
| /** | ||
| * Signifies that a tablet is never eligible to be automatically merged. | ||
| * | ||
| * @return a {@link TabletMergeability} with an empty delay signaling never merge | ||
| */ | ||
| public static TabletMergeability never() { | ||
| return NEVER; | ||
| } | ||
|
|
||
| /** | ||
| * Signifies that a tablet is eligible now to be automatically merged | ||
| * | ||
| * @return a {@link TabletMergeability} with a delay of 0 signaling never merge | ||
| */ | ||
| public static TabletMergeability always() { | ||
| return ALWAYS; | ||
| } | ||
|
|
||
| /** | ||
| * Creates a {@link TabletMergeability} that signals a tablet has a delay to a point in the future | ||
| * before it is automatically eligible to be merged. The duration must be positive value. | ||
| * | ||
| * @param delay the duration of the delay | ||
| * | ||
| * @return a {@link TabletMergeability} from the given delay. | ||
| */ | ||
| public static TabletMergeability after(Duration delay) { | ||
| Preconditions.checkArgument(delay.toNanos() >= 0, "Duration of delay must be greater than 0."); | ||
| return new TabletMergeability(delay); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
144 changes: 144 additions & 0 deletions
144
core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMergeabilityMetadata.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,144 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * https://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
| package org.apache.accumulo.core.metadata.schema; | ||
|
|
||
| import static org.apache.accumulo.core.util.LazySingletons.GSON; | ||
|
|
||
| import java.io.Serializable; | ||
| import java.time.Duration; | ||
| import java.util.Objects; | ||
| import java.util.Optional; | ||
| import java.util.concurrent.TimeUnit; | ||
|
|
||
| import org.apache.accumulo.core.client.admin.TabletMergeability; | ||
| import org.apache.accumulo.core.data.Value; | ||
| import org.apache.accumulo.core.util.time.SteadyTime; | ||
|
|
||
| import com.google.common.base.Preconditions; | ||
|
|
||
| public class TabletMergeabilityMetadata implements Serializable { | ||
| private static final long serialVersionUID = 1L; | ||
|
|
||
| private static final TabletMergeabilityMetadata NEVER = | ||
| new TabletMergeabilityMetadata(TabletMergeability.never());; | ||
|
|
||
| private final TabletMergeability tabletMergeability; | ||
| private final SteadyTime steadyTime; | ||
|
|
||
| private TabletMergeabilityMetadata(TabletMergeability tabletMergeability, SteadyTime steadyTime) { | ||
| this.tabletMergeability = Objects.requireNonNull(tabletMergeability); | ||
| this.steadyTime = steadyTime; | ||
| // This makes sure that SteadyTime is set if TabletMergeability has a delay, and is null | ||
| // if TabletMergeability is NEVER as we don't need to store it in that case | ||
| Preconditions.checkArgument(tabletMergeability.isNever() == (steadyTime == null), | ||
| "SteadyTime must be set if and only if TabletMergeability delay is >= 0"); | ||
| } | ||
|
|
||
| private TabletMergeabilityMetadata(TabletMergeability tabletMergeability) { | ||
| this(tabletMergeability, null); | ||
| } | ||
|
|
||
| public TabletMergeability getTabletMergeability() { | ||
| return tabletMergeability; | ||
| } | ||
|
|
||
| public Optional<SteadyTime> getSteadyTime() { | ||
| return Optional.ofNullable(steadyTime); | ||
| } | ||
|
|
||
| public boolean isMergeable(SteadyTime currentTime) { | ||
| if (tabletMergeability.isNever()) { | ||
| return false; | ||
| } | ||
| // Steady time should never be null unless TabletMergeability is NEVER | ||
| Preconditions.checkState(steadyTime != null, "SteadyTime should be set"); | ||
| var totalDelay = steadyTime.getDuration().plus(tabletMergeability.getDelay().orElseThrow()); | ||
| return currentTime.getDuration().compareTo(totalDelay) >= 0; | ||
| } | ||
|
|
||
| private static class GSonData { | ||
| boolean never; | ||
| Long delay; | ||
| Long steadyTime; | ||
| } | ||
|
|
||
| String toJson() { | ||
| GSonData jData = new GSonData(); | ||
| jData.never = tabletMergeability.isNever(); | ||
| jData.delay = tabletMergeability.getDelay().map(Duration::toNanos).orElse(null); | ||
| jData.steadyTime = steadyTime != null ? steadyTime.getNanos() : null; | ||
| return GSON.get().toJson(jData); | ||
| } | ||
|
|
||
| static TabletMergeabilityMetadata fromJson(String json) { | ||
| GSonData jData = GSON.get().fromJson(json, GSonData.class); | ||
| if (jData.never) { | ||
| Preconditions.checkArgument(jData.delay == null && jData.steadyTime == null, | ||
| "delay and steadyTime should be null if mergeability 'never' is true"); | ||
| } else { | ||
| Preconditions.checkArgument(jData.delay != null && jData.steadyTime != null, | ||
| "delay and steadyTime should both be set if mergeability 'never' is false"); | ||
| } | ||
| TabletMergeability tabletMergeability = jData.never ? TabletMergeability.never() | ||
| : TabletMergeability.after(Duration.ofNanos(jData.delay)); | ||
| SteadyTime steadyTime = | ||
| jData.steadyTime != null ? SteadyTime.from(jData.steadyTime, TimeUnit.NANOSECONDS) : null; | ||
| return new TabletMergeabilityMetadata(tabletMergeability, steadyTime); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean equals(Object o) { | ||
| if (o == null || getClass() != o.getClass()) { | ||
| return false; | ||
| } | ||
| TabletMergeabilityMetadata that = (TabletMergeabilityMetadata) o; | ||
| return Objects.equals(tabletMergeability, that.tabletMergeability) | ||
| && Objects.equals(steadyTime, that.steadyTime); | ||
| } | ||
|
|
||
| @Override | ||
| public int hashCode() { | ||
| return Objects.hash(tabletMergeability, steadyTime); | ||
| } | ||
|
|
||
| @Override | ||
| public String toString() { | ||
| return "TabletMergeabilityMetadata{" + tabletMergeability + ", " + steadyTime + '}'; | ||
| } | ||
|
|
||
| public static TabletMergeabilityMetadata never() { | ||
| return NEVER; | ||
| } | ||
|
|
||
| public static TabletMergeabilityMetadata always(SteadyTime currentTime) { | ||
| return new TabletMergeabilityMetadata(TabletMergeability.always(), currentTime); | ||
| } | ||
|
|
||
| public static TabletMergeabilityMetadata after(Duration delay, SteadyTime currentTime) { | ||
| return new TabletMergeabilityMetadata(TabletMergeability.after(delay), currentTime); | ||
| } | ||
|
|
||
| public static Value toValue(TabletMergeabilityMetadata tmm) { | ||
| return new Value(tmm.toJson()); | ||
| } | ||
|
|
||
| public static TabletMergeabilityMetadata fromValue(Value value) { | ||
| return TabletMergeabilityMetadata.fromJson(value.toString()); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.