From 36bfda98c1059fcc476cd742140d82f837db30a7 Mon Sep 17 00:00:00 2001 From: linlin-s Date: Tue, 5 Dec 2023 17:15:39 -0800 Subject: [PATCH] Add option --auto-flush to enable benchmarking of ion-java writer with auto-flush configured. (#59) --- src/com/amazon/ion/benchmark/Constants.java | 1 + .../amazon/ion/benchmark/IonUtilities.java | 5 +++ src/com/amazon/ion/benchmark/Main.java | 6 +++- .../ion/benchmark/OptionsCombinationBase.java | 5 ++- .../ion/benchmark/OptionsMatrixBase.java | 12 ++++++- tst/com/amazon/ion/benchmark/OptionsTest.java | 33 +++++++++++++++++++ 6 files changed, 59 insertions(+), 3 deletions(-) diff --git a/src/com/amazon/ion/benchmark/Constants.java b/src/com/amazon/ion/benchmark/Constants.java index 1b08068..2567abd 100644 --- a/src/com/amazon/ion/benchmark/Constants.java +++ b/src/com/amazon/ion/benchmark/Constants.java @@ -25,6 +25,7 @@ class Constants { static final String JSON_USE_BIG_DECIMALS_NAME = "g"; static final String AUTO_VALUE = "auto"; static final String NONE_VALUE = "none"; + static final String AUTO_FLUSH_ENABLED = "m"; private Constants() { // Do not instantiate. diff --git a/src/com/amazon/ion/benchmark/IonUtilities.java b/src/com/amazon/ion/benchmark/IonUtilities.java index 355f0e3..2d73165 100644 --- a/src/com/amazon/ion/benchmark/IonUtilities.java +++ b/src/com/amazon/ion/benchmark/IonUtilities.java @@ -168,6 +168,11 @@ static IonWriterSupplier newBinaryWriterSupplier(OptionsCombinationBase options) } else { builder.withFloatBinary32Disabled(); } + if(options.autoFlush) { + builder.withAutoFlushEnabled(); + } else { + builder.withAutoFlushDisabled(); + } if (options instanceof WriteOptionsCombination) { // When this method is used by the read benchmark for converting the input file, 'options' will be a // ReadOptionsCombination, which does not have the 'ionWriterUserBufferSize' value, because this value diff --git a/src/com/amazon/ion/benchmark/Main.java b/src/com/amazon/ion/benchmark/Main.java index 1dc5c9d..f4fbf7b 100644 --- a/src/com/amazon/ion/benchmark/Main.java +++ b/src/com/amazon/ion/benchmark/Main.java @@ -30,7 +30,7 @@ public class Main { + "[--results-file ] [--io-type ]... [--io-buffer-size ]... [--format ]... " + "[--api ]... [--ion-imports-for-input ] [--ion-imports-for-benchmark ]... " + "[--ion-flush-period ]... [--ion-length-preallocation ]... [--ion-float-width ]... " - + "[--ion-use-symbol-tokens ]... [--ion-writer-block-size ]... " + + "[--ion-use-symbol-tokens ]... [--ion-writer-block-size ]... [--auto-flush ]..." + "[--json-use-big-decimals ]... \n" + " ion-java-benchmark read [--profile] [--limit ] [--mode ] [--time-unit ] " @@ -200,6 +200,10 @@ public class Main { + "specified multiple times to compare different values. Ignored unless the format is ion_binary. " + "[default: auto]\n" + + " -m --auto-flush If this option is enabled, then the flush operation will be executed " + + "automatically when the size of the value exceeds the writer's block size. This option may be specified multiple times to compare different values." + + "[default: false]\n" + // 'read' options: + " -s --paths A file containing a sequence of Ion s-expressions representing " diff --git a/src/com/amazon/ion/benchmark/OptionsCombinationBase.java b/src/com/amazon/ion/benchmark/OptionsCombinationBase.java index d774b4d..32be898 100644 --- a/src/com/amazon/ion/benchmark/OptionsCombinationBase.java +++ b/src/com/amazon/ion/benchmark/OptionsCombinationBase.java @@ -19,9 +19,10 @@ import java.nio.file.Path; import java.util.function.Function; +import static com.amazon.ion.benchmark.Constants.API_NAME; +import static com.amazon.ion.benchmark.Constants.AUTO_FLUSH_ENABLED; import static com.amazon.ion.benchmark.Constants.FLUSH_PERIOD_NAME; import static com.amazon.ion.benchmark.Constants.FORMAT_NAME; -import static com.amazon.ion.benchmark.Constants.API_NAME; import static com.amazon.ion.benchmark.Constants.ION_FLOAT_WIDTH_NAME; import static com.amazon.ion.benchmark.Constants.ION_IMPORTS_FOR_BENCHMARK_NAME; import static com.amazon.ion.benchmark.Constants.ION_IMPORTS_FOR_INPUT_NAME; @@ -49,6 +50,7 @@ abstract class OptionsCombinationBase { final boolean useSymbolTokens; final int limit; final boolean jsonUseBigDecimals; + final boolean autoFlush; /** * Retrieves and translates a value from the struct, if the field is present and is not the 'auto' value. Otherwise, @@ -89,6 +91,7 @@ static T getOrDefault(IonStruct options, String fieldName, Function ((IonBool) val).booleanValue(), false); limit = getOrDefault(optionsCombinationStruct, LIMIT_NAME, val -> ((IonInt) val).intValue(), Integer.MAX_VALUE); jsonUseBigDecimals = getOrDefault(optionsCombinationStruct, JSON_USE_BIG_DECIMALS_NAME, val -> ((IonBool) val).booleanValue(), true); + autoFlush = getOrDefault(optionsCombinationStruct, AUTO_FLUSH_ENABLED, val -> ((IonBool) val).booleanValue(), false); } /** diff --git a/src/com/amazon/ion/benchmark/OptionsMatrixBase.java b/src/com/amazon/ion/benchmark/OptionsMatrixBase.java index ae227a0..b971002 100644 --- a/src/com/amazon/ion/benchmark/OptionsMatrixBase.java +++ b/src/com/amazon/ion/benchmark/OptionsMatrixBase.java @@ -23,9 +23,10 @@ import java.util.function.Predicate; import java.util.function.Supplier; +import static com.amazon.ion.benchmark.Constants.API_NAME; +import static com.amazon.ion.benchmark.Constants.AUTO_FLUSH_ENABLED; import static com.amazon.ion.benchmark.Constants.FLUSH_PERIOD_NAME; import static com.amazon.ion.benchmark.Constants.FORMAT_NAME; -import static com.amazon.ion.benchmark.Constants.API_NAME; import static com.amazon.ion.benchmark.Constants.ION_FLOAT_WIDTH_NAME; import static com.amazon.ion.benchmark.Constants.ION_IMPORTS_FOR_BENCHMARK_NAME; import static com.amazon.ion.benchmark.Constants.ION_IMPORTS_FOR_INPUT_NAME; @@ -403,6 +404,15 @@ private static String getFileOrDefault(String fileNameOrNone, String defaultFile () -> ION_SYSTEM.newBool(true), OPTION_ONLY_APPLIES_TO_JSON ); + parseAndCombine( + optionsMatrix.get("--auto-flush"), + AUTO_FLUSH_ENABLED, + OptionsMatrixBase::getTrueOrNull, + ION_SYSTEM::newBool, + optionsCombinationStructs, + () -> ION_SYSTEM.newBool(false), + OPTION_ONLY_APPLIES_TO_ION_STREAMING + ); parseCommandSpecificOptions(optionsMatrix, optionsCombinationStructs); serializedOptionsCombinations = serializeOptionsCombinations(optionsCombinationStructs); if (profile) { diff --git a/tst/com/amazon/ion/benchmark/OptionsTest.java b/tst/com/amazon/ion/benchmark/OptionsTest.java index 65ba936..2500951 100644 --- a/tst/com/amazon/ion/benchmark/OptionsTest.java +++ b/tst/com/amazon/ion/benchmark/OptionsTest.java @@ -353,6 +353,7 @@ private static class ExpectedWriteOptionsCombination extends ExpectedOptionsCombinationBase { Integer ionWriterBlockSize = null; + Boolean autoFlush = null; static ExpectedWriteOptionsCombination defaultOptions() { return new ExpectedWriteOptionsCombination(); @@ -363,6 +364,11 @@ final ExpectedWriteOptionsCombination ionWriterBlockSize(Integer ionWriterBlockS return this; } + final ExpectedWriteOptionsCombination autoFlush(Boolean autoFlush) { + this.autoFlush = autoFlush; + return this; + } + @Override void assertOptionsEqual(WriteOptionsCombination that) { super.assertOptionsEqual(that); @@ -1224,6 +1230,33 @@ public void ionWriterBlockSize() throws Exception { assertTrue(expectedCombinations.isEmpty()); } + @Test + public void autoFlush() throws Exception { + List optionsCombinations = parseOptionsCombinations( + "write", + "--auto-flush", + "true", + "--auto-flush", + "false", + "--io-type", + "buffer", + "binaryStructs.10n" + ); + assertEquals(2, optionsCombinations.size()); + List expectedCombinations = new ArrayList<>(2); + + expectedCombinations.add(ExpectedWriteOptionsCombination.defaultOptions().autoFlush(true)); + expectedCombinations.add(ExpectedWriteOptionsCombination.defaultOptions().autoFlush(false)); + + for (WriteOptionsCombination optionsCombination : optionsCombinations) { + expectedCombinations.removeIf(candidate -> nullSafeEquals(candidate.autoFlush, optionsCombination.autoFlush)); + + assertWriteTaskExecutesCorrectly("binaryStructs.10n", optionsCombination, Format.ION_BINARY, IoType.BUFFER); + assertWriteTaskExecutesCorrectly("textStructs.ion", optionsCombination, Format.ION_BINARY, IoType.BUFFER); + } + assertTrue(expectedCombinations.isEmpty()); + } + @Test public void writeAllTypes() throws Exception { List optionsCombinations = parseOptionsCombinations(