From e6118bbc9ca95c09086c7d993d7e8fb9bda0ba9b Mon Sep 17 00:00:00 2001
From: Vaidhy Gopalan <vaidhy@gmail.com>
Date: Mon, 8 Jan 2024 18:40:54 -0800
Subject: [PATCH 01/64] Calculate average by vaidhy

---
 .../onebrc/CalculateAverage_vaidhy.java       | 268 ++++++++++++++++++
 1 file changed, 268 insertions(+)
 create mode 100644 src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
new file mode 100644
index 000000000..7b4fb3dac
--- /dev/null
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -0,0 +1,268 @@
+/*
+ *  Copyright 2023 The original authors
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package dev.morling.onebrc;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.charset.StandardCharsets;
+import java.util.IntSummaryStatistics;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.TreeMap;
+import java.util.function.Consumer;
+
+public class CalculateAverage_vaidhy {
+
+    private final FileService fileService;
+    private final Consumer<String> lineConsumer;
+
+    interface FileService {
+        /**
+         * Returns the size of the file in number of characters.
+         * (Extra credit: assume byte size instead of char size)
+         */
+        // Possible implementation for byte case in HTTP:
+        // byte size = Content-Length header. using HEAD or empty Range.
+        int length();
+
+        /**
+         * Returns substring of the file from character indices.
+         * Expects 0 <= start <= start + length <= fileSize
+         * (Extra credit: assume sub-byte array instead of sub char array)
+         */
+        // Possible implementation for byte case in HTTP:
+        // Using Http Request header "Range", typically used for continuing
+        // partial downloads.
+        byte[] range(int offset, int length);
+
+    }
+
+    public CalculateAverage_vaidhy(FileService fileService,
+                                   Consumer<String> lineConsumer) {
+        this.fileService = fileService;
+        this.lineConsumer = lineConsumer;
+    }
+
+    /// SAMPLE CANDIDATE CODE STARTS
+
+    /**
+     * Reads from a given offset till the end, it calls server in
+     * blocks of scanSize whenever cursor passes the current block.
+     * Typically when hasNext() is called. hasNext() is efficient
+     * in the sense calling second time is cheap if next() is not
+     * called in between. Cheap in the sense no call to server is
+     * made.
+     */
+    // Space complexity = O(scanSize)
+    static class CharStream implements Iterator<Character> {
+
+        private final FileService fileService;
+        private int offset;
+        private final int scanSize;
+        private int index = 0;
+        private String currentChunk = "";
+        private final int fileLength;
+
+        public CharStream(FileService fileService, int offset, int scanSize) {
+            this.fileService = fileService;
+            this.offset = offset;
+            this.scanSize = scanSize;
+            this.fileLength = fileService.length();
+            if (scanSize <= 0) {
+                throw new IllegalArgumentException("scan size must be > 0");
+            }
+            if (offset < 0) {
+                throw new IllegalArgumentException("offset must be >= 0");
+            }
+        }
+
+        @Override
+        public boolean hasNext() {
+            while (index >= currentChunk.length()) {
+                if (offset < fileLength) {
+                    int scanWindow = Math.min(offset + scanSize, fileLength) - offset;
+                    currentChunk = fileService.range(offset, scanWindow);
+                    offset += scanWindow;
+                    index = 0;
+                }
+                else {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        @Override
+        public Character next() {
+            if (hasNext()) {
+                char ch = currentChunk.charAt(index);
+                index++;
+                return ch;
+            }
+            else {
+                throw new NoSuchElementException();
+            }
+        }
+    }
+
+    /**
+     * Reads lines from a given character stream, hasNext() is always
+     * efficient, all work is done only in next().
+     */
+    // Space complexity: O(max line length) in next() call, structure is O(1)
+    // not counting charStream as it is only a reference, we will count that
+    // in worker space.
+    static class LineStream implements Iterator<String> {
+        private final Iterator<Character> charStream;
+        private int readIndex;
+        private final int length;
+
+        public LineStream(Iterator<Character> charStream, int length) {
+            this.charStream = charStream;
+            this.readIndex = 0;
+            this.length = length;
+        }
+
+        @Override
+        public boolean hasNext() {
+            return readIndex <= length && charStream.hasNext();
+        }
+
+        @Override
+        public String next() {
+            if (hasNext()) {
+                StringBuilder builder = new StringBuilder();
+                while (charStream.hasNext()) {
+                    char ch = charStream.next();
+                    readIndex++;
+                    if (ch == '\n') {
+                        break;
+                    }
+                    builder.append(ch);
+                }
+                return builder.toString();
+            }
+            else {
+                throw new NoSuchElementException();
+            }
+        }
+    }
+
+    // Space complexity: O(scanSize) + O(max line length)
+    public void worker(int offset, int chunkSize, int scanSize) {
+        Iterator<Character> charStream = new CharStream(fileService, offset, scanSize);
+        Iterator<String> lineStream = new LineStream(charStream, chunkSize);
+
+        if (offset != 0) {
+            if (lineStream.hasNext()) {
+                // Skip the first line.
+                lineStream.next();
+            }
+            else {
+                // No lines then do nothing.
+                return;
+            }
+        }
+        while (lineStream.hasNext()) {
+            lineConsumer.accept(lineStream.next());
+        }
+    }
+
+    // Space complexity: O(number of workers), not counting
+    // workers space assuming they are running in different hosts.
+    public void master(int chunkSize, int scanSize) {
+        int len = fileService.length();
+        for (int offset = 0; offset < len; offset += chunkSize) {
+            int workerLength = Math.min(len, offset + chunkSize) - offset;
+            worker(offset, workerLength, scanSize);
+        }
+    }
+
+    /// SAMPLE CANDIDATE CODE ENDS
+
+    static class MockFileService implements FileService {
+        private final String contents;
+
+        public MockFileService(String contents) {
+            this.contents = contents;
+        }
+
+        // Provided by the file hosting service:
+        @Override
+        public int length() {
+            // Mock implementation:
+            return contents.length();
+        }
+
+        public String range(int offset, int length) {
+            // Mock implementation
+            return contents.substring(offset, offset + length);
+        }
+    }
+
+    private static final String FILE = "./measurements_1M.txt";
+
+    public interface ChunkProcessor {
+
+        void process(byte[] line);
+
+        Map<String, IntSummaryStatistics> summary();
+    }
+
+    public static class ChunkProcessorImpl implements ChunkProcessor {
+
+        private final Map<String, IntSummaryStatistics> statistics = new TreeMap<>();
+
+        @Override
+        public void process(byte[] line) {
+            String lineStr = new String(line, StandardCharsets.UTF_8);
+            int indexSemi = lineStr.indexOf(';');
+            String station = lineStr.substring(0, indexSemi);
+            String value = lineStr.substring(indexSemi + 1);
+            double val = Double.parseDouble(value);
+            int normalized = (int) (val * 10);
+            statistics.computeIfAbsent(station, (ignore) -> new IntSummaryStatistics());
+
+            statistics.get(station).accept(normalized);
+        }
+
+        @Override
+        public Map<String, IntSummaryStatistics> summary() {
+            return statistics;
+        }
+    }
+
+    public static void main(String[] args) throws IOException {
+
+        ChunkProcessor chunkProcessor = new ChunkProcessorImpl();
+
+        try (FileInputStream fis = new FileInputStream(FILE)) {
+
+            Reader io = new InputStreamReader(fis, StandardCharsets.UTF_8);
+            BufferedReader bufferedReader = new BufferedReader(io);
+            String line;
+            while ((line = bufferedReader.readLine()) != null) {
+                chunkProcessor.process(line.getBytes(StandardCharsets.UTF_8));
+            }
+        }
+
+        System.out.println(chunkProcessor.summary());
+    }
+}

From a8cc335efb2cb2f1e87eb31949c296c236fd4bab Mon Sep 17 00:00:00 2001
From: Vaidhy Gopalan <vaidhy@gmail.com>
Date: Mon, 8 Jan 2024 18:41:20 -0800
Subject: [PATCH 02/64] Calculate average by vaidhy

---
 calculate_average_vaidhy.sh | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100755 calculate_average_vaidhy.sh

diff --git a/calculate_average_vaidhy.sh b/calculate_average_vaidhy.sh
new file mode 100755
index 000000000..29494bd63
--- /dev/null
+++ b/calculate_average_vaidhy.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+#
+#  Copyright 2023 The original authors
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+source "$HOME/.sdkman/bin/sdkman-init.sh"
+sdk use java 21.0.1-open 1>&2
+
+JAVA_OPTS="--enable-preview"
+time java $JAVA_OPTS --class-path target/average-1.0.0-SNAPSHOT.jar dev.morling.onebrc.CalculateAverage_vaidhy

From c2e39bfe76af00ed5013cb9ae3a781b4fbb1f993 Mon Sep 17 00:00:00 2001
From: Anita S V <anitasvasu@gmail.com>
Date: Tue, 9 Jan 2024 00:01:42 -0800
Subject: [PATCH 03/64] More changes

---
 .../onebrc/CalculateAverage_vaidhy.java       | 318 +++++++++++++-----
 1 file changed, 232 insertions(+), 86 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 7b4fb3dac..572efa3bd 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -15,23 +15,32 @@
  */
 package dev.morling.onebrc;
 
-import java.io.BufferedReader;
-import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.Reader;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.WritableByteChannel;
 import java.nio.charset.StandardCharsets;
-import java.util.IntSummaryStatistics;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.TreeMap;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.*;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.ForkJoinTask;
 import java.util.function.Consumer;
+import java.util.function.Function;
+import java.util.function.Supplier;
 
-public class CalculateAverage_vaidhy {
+public class CalculateAverage_vaidhy<T> {
+
+    private static final String FILE = "./measurements.txt";
 
     private final FileService fileService;
-    private final Consumer<String> lineConsumer;
+    private final Supplier<MapReduce<T>> chunkProcessCreator;
+    private final Function<List<T>, T> reducer;
+
+    interface MapReduce<T> extends Consumer<EfficientString> {
+        T result();
+    }
 
     interface FileService {
         /**
@@ -40,7 +49,7 @@ interface FileService {
          */
         // Possible implementation for byte case in HTTP:
         // byte size = Content-Length header. using HEAD or empty Range.
-        int length();
+        long length();
 
         /**
          * Returns substring of the file from character indices.
@@ -50,14 +59,15 @@ interface FileService {
         // Possible implementation for byte case in HTTP:
         // Using Http Request header "Range", typically used for continuing
         // partial downloads.
-        byte[] range(int offset, int length);
-
+        byte[] range(long offset, int length);
     }
 
     public CalculateAverage_vaidhy(FileService fileService,
-                                   Consumer<String> lineConsumer) {
+                                   Supplier<MapReduce<T>> mapReduce,
+                                   Function<List<T>, T> reducer) {
         this.fileService = fileService;
-        this.lineConsumer = lineConsumer;
+        this.chunkProcessCreator = mapReduce;
+        this.reducer = reducer;
     }
 
     /// SAMPLE CANDIDATE CODE STARTS
@@ -71,16 +81,16 @@ public CalculateAverage_vaidhy(FileService fileService,
      * made.
      */
     // Space complexity = O(scanSize)
-    static class CharStream implements Iterator<Character> {
+    static class ByteStream {
 
         private final FileService fileService;
-        private int offset;
-        private final int scanSize;
+        private long offset;
+        private final long scanSize;
         private int index = 0;
-        private String currentChunk = "";
-        private final int fileLength;
+        private byte[] currentChunk = new byte[0];
+        private final long fileLength;
 
-        public CharStream(FileService fileService, int offset, int scanSize) {
+        public ByteStream(FileService fileService, long offset, int scanSize) {
             this.fileService = fileService;
             this.offset = offset;
             this.scanSize = scanSize;
@@ -93,11 +103,10 @@ public CharStream(FileService fileService, int offset, int scanSize) {
             }
         }
 
-        @Override
         public boolean hasNext() {
-            while (index >= currentChunk.length()) {
+            while (index >= currentChunk.length) {
                 if (offset < fileLength) {
-                    int scanWindow = Math.min(offset + scanSize, fileLength) - offset;
+                    int scanWindow = (int) (Math.min(offset + scanSize, fileLength) - offset);
                     currentChunk = fileService.range(offset, scanWindow);
                     offset += scanWindow;
                     index = 0;
@@ -109,10 +118,9 @@ public boolean hasNext() {
             return true;
         }
 
-        @Override
-        public Character next() {
+        public byte next() {
             if (hasNext()) {
-                char ch = currentChunk.charAt(index);
+                byte ch = currentChunk[index];
                 index++;
                 return ch;
             }
@@ -129,35 +137,36 @@ public Character next() {
     // Space complexity: O(max line length) in next() call, structure is O(1)
     // not counting charStream as it is only a reference, we will count that
     // in worker space.
-    static class LineStream implements Iterator<String> {
-        private final Iterator<Character> charStream;
+    static class LineStream implements Iterator<EfficientString> {
+        private final ByteStream byteStream;
         private int readIndex;
-        private final int length;
+        private final long length;
 
-        public LineStream(Iterator<Character> charStream, int length) {
-            this.charStream = charStream;
+        public LineStream(ByteStream byteStream, long length) {
+            this.byteStream = byteStream;
             this.readIndex = 0;
             this.length = length;
         }
 
         @Override
         public boolean hasNext() {
-            return readIndex <= length && charStream.hasNext();
+            return readIndex <= length && byteStream.hasNext();
         }
 
         @Override
-        public String next() {
+        public EfficientString next() {
             if (hasNext()) {
-                StringBuilder builder = new StringBuilder();
-                while (charStream.hasNext()) {
-                    char ch = charStream.next();
+                byte[] line = new byte[128];
+                int i = 0;
+                while (byteStream.hasNext()) {
+                    byte ch = byteStream.next();
                     readIndex++;
-                    if (ch == '\n') {
+                    if (ch == 0x0a) {
                         break;
                     }
-                    builder.append(ch);
+                    line[i++] = ch;
                 }
-                return builder.toString();
+                return new EfficientString(line, i);
             }
             else {
                 throw new NoSuchElementException();
@@ -166,9 +175,9 @@ public String next() {
     }
 
     // Space complexity: O(scanSize) + O(max line length)
-    public void worker(int offset, int chunkSize, int scanSize) {
-        Iterator<Character> charStream = new CharStream(fileService, offset, scanSize);
-        Iterator<String> lineStream = new LineStream(charStream, chunkSize);
+    public void worker(long offset, long chunkSize, int scanSize, Consumer<EfficientString> lineConsumer) {
+        ByteStream byteStream = new ByteStream(fileService, offset, scanSize);
+        Iterator<EfficientString> lineStream = new LineStream(byteStream, chunkSize);
 
         if (offset != 0) {
             if (lineStream.hasNext()) {
@@ -187,82 +196,219 @@ public void worker(int offset, int chunkSize, int scanSize) {
 
     // Space complexity: O(number of workers), not counting
     // workers space assuming they are running in different hosts.
-    public void master(int chunkSize, int scanSize) {
-        int len = fileService.length();
-        for (int offset = 0; offset < len; offset += chunkSize) {
-            int workerLength = Math.min(len, offset + chunkSize) - offset;
-            worker(offset, workerLength, scanSize);
+    public T master(long chunkSize, int scanSize) {
+        long len = fileService.length();
+        List<ForkJoinTask<T>> summaries = new ArrayList<>();
+        ForkJoinPool commonPool = ForkJoinPool.commonPool();
+
+        for (long offset = 0; offset < len; offset += chunkSize) {
+            long workerLength = Math.min(len, offset + chunkSize) - offset;
+            System.out.println("Worker length: " + workerLength);
+            MapReduce<T> mr = chunkProcessCreator.get();
+            final long transferOffset = offset;
+            ForkJoinTask<T> task = commonPool.submit(() -> {
+                worker(transferOffset, workerLength, scanSize, mr);
+                return mr.result();
+            });
+            summaries.add(task);
         }
+        List<T> summariesDone = summaries.stream()
+                .map(task -> {
+                    try {
+                        return task.get();
+                    }
+                    catch (InterruptedException | ExecutionException e) {
+                        throw new RuntimeException(e);
+                    }
+                })
+                .toList();
+        return reducer.apply(summariesDone);
     }
 
     /// SAMPLE CANDIDATE CODE ENDS
 
-    static class MockFileService implements FileService {
-        private final String contents;
+    static class DiskFileService implements FileService {
+
+        private final FileChannel fileChannel;
 
-        public MockFileService(String contents) {
-            this.contents = contents;
+        DiskFileService(String fileName) throws IOException {
+            this.fileChannel = FileChannel.open(Path.of(fileName),
+                    StandardOpenOption.READ);
         }
 
-        // Provided by the file hosting service:
         @Override
-        public int length() {
-            // Mock implementation:
-            return contents.length();
+        public long length() {
+            try {
+                return this.fileChannel.size();
+            }
+            catch (IOException e) {
+                throw new RuntimeException(e);
+            }
         }
 
-        public String range(int offset, int length) {
-            // Mock implementation
-            return contents.substring(offset, offset + length);
+        @Override
+        public byte[] range(long offset, int length) {
+            byte[] newArr = new byte[length];
+            ByteBuffer outputBuffer = ByteBuffer.wrap(newArr);
+            try {
+                fileChannel.transferTo(offset, length, new WritableByteChannel() {
+                    @Override
+                    public int write(ByteBuffer src) {
+                        int rem = src.remaining();
+                        outputBuffer.put(src);
+                        return rem;
+                    }
+
+                    @Override
+                    public boolean isOpen() {
+                        return true;
+                    }
+
+                    @Override
+                    public void close() {
+                    }
+                });
+            }
+            catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+            return newArr;
         }
     }
 
-    private static final String FILE = "./measurements_1M.txt";
-
-    public interface ChunkProcessor {
+    record EfficientString(byte[] arr, int length) {
 
-        void process(byte[] line);
+        @Override
+        public int hashCode() {
+            int h = 0;
+            for (int i = 0; i < length; i++) {
+                h = (h * 32) + arr[i];
+            }
+            return h;
+        }
 
-        Map<String, IntSummaryStatistics> summary();
+        @Override
+        public boolean equals(Object o) {
+            if (o == this) {
+                return true;
+            }
+            EfficientString eso = (EfficientString) o;
+            if (eso.length != this.length) {
+                return false;
+            }
+            return Arrays.equals(this.arr, 0, this.length,
+                    eso.arr, 0, eso.length);
+        }
     }
 
-    public static class ChunkProcessorImpl implements ChunkProcessor {
+    private static final EfficientString EMPTY = new EfficientString(new byte[0], 0);
+
+    public static class ChunkProcessorImpl implements MapReduce<Map<EfficientString, IntSummaryStatistics>> {
 
-        private final Map<String, IntSummaryStatistics> statistics = new TreeMap<>();
+        private final Map<EfficientString, IntSummaryStatistics> statistics = new HashMap<>(10000);
 
         @Override
-        public void process(byte[] line) {
-            String lineStr = new String(line, StandardCharsets.UTF_8);
-            int indexSemi = lineStr.indexOf(';');
-            String station = lineStr.substring(0, indexSemi);
-            String value = lineStr.substring(indexSemi + 1);
-            double val = Double.parseDouble(value);
-            int normalized = (int) (val * 10);
-            statistics.computeIfAbsent(station, (ignore) -> new IntSummaryStatistics());
-
-            statistics.get(station).accept(normalized);
+        public void accept(EfficientString line) {
+            EfficientString station = EMPTY;
+
+            int i;
+            for (i = 0; i < line.length; i++) {
+                if (line.arr[i] == ';') {
+                    station = new EfficientString(line.arr, i);
+                    break;
+                }
+            }
+
+            int normalized = parseDoubleNew(line.arr, i + 1, line.length);
+
+            IntSummaryStatistics stats = statistics.get(station);
+            if (stats == null) {
+                stats = new IntSummaryStatistics();
+                statistics.put(station, stats);
+            }
+            stats.accept(normalized);
+        }
+
+        private static int parseDoubleNew(byte[] value, int offset, int length) {
+            int normalized = 0;
+            int index = offset;
+            boolean sign = true;
+            if (value[index] == '-') {
+                index++;
+                sign = false;
+            }
+            // boolean hasDot = false;
+            for (; index < length; index++) {
+                byte ch = value[index];
+                if (ch != '.') {
+                    normalized = normalized * 10 + (ch - '0');
+                }
+                // else {
+                // hasDot = true;
+                // }
+            }
+            // if (!hasDot) {
+            // normalized *= 10;
+            // }
+            if (!sign) {
+                normalized = -normalized;
+            }
+            return normalized;
         }
 
         @Override
-        public Map<String, IntSummaryStatistics> summary() {
+        public Map<EfficientString, IntSummaryStatistics> result() {
             return statistics;
         }
     }
 
     public static void main(String[] args) throws IOException {
+        DiskFileService diskFileService = new DiskFileService(FILE);
+
+        CalculateAverage_vaidhy<Map<EfficientString, IntSummaryStatistics>> calculateAverageVaidhy = new CalculateAverage_vaidhy<>(
+                diskFileService,
+                ChunkProcessorImpl::new,
+                CalculateAverage_vaidhy::combineOutputs);
+
+        int proc = ForkJoinPool.commonPool().getParallelism();
+        long fileSize = diskFileService.length();
+        long chunkSize = Math.ceilDiv(fileSize, proc);
+        Map<EfficientString, IntSummaryStatistics> output = calculateAverageVaidhy.master(chunkSize,
+                Math.min(10 * 1024 * 1024, (int) chunkSize));
+        Map<String, String> outputStr = toPrintMap(output);
+        System.out.println(outputStr);
+    }
 
-        ChunkProcessor chunkProcessor = new ChunkProcessorImpl();
+    private static Map<String, String> toPrintMap(Map<EfficientString, IntSummaryStatistics> output) {
 
-        try (FileInputStream fis = new FileInputStream(FILE)) {
+        Map<String, String> outputStr = new TreeMap<>();
+        for (Map.Entry<EfficientString, IntSummaryStatistics> entry : output.entrySet()) {
+            IntSummaryStatistics stat = entry.getValue();
+            outputStr.put(new String(
+                    Arrays.copyOf(entry.getKey().arr, entry.getKey().length), StandardCharsets.UTF_8),
+                    (stat.getMin() / 10.0) + "/" +
+                            (Math.round(stat.getAverage()) / 10.0) + "/" +
+                            (stat.getMax() / 10.0));
+        }
+        return outputStr;
+    }
 
-            Reader io = new InputStreamReader(fis, StandardCharsets.UTF_8);
-            BufferedReader bufferedReader = new BufferedReader(io);
-            String line;
-            while ((line = bufferedReader.readLine()) != null) {
-                chunkProcessor.process(line.getBytes(StandardCharsets.UTF_8));
+    private static Map<EfficientString, IntSummaryStatistics> combineOutputs(List<Map<EfficientString, IntSummaryStatistics>> list) {
+        Map<EfficientString, IntSummaryStatistics> output = new HashMap<>(10000);
+        for (Map<EfficientString, IntSummaryStatistics> map : list) {
+            for (Map.Entry<EfficientString, IntSummaryStatistics> entry : map.entrySet()) {
+                output.compute(entry.getKey(), (ignore, val) -> {
+                    if (val == null) {
+                        return entry.getValue();
+                    }
+                    else {
+                        val.combine(entry.getValue());
+                        return val;
+                    }
+                });
             }
         }
 
-        System.out.println(chunkProcessor.summary());
+        return output;
     }
 }

From 812f81f41c60b90440936207c69ac8774646a63f Mon Sep 17 00:00:00 2001
From: Anita S V <anitasvasu@gmail.com>
Date: Tue, 9 Jan 2024 00:02:42 -0800
Subject: [PATCH 04/64] remove worker log

---
 src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 572efa3bd..4c1c942d5 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -203,7 +203,6 @@ public T master(long chunkSize, int scanSize) {
 
         for (long offset = 0; offset < len; offset += chunkSize) {
             long workerLength = Math.min(len, offset + chunkSize) - offset;
-            System.out.println("Worker length: " + workerLength);
             MapReduce<T> mr = chunkProcessCreator.get();
             final long transferOffset = offset;
             ForkJoinTask<T> task = commonPool.submit(() -> {

From 4646f782fc01605c76eea8734929ed8f9a1953f9 Mon Sep 17 00:00:00 2001
From: Anita S V <anitasvasu@gmail.com>
Date: Tue, 9 Jan 2024 01:12:24 -0800
Subject: [PATCH 05/64] Pass -Dparellelism and switch back to open

---
 calculate_average_vaidhy.sh                                   | 2 +-
 src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/calculate_average_vaidhy.sh b/calculate_average_vaidhy.sh
index 29494bd63..fdb4b531c 100755
--- a/calculate_average_vaidhy.sh
+++ b/calculate_average_vaidhy.sh
@@ -18,5 +18,5 @@
 source "$HOME/.sdkman/bin/sdkman-init.sh"
 sdk use java 21.0.1-open 1>&2
 
-JAVA_OPTS="--enable-preview"
+JAVA_OPTS="--enable-preview -Djava.util.concurrent.ForkJoinPool.common.parallelism=8"
 time java $JAVA_OPTS --class-path target/average-1.0.0-SNAPSHOT.jar dev.morling.onebrc.CalculateAverage_vaidhy
diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 4c1c942d5..377db6797 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -369,7 +369,7 @@ public static void main(String[] args) throws IOException {
                 ChunkProcessorImpl::new,
                 CalculateAverage_vaidhy::combineOutputs);
 
-        int proc = ForkJoinPool.commonPool().getParallelism();
+        int proc = 2 * ForkJoinPool.commonPool().getParallelism();
         long fileSize = diskFileService.length();
         long chunkSize = Math.ceilDiv(fileSize, proc);
         Map<EfficientString, IntSummaryStatistics> output = calculateAverageVaidhy.master(chunkSize,

From 64a11020966088702057c7d40f91340d0cff431d Mon Sep 17 00:00:00 2001
From: Anita S V <anitasvasu@gmail.com>
Date: Tue, 9 Jan 2024 01:34:05 -0800
Subject: [PATCH 06/64] Try out mmap

---
 .../onebrc/CalculateAverage_vaidhy.java       | 116 ++++++------------
 1 file changed, 35 insertions(+), 81 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 377db6797..9eb093bb3 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -15,10 +15,12 @@
  */
 package dev.morling.onebrc;
 
+import sun.misc.Unsafe;
+
 import java.io.IOException;
-import java.nio.ByteBuffer;
+import java.lang.foreign.Arena;
+import java.lang.reflect.Field;
 import java.nio.channels.FileChannel;
-import java.nio.channels.WritableByteChannel;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
@@ -34,6 +36,19 @@ public class CalculateAverage_vaidhy<T> {
 
     private static final String FILE = "./measurements.txt";
 
+    private static final Unsafe UNSAFE = initUnsafe();
+
+    private static Unsafe initUnsafe() {
+        try {
+            Field theUnsafe = Unsafe.class.getDeclaredField("theUnsafe");
+            theUnsafe.setAccessible(true);
+            return (Unsafe) theUnsafe.get(Unsafe.class);
+        }
+        catch (NoSuchFieldException | IllegalAccessException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     private final FileService fileService;
     private final Supplier<MapReduce<T>> chunkProcessCreator;
     private final Function<List<T>, T> reducer;
@@ -43,23 +58,9 @@ interface MapReduce<T> extends Consumer<EfficientString> {
     }
 
     interface FileService {
-        /**
-         * Returns the size of the file in number of characters.
-         * (Extra credit: assume byte size instead of char size)
-         */
-        // Possible implementation for byte case in HTTP:
-        // byte size = Content-Length header. using HEAD or empty Range.
         long length();
 
-        /**
-         * Returns substring of the file from character indices.
-         * Expects 0 <= start <= start + length <= fileSize
-         * (Extra credit: assume sub-byte array instead of sub char array)
-         */
-        // Possible implementation for byte case in HTTP:
-        // Using Http Request header "Range", typically used for continuing
-        // partial downloads.
-        byte[] range(long offset, int length);
+        byte getByte(long position);
     }
 
     public CalculateAverage_vaidhy(FileService fileService,
@@ -84,49 +85,24 @@ public CalculateAverage_vaidhy(FileService fileService,
     static class ByteStream {
 
         private final FileService fileService;
-        private long offset;
-        private final long scanSize;
-        private int index = 0;
-        private byte[] currentChunk = new byte[0];
-        private final long fileLength;
+        private long position;
+        private long fileLength;
 
-        public ByteStream(FileService fileService, long offset, int scanSize) {
+        public ByteStream(FileService fileService, long offset) {
             this.fileService = fileService;
-            this.offset = offset;
-            this.scanSize = scanSize;
             this.fileLength = fileService.length();
-            if (scanSize <= 0) {
-                throw new IllegalArgumentException("scan size must be > 0");
-            }
+            this.position = offset;
             if (offset < 0) {
                 throw new IllegalArgumentException("offset must be >= 0");
             }
         }
 
         public boolean hasNext() {
-            while (index >= currentChunk.length) {
-                if (offset < fileLength) {
-                    int scanWindow = (int) (Math.min(offset + scanSize, fileLength) - offset);
-                    currentChunk = fileService.range(offset, scanWindow);
-                    offset += scanWindow;
-                    index = 0;
-                }
-                else {
-                    return false;
-                }
-            }
-            return true;
+            return position < fileLength;
         }
 
         public byte next() {
-            if (hasNext()) {
-                byte ch = currentChunk[index];
-                index++;
-                return ch;
-            }
-            else {
-                throw new NoSuchElementException();
-            }
+            return fileService.getByte(position++);
         }
     }
 
@@ -175,8 +151,8 @@ public EfficientString next() {
     }
 
     // Space complexity: O(scanSize) + O(max line length)
-    public void worker(long offset, long chunkSize, int scanSize, Consumer<EfficientString> lineConsumer) {
-        ByteStream byteStream = new ByteStream(fileService, offset, scanSize);
+    public void worker(long offset, long chunkSize, Consumer<EfficientString> lineConsumer) {
+        ByteStream byteStream = new ByteStream(fileService, offset);
         Iterator<EfficientString> lineStream = new LineStream(byteStream, chunkSize);
 
         if (offset != 0) {
@@ -196,7 +172,7 @@ public void worker(long offset, long chunkSize, int scanSize, Consumer<Efficient
 
     // Space complexity: O(number of workers), not counting
     // workers space assuming they are running in different hosts.
-    public T master(long chunkSize, int scanSize) {
+    public T master(long chunkSize) {
         long len = fileService.length();
         List<ForkJoinTask<T>> summaries = new ArrayList<>();
         ForkJoinPool commonPool = ForkJoinPool.commonPool();
@@ -206,7 +182,7 @@ public T master(long chunkSize, int scanSize) {
             MapReduce<T> mr = chunkProcessCreator.get();
             final long transferOffset = offset;
             ForkJoinTask<T> task = commonPool.submit(() -> {
-                worker(transferOffset, workerLength, scanSize, mr);
+                worker(transferOffset, workerLength, mr);
                 return mr.result();
             });
             summaries.add(task);
@@ -229,10 +205,13 @@ public T master(long chunkSize, int scanSize) {
     static class DiskFileService implements FileService {
 
         private final FileChannel fileChannel;
+        private final long mappedAddress;
 
         DiskFileService(String fileName) throws IOException {
             this.fileChannel = FileChannel.open(Path.of(fileName),
                     StandardOpenOption.READ);
+            this.mappedAddress = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0,
+                    fileChannel.size(), Arena.global()).address();
         }
 
         @Override
@@ -246,32 +225,8 @@ public long length() {
         }
 
         @Override
-        public byte[] range(long offset, int length) {
-            byte[] newArr = new byte[length];
-            ByteBuffer outputBuffer = ByteBuffer.wrap(newArr);
-            try {
-                fileChannel.transferTo(offset, length, new WritableByteChannel() {
-                    @Override
-                    public int write(ByteBuffer src) {
-                        int rem = src.remaining();
-                        outputBuffer.put(src);
-                        return rem;
-                    }
-
-                    @Override
-                    public boolean isOpen() {
-                        return true;
-                    }
-
-                    @Override
-                    public void close() {
-                    }
-                });
-            }
-            catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-            return newArr;
+        public byte getByte(long position) {
+            return UNSAFE.getByte(mappedAddress + position);
         }
     }
 
@@ -369,11 +324,10 @@ public static void main(String[] args) throws IOException {
                 ChunkProcessorImpl::new,
                 CalculateAverage_vaidhy::combineOutputs);
 
-        int proc = 2 * ForkJoinPool.commonPool().getParallelism();
+        int proc = 16 * ForkJoinPool.commonPool().getParallelism();
         long fileSize = diskFileService.length();
         long chunkSize = Math.ceilDiv(fileSize, proc);
-        Map<EfficientString, IntSummaryStatistics> output = calculateAverageVaidhy.master(chunkSize,
-                Math.min(10 * 1024 * 1024, (int) chunkSize));
+        Map<EfficientString, IntSummaryStatistics> output = calculateAverageVaidhy.master(chunkSize);
         Map<String, String> outputStr = toPrintMap(output);
         System.out.println(outputStr);
     }

From 528c9af23931850c99d1b3db73dacc567bfc7cc3 Mon Sep 17 00:00:00 2001
From: Anita S V <anitasvasu@gmail.com>
Date: Tue, 9 Jan 2024 10:05:54 -0800
Subject: [PATCH 07/64] Improve mmap solution

---
 .../onebrc/CalculateAverage_vaidhy.java       | 60 ++++++++++---------
 1 file changed, 33 insertions(+), 27 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 9eb093bb3..3b2b112d2 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 import java.lang.foreign.Arena;
+import java.lang.foreign.MemorySegment;
 import java.lang.reflect.Field;
 import java.nio.channels.FileChannel;
 import java.nio.charset.StandardCharsets;
@@ -131,22 +132,17 @@ public boolean hasNext() {
 
         @Override
         public EfficientString next() {
-            if (hasNext()) {
-                byte[] line = new byte[128];
-                int i = 0;
-                while (byteStream.hasNext()) {
-                    byte ch = byteStream.next();
-                    readIndex++;
-                    if (ch == 0x0a) {
-                        break;
-                    }
-                    line[i++] = ch;
+            byte[] line = new byte[128];
+            int i = 0;
+            while (byteStream.hasNext()) {
+                byte ch = byteStream.next();
+                readIndex++;
+                if (ch == 0x0a) {
+                    break;
                 }
-                return new EfficientString(line, i);
-            }
-            else {
-                throw new NoSuchElementException();
+                line[i++] = ch;
             }
+            return new EfficientString(line, i);
         }
     }
 
@@ -250,8 +246,12 @@ public boolean equals(Object o) {
             if (eso.length != this.length) {
                 return false;
             }
-            return Arrays.equals(this.arr, 0, this.length,
-                    eso.arr, 0, eso.length);
+            for (int i = 0; i < length; i++) {
+                if (arr[i] != eso.arr[i]) {
+                    return false;
+                }
+            }
+            return true;
         }
     }
 
@@ -263,18 +263,15 @@ public static class ChunkProcessorImpl implements MapReduce<Map<EfficientString,
 
         @Override
         public void accept(EfficientString line) {
-            EfficientString station = EMPTY;
+            EfficientString station = getStation(line);
 
-            int i;
-            for (i = 0; i < line.length; i++) {
-                if (line.arr[i] == ';') {
-                    station = new EfficientString(line.arr, i);
-                    break;
-                }
-            }
+            int normalized = parseDouble(line.arr,
+                    station.length + 1, line.length);
 
-            int normalized = parseDoubleNew(line.arr, i + 1, line.length);
+            updateStats(station, normalized);
+        }
 
+        private void updateStats(EfficientString station, int normalized) {
             IntSummaryStatistics stats = statistics.get(station);
             if (stats == null) {
                 stats = new IntSummaryStatistics();
@@ -283,7 +280,16 @@ public void accept(EfficientString line) {
             stats.accept(normalized);
         }
 
-        private static int parseDoubleNew(byte[] value, int offset, int length) {
+        private static EfficientString getStation(EfficientString line) {
+            for (int i = 0; i < line.length; i++) {
+                if (line.arr[i] == ';') {
+                    return new EfficientString(line.arr, i);
+                }
+            }
+            return EMPTY;
+        }
+
+        private static int parseDouble(byte[] value, int offset, int length) {
             int normalized = 0;
             int index = offset;
             boolean sign = true;
@@ -324,7 +330,7 @@ public static void main(String[] args) throws IOException {
                 ChunkProcessorImpl::new,
                 CalculateAverage_vaidhy::combineOutputs);
 
-        int proc = 16 * ForkJoinPool.commonPool().getParallelism();
+        int proc = ForkJoinPool.commonPool().getParallelism();
         long fileSize = diskFileService.length();
         long chunkSize = Math.ceilDiv(fileSize, proc);
         Map<EfficientString, IntSummaryStatistics> output = calculateAverageVaidhy.master(chunkSize);

From 06588aebec1feb67117ca2552b2d2e0a911c9d1a Mon Sep 17 00:00:00 2001
From: Anita S V <anitasvasu@gmail.com>
Date: Wed, 10 Jan 2024 21:46:58 -0800
Subject: [PATCH 08/64] no copy version

---
 .../onebrc/CalculateAverage_vaidhy.java       | 328 ++++++++----------
 1 file changed, 147 insertions(+), 181 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 3b2b112d2..01924dcfc 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -19,7 +19,6 @@
 
 import java.io.IOException;
 import java.lang.foreign.Arena;
-import java.lang.foreign.MemorySegment;
 import java.lang.reflect.Field;
 import java.nio.channels.FileChannel;
 import java.nio.charset.StandardCharsets;
@@ -27,9 +26,9 @@
 import java.nio.file.StandardOpenOption;
 import java.util.*;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ForkJoinPool;
-import java.util.concurrent.ForkJoinTask;
-import java.util.function.Consumer;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.function.Function;
 import java.util.function.Supplier;
 
@@ -37,8 +36,6 @@ public class CalculateAverage_vaidhy<T> {
 
     private static final String FILE = "./measurements.txt";
 
-    private static final Unsafe UNSAFE = initUnsafe();
-
     private static Unsafe initUnsafe() {
         try {
             Field theUnsafe = Unsafe.class.getDeclaredField("theUnsafe");
@@ -50,18 +47,91 @@ private static Unsafe initUnsafe() {
         }
     }
 
-    private final FileService fileService;
-    private final Supplier<MapReduce<T>> chunkProcessCreator;
-    private final Function<List<T>, T> reducer;
+    private static final Unsafe UNSAFE = initUnsafe();
+
+    record ByteSlice(long from, long to) {
+
+        @Override
+        public int hashCode() {
+            int h = 0;
+            for (long i = from; i < to; i++) {
+                h = (h * 31) ^ UNSAFE.getByte(i);
+            }
+            return h;
+        }
+
+        public int length() {
+            return (int) (to - from);
+        }
+
+        public byte get(int i) {
+            return UNSAFE.getByte(from + i);
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (o instanceof ByteSlice bs) {
+                int len = this.length();
+                if (bs.length() != len) {
+                    return false;
+                }
+                for (int i = 0; i < len; i++) {
+                    if (this.get(i) != bs.get(i)) {
+                        return false;
+                    }
+                }
+                return true;
+            } else {
+                return false;
+            }
+        }
+
+        @Override
+        public String toString() {
+            byte[] copy = new byte[this.length()];
+            for (int i = 0; i < copy.length; i++) {
+                copy[i] = get(i);
+            }
+            return new String(copy, StandardCharsets.UTF_8);
+        }
+    }
+
+    private static int parseDouble(ByteSlice slice) {
+        int normalized = 0;
+        boolean sign = true;
+        int index = 0;
+        if (slice.get(index) == '-') {
+            index++;
+            sign = false;
+        }
+        int length = slice.length();
+        for (; index < length; index++) {
+            byte ch = slice.get(index);
+            if (ch != '.') {
+                normalized = normalized * 10 + (ch - '0');
+            }
+        }
+        if (!sign) {
+            normalized = -normalized;
+        }
+        return normalized;
+    }
+
+    interface MapReduce<T> {
+
+        void process(ByteSlice key, ByteSlice value);
 
-    interface MapReduce<T> extends Consumer<EfficientString> {
         T result();
     }
 
+    private final FileService fileService;
+    private final Supplier<MapReduce<T>> chunkProcessCreator;
+    private final Function<List<T>, T> reducer;
+
     interface FileService {
         long length();
 
-        byte getByte(long position);
+        long address();
     }
 
     public CalculateAverage_vaidhy(FileService fileService,
@@ -72,41 +142,6 @@ public CalculateAverage_vaidhy(FileService fileService,
         this.reducer = reducer;
     }
 
-    /// SAMPLE CANDIDATE CODE STARTS
-
-    /**
-     * Reads from a given offset till the end, it calls server in
-     * blocks of scanSize whenever cursor passes the current block.
-     * Typically when hasNext() is called. hasNext() is efficient
-     * in the sense calling second time is cheap if next() is not
-     * called in between. Cheap in the sense no call to server is
-     * made.
-     */
-    // Space complexity = O(scanSize)
-    static class ByteStream {
-
-        private final FileService fileService;
-        private long position;
-        private long fileLength;
-
-        public ByteStream(FileService fileService, long offset) {
-            this.fileService = fileService;
-            this.fileLength = fileService.length();
-            this.position = offset;
-            if (offset < 0) {
-                throw new IllegalArgumentException("offset must be >= 0");
-            }
-        }
-
-        public boolean hasNext() {
-            return position < fileLength;
-        }
-
-        public byte next() {
-            return fileService.getByte(position++);
-        }
-    }
-
     /**
      * Reads lines from a given character stream, hasNext() is always
      * efficient, all work is done only in next().
@@ -114,47 +149,51 @@ public byte next() {
     // Space complexity: O(max line length) in next() call, structure is O(1)
     // not counting charStream as it is only a reference, we will count that
     // in worker space.
-    static class LineStream implements Iterator<EfficientString> {
-        private final ByteStream byteStream;
-        private int readIndex;
-        private final long length;
-
-        public LineStream(ByteStream byteStream, long length) {
-            this.byteStream = byteStream;
-            this.readIndex = 0;
-            this.length = length;
+    static class LineStream {
+        private final long fileEnd;
+        private final long chunkEnd;
+
+        private long position;
+
+        public LineStream(FileService fileService, long offset, long chunkSize) {
+            long fileStart = fileService.address();
+            this.fileEnd = fileStart + fileService.length();
+            this.chunkEnd = fileStart + offset + chunkSize;
+            this.position = fileStart + offset;
         }
 
-        @Override
         public boolean hasNext() {
-            return readIndex <= length && byteStream.hasNext();
+            return position <= chunkEnd && position < fileEnd;
         }
 
-        @Override
-        public EfficientString next() {
-            byte[] line = new byte[128];
-            int i = 0;
-            while (byteStream.hasNext()) {
-                byte ch = byteStream.next();
-                readIndex++;
-                if (ch == 0x0a) {
-                    break;
+        public ByteSlice until(byte ch) {
+            for (long i = position; i < fileEnd; i++) {
+                if (UNSAFE.getByte(i) == ch) {
+                    try {
+                        return new ByteSlice(position, i);
+                    }
+                    finally {
+                        position = i + 1;
+                    }
                 }
-                line[i++] = ch;
             }
-            return new EfficientString(line, i);
+            try {
+                return new ByteSlice(position, fileEnd);
+            }
+            finally {
+                position = fileEnd;
+            }
         }
     }
 
     // Space complexity: O(scanSize) + O(max line length)
-    public void worker(long offset, long chunkSize, Consumer<EfficientString> lineConsumer) {
-        ByteStream byteStream = new ByteStream(fileService, offset);
-        Iterator<EfficientString> lineStream = new LineStream(byteStream, chunkSize);
+    private void worker(long offset, long chunkSize, MapReduce<T> lineConsumer) {
+        LineStream lineStream = new LineStream(fileService, offset, chunkSize);
 
         if (offset != 0) {
             if (lineStream.hasNext()) {
                 // Skip the first line.
-                lineStream.next();
+                lineStream.until((byte) '\n');
             }
             else {
                 // No lines then do nothing.
@@ -162,22 +201,23 @@ public void worker(long offset, long chunkSize, Consumer<EfficientString> lineCo
             }
         }
         while (lineStream.hasNext()) {
-            lineConsumer.accept(lineStream.next());
+            ByteSlice key = lineStream.until((byte) ';');
+            ByteSlice value = lineStream.until((byte) '\n');
+            lineConsumer.process(key, value);
         }
     }
 
     // Space complexity: O(number of workers), not counting
     // workers space assuming they are running in different hosts.
-    public T master(long chunkSize) {
+    public T master(long chunkSize, ExecutorService executor) {
         long len = fileService.length();
-        List<ForkJoinTask<T>> summaries = new ArrayList<>();
-        ForkJoinPool commonPool = ForkJoinPool.commonPool();
+        List<Future<T>> summaries = new ArrayList<>();
 
         for (long offset = 0; offset < len; offset += chunkSize) {
             long workerLength = Math.min(len, offset + chunkSize) - offset;
             MapReduce<T> mr = chunkProcessCreator.get();
             final long transferOffset = offset;
-            ForkJoinTask<T> task = commonPool.submit(() -> {
+            Future<T> task = executor.submit(() -> {
                 worker(transferOffset, workerLength, mr);
                 return mr.result();
             });
@@ -200,124 +240,45 @@ public T master(long chunkSize) {
 
     static class DiskFileService implements FileService {
 
-        private final FileChannel fileChannel;
+        private final long fileSize;
         private final long mappedAddress;
 
         DiskFileService(String fileName) throws IOException {
-            this.fileChannel = FileChannel.open(Path.of(fileName),
+            FileChannel fileChannel = FileChannel.open(Path.of(fileName),
                     StandardOpenOption.READ);
+            this.fileSize = fileChannel.size();
             this.mappedAddress = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0,
-                    fileChannel.size(), Arena.global()).address();
+                    fileSize, Arena.global()).address();
         }
 
         @Override
         public long length() {
-            try {
-                return this.fileChannel.size();
-            }
-            catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        }
-
-        @Override
-        public byte getByte(long position) {
-            return UNSAFE.getByte(mappedAddress + position);
-        }
-    }
-
-    record EfficientString(byte[] arr, int length) {
-
-        @Override
-        public int hashCode() {
-            int h = 0;
-            for (int i = 0; i < length; i++) {
-                h = (h * 32) + arr[i];
-            }
-            return h;
+            return fileSize;
         }
 
         @Override
-        public boolean equals(Object o) {
-            if (o == this) {
-                return true;
-            }
-            EfficientString eso = (EfficientString) o;
-            if (eso.length != this.length) {
-                return false;
-            }
-            for (int i = 0; i < length; i++) {
-                if (arr[i] != eso.arr[i]) {
-                    return false;
-                }
-            }
-            return true;
+        public long address() {
+            return mappedAddress;
         }
     }
 
-    private static final EfficientString EMPTY = new EfficientString(new byte[0], 0);
-
-    public static class ChunkProcessorImpl implements MapReduce<Map<EfficientString, IntSummaryStatistics>> {
+    public static class ChunkProcessorImpl implements MapReduce<Map<ByteSlice, IntSummaryStatistics>> {
 
-        private final Map<EfficientString, IntSummaryStatistics> statistics = new HashMap<>(10000);
+        private final Map<ByteSlice, IntSummaryStatistics> statistics = new HashMap<>(10000);
 
         @Override
-        public void accept(EfficientString line) {
-            EfficientString station = getStation(line);
-
-            int normalized = parseDouble(line.arr,
-                    station.length + 1, line.length);
-
-            updateStats(station, normalized);
-        }
-
-        private void updateStats(EfficientString station, int normalized) {
+        public void process(ByteSlice station, ByteSlice value) {
+            int temperature = parseDouble(value);
             IntSummaryStatistics stats = statistics.get(station);
             if (stats == null) {
                 stats = new IntSummaryStatistics();
                 statistics.put(station, stats);
             }
-            stats.accept(normalized);
-        }
-
-        private static EfficientString getStation(EfficientString line) {
-            for (int i = 0; i < line.length; i++) {
-                if (line.arr[i] == ';') {
-                    return new EfficientString(line.arr, i);
-                }
-            }
-            return EMPTY;
-        }
-
-        private static int parseDouble(byte[] value, int offset, int length) {
-            int normalized = 0;
-            int index = offset;
-            boolean sign = true;
-            if (value[index] == '-') {
-                index++;
-                sign = false;
-            }
-            // boolean hasDot = false;
-            for (; index < length; index++) {
-                byte ch = value[index];
-                if (ch != '.') {
-                    normalized = normalized * 10 + (ch - '0');
-                }
-                // else {
-                // hasDot = true;
-                // }
-            }
-            // if (!hasDot) {
-            // normalized *= 10;
-            // }
-            if (!sign) {
-                normalized = -normalized;
-            }
-            return normalized;
+            stats.accept(temperature);
         }
 
         @Override
-        public Map<EfficientString, IntSummaryStatistics> result() {
+        public Map<ByteSlice, IntSummaryStatistics> result() {
             return statistics;
         }
     }
@@ -325,26 +286,31 @@ public Map<EfficientString, IntSummaryStatistics> result() {
     public static void main(String[] args) throws IOException {
         DiskFileService diskFileService = new DiskFileService(FILE);
 
-        CalculateAverage_vaidhy<Map<EfficientString, IntSummaryStatistics>> calculateAverageVaidhy = new CalculateAverage_vaidhy<>(
+        CalculateAverage_vaidhy<Map<ByteSlice, IntSummaryStatistics>> calculateAverageVaidhy = new CalculateAverage_vaidhy<>(
                 diskFileService,
                 ChunkProcessorImpl::new,
                 CalculateAverage_vaidhy::combineOutputs);
 
-        int proc = ForkJoinPool.commonPool().getParallelism();
+        int proc = 4 * Runtime.getRuntime().availableProcessors();
+
+        int shards = 4 * proc;
         long fileSize = diskFileService.length();
-        long chunkSize = Math.ceilDiv(fileSize, proc);
-        Map<EfficientString, IntSummaryStatistics> output = calculateAverageVaidhy.master(chunkSize);
+        long chunkSize = Math.ceilDiv(fileSize, shards);
+
+        ExecutorService executor = Executors.newFixedThreadPool(proc);
+        Map<ByteSlice, IntSummaryStatistics> output = calculateAverageVaidhy.master(chunkSize, executor);
+        executor.shutdown();
+
         Map<String, String> outputStr = toPrintMap(output);
         System.out.println(outputStr);
     }
 
-    private static Map<String, String> toPrintMap(Map<EfficientString, IntSummaryStatistics> output) {
+    private static Map<String, String> toPrintMap(Map<ByteSlice, IntSummaryStatistics> output) {
 
         Map<String, String> outputStr = new TreeMap<>();
-        for (Map.Entry<EfficientString, IntSummaryStatistics> entry : output.entrySet()) {
+        for (Map.Entry<ByteSlice, IntSummaryStatistics> entry : output.entrySet()) {
             IntSummaryStatistics stat = entry.getValue();
-            outputStr.put(new String(
-                    Arrays.copyOf(entry.getKey().arr, entry.getKey().length), StandardCharsets.UTF_8),
+            outputStr.put(entry.getKey().toString(),
                     (stat.getMin() / 10.0) + "/" +
                             (Math.round(stat.getAverage()) / 10.0) + "/" +
                             (stat.getMax() / 10.0));
@@ -352,10 +318,10 @@ private static Map<String, String> toPrintMap(Map<EfficientString, IntSummarySta
         return outputStr;
     }
 
-    private static Map<EfficientString, IntSummaryStatistics> combineOutputs(List<Map<EfficientString, IntSummaryStatistics>> list) {
-        Map<EfficientString, IntSummaryStatistics> output = new HashMap<>(10000);
-        for (Map<EfficientString, IntSummaryStatistics> map : list) {
-            for (Map.Entry<EfficientString, IntSummaryStatistics> entry : map.entrySet()) {
+    private static Map<ByteSlice, IntSummaryStatistics> combineOutputs(List<Map<ByteSlice, IntSummaryStatistics>> list) {
+        Map<ByteSlice, IntSummaryStatistics> output = new HashMap<>(10000);
+        for (Map<ByteSlice, IntSummaryStatistics> map : list) {
+            for (Map.Entry<ByteSlice, IntSummaryStatistics> entry : map.entrySet()) {
                 output.compute(entry.getKey(), (ignore, val) -> {
                     if (val == null) {
                         return entry.getValue();

From 56c08bf2ec4223f4c0974d2c3debaa8481e70724 Mon Sep 17 00:00:00 2001
From: Anita S V <anitasvasu@gmail.com>
Date: Wed, 10 Jan 2024 21:56:07 -0800
Subject: [PATCH 09/64] reduce threads

---
 .../java/dev/morling/onebrc/CalculateAverage_vaidhy.java  | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 01924dcfc..685f37560 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -291,9 +291,9 @@ public static void main(String[] args) throws IOException {
                 ChunkProcessorImpl::new,
                 CalculateAverage_vaidhy::combineOutputs);
 
-        int proc = 4 * Runtime.getRuntime().availableProcessors();
+        int proc = 2 * Runtime.getRuntime().availableProcessors();
 
-        int shards = 4 * proc;
+        int shards = proc;
         long fileSize = diskFileService.length();
         long chunkSize = Math.ceilDiv(fileSize, shards);
 
@@ -311,9 +311,7 @@ private static Map<String, String> toPrintMap(Map<ByteSlice, IntSummaryStatistic
         for (Map.Entry<ByteSlice, IntSummaryStatistics> entry : output.entrySet()) {
             IntSummaryStatistics stat = entry.getValue();
             outputStr.put(entry.getKey().toString(),
-                    (stat.getMin() / 10.0) + "/" +
-                            (Math.round(stat.getAverage()) / 10.0) + "/" +
-                            (stat.getMax() / 10.0));
+                    STR."\{stat.getMin() / 10.0}/\{Math.round(stat.getAverage()) / 10.0}/\{stat.getMax() / 10.0}");
         }
         return outputStr;
     }

From a6a030ddb65ecd6936feb5bbe060d667650e9aa7 Mon Sep 17 00:00:00 2001
From: Vaidhy Gopalan <vaidhy@gmail.com>
Date: Thu, 11 Jan 2024 14:48:47 -0800
Subject: [PATCH 10/64] hash code computed on the fly

---
 .../onebrc/CalculateAverage_vaidhy.java       | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 685f37560..b6e6014f0 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -49,10 +49,11 @@ private static Unsafe initUnsafe() {
 
     private static final Unsafe UNSAFE = initUnsafe();
 
-    record ByteSlice(long from, long to) {
+    record ByteSlice(long from, long to, int hCode) {
 
         @Override
         public int hashCode() {
+            if (hCode != 0) { return hCode; }
             int h = 0;
             for (long i = from; i < to; i++) {
                 h = (h * 31) ^ UNSAFE.getByte(i);
@@ -166,11 +167,15 @@ public boolean hasNext() {
             return position <= chunkEnd && position < fileEnd;
         }
 
-        public ByteSlice until(byte ch) {
+        public ByteSlice until(byte ch, boolean computeHash) {
+            int h = 0;
             for (long i = position; i < fileEnd; i++) {
+                if (computeHash) {
+                    h = (h * 31) ^ UNSAFE.getByte(i);
+                }
                 if (UNSAFE.getByte(i) == ch) {
                     try {
-                        return new ByteSlice(position, i);
+                        return new ByteSlice(position, i, h);
                     }
                     finally {
                         position = i + 1;
@@ -178,7 +183,7 @@ public ByteSlice until(byte ch) {
                 }
             }
             try {
-                return new ByteSlice(position, fileEnd);
+                return new ByteSlice(position, fileEnd, h);
             }
             finally {
                 position = fileEnd;
@@ -193,7 +198,7 @@ private void worker(long offset, long chunkSize, MapReduce<T> lineConsumer) {
         if (offset != 0) {
             if (lineStream.hasNext()) {
                 // Skip the first line.
-                lineStream.until((byte) '\n');
+                lineStream.until((byte) '\n', false);
             }
             else {
                 // No lines then do nothing.
@@ -201,8 +206,8 @@ private void worker(long offset, long chunkSize, MapReduce<T> lineConsumer) {
             }
         }
         while (lineStream.hasNext()) {
-            ByteSlice key = lineStream.until((byte) ';');
-            ByteSlice value = lineStream.until((byte) '\n');
+            ByteSlice key = lineStream.until((byte) ';', true);
+            ByteSlice value = lineStream.until((byte) '\n', false);
             lineConsumer.process(key, value);
         }
     }

From af61635727314431c376edd787da92fd3fd9c5fe Mon Sep 17 00:00:00 2001
From: Vaidhy Gopalan <vaidhy@gmail.com>
Date: Thu, 11 Jan 2024 14:59:20 -0800
Subject: [PATCH 11/64] Reuse the char (Do not know if it helps)

---
 .../dev/morling/onebrc/CalculateAverage_vaidhy.java    | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index b6e6014f0..49c128b84 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -169,11 +169,9 @@ public boolean hasNext() {
 
         public ByteSlice until(byte ch, boolean computeHash) {
             int h = 0;
+            byte inCh;
             for (long i = position; i < fileEnd; i++) {
-                if (computeHash) {
-                    h = (h * 31) ^ UNSAFE.getByte(i);
-                }
-                if (UNSAFE.getByte(i) == ch) {
+                if ((inCh = UNSAFE.getByte(i)) == ch) {
                     try {
                         return new ByteSlice(position, i, h);
                     }
@@ -181,7 +179,11 @@ public ByteSlice until(byte ch, boolean computeHash) {
                         position = i + 1;
                     }
                 }
+                if (computeHash) {
+                    h = (h * 31) ^ inCh;
+                }
             }
+
             try {
                 return new ByteSlice(position, fileEnd, h);
             }

From 47a83be4da99cc3367639737015632bb7705e4b2 Mon Sep 17 00:00:00 2001
From: Anita SV <anitvasu@amazon.com>
Date: Thu, 11 Jan 2024 15:00:21 -0800
Subject: [PATCH 12/64] primitive hash map

---
 .../onebrc/CalculateAverage_vaidhy.java       | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index b6e6014f0..921dce4ca 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -34,6 +34,32 @@
 
 public class CalculateAverage_vaidhy<T> {
 
+    private record Entry(long startAddress, long endAddress, IntSummaryStatistics value) {
+    }
+
+    private static class PrimitiveHash {
+        Entry [] entries;
+
+        PrimitiveHash(int capacity) {
+            this.entries = new Entry[capacity];
+        }
+
+        public IntSummaryStatistics get(long startAddress, long endAddress, int hash) {
+            int i = hash, len = entries.length;
+            do {
+                Entry entry = entries[i];
+                if (entry.startAddress == 0) {
+                    return null;
+                }
+                i++;
+                if (i == len) {
+                    i = 0;
+                }
+            } while (i != hash);
+            return null;
+        }
+    }
+
     private static final String FILE = "./measurements.txt";
 
     private static Unsafe initUnsafe() {

From 2455f9a83ab5fdc6e49448e8f3db366a8dd4cbda Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Thu, 11 Jan 2024 16:04:07 -0800
Subject: [PATCH 13/64] Primite HashMap

---
 .../onebrc/CalculateAverage_vaidhy.java       | 280 +++++++++++-------
 1 file changed, 166 insertions(+), 114 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index a6b25b6a4..6006787b0 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -32,24 +32,52 @@
 import java.util.function.Function;
 import java.util.function.Supplier;
 
-public class CalculateAverage_vaidhy<T> {
+public class CalculateAverage_vaidhy<I, T> {
 
-    private record Entry(long startAddress, long endAddress, IntSummaryStatistics value) {
+    private static final class HashEntry {
+        private long startAddress;
+        private long endAddress;
+        private long hash;
+        IntSummaryStatistics value;
     }
 
-    private static class PrimitiveHash {
-        Entry [] entries;
+    private static class PrimitiveHashMap {
+        HashEntry[] entries;
 
-        PrimitiveHash(int capacity) {
-            this.entries = new Entry[capacity];
+        PrimitiveHashMap(int capacity) {
+            this.entries = new HashEntry[capacity];
+            for (int i = 0; i < capacity; i++) {
+                this.entries[i] = new HashEntry();
+            }
         }
 
-        public IntSummaryStatistics get(long startAddress, long endAddress, int hash) {
-            int i = hash, len = entries.length;
+        public HashEntry find(long startAddress, long endAddress, long hash) {
+            int len = entries.length;
+            int i = Math.floorMod(hash, len);
+            long lookupLength = endAddress - startAddress;
+
             do {
-                Entry entry = entries[i];
-                if (entry.startAddress == 0) {
-                    return null;
+                HashEntry entry = entries[i];
+                if (entry.value == null) {
+                    return entry;
+                }
+                if (entry.hash == hash) {
+                    long entryLength = endAddress - startAddress;
+                    if (entryLength == lookupLength) {
+                        long entryIndex = entry.startAddress;
+                        long lookupIndex = startAddress;
+                        boolean found = true;
+                        for (; lookupIndex < endAddress; lookupIndex++) {
+                            if (UNSAFE.getByte(entryIndex) != UNSAFE.getByte(lookupIndex)) {
+                                found = false;
+                                break;
+                            }
+                            entryIndex++;
+                        }
+                        if (found) {
+                            return entry;
+                        }
+                    }
                 }
                 i++;
                 if (i == len) {
@@ -74,66 +102,65 @@ private static Unsafe initUnsafe() {
     }
 
     private static final Unsafe UNSAFE = initUnsafe();
-
-    record ByteSlice(long from, long to, int hCode) {
-
-        @Override
-        public int hashCode() {
-            if (hCode != 0) { return hCode; }
-            int h = 0;
-            for (long i = from; i < to; i++) {
-                h = (h * 31) ^ UNSAFE.getByte(i);
-            }
-            return h;
-        }
-
-        public int length() {
-            return (int) (to - from);
-        }
-
-        public byte get(int i) {
-            return UNSAFE.getByte(from + i);
-        }
-
-        @Override
-        public boolean equals(Object o) {
-            if (o instanceof ByteSlice bs) {
-                int len = this.length();
-                if (bs.length() != len) {
-                    return false;
-                }
-                for (int i = 0; i < len; i++) {
-                    if (this.get(i) != bs.get(i)) {
-                        return false;
-                    }
-                }
-                return true;
-            } else {
-                return false;
-            }
-        }
-
-        @Override
-        public String toString() {
-            byte[] copy = new byte[this.length()];
-            for (int i = 0; i < copy.length; i++) {
-                copy[i] = get(i);
-            }
-            return new String(copy, StandardCharsets.UTF_8);
-        }
-    }
-
-    private static int parseDouble(ByteSlice slice) {
+    //
+    // record ByteSlice(long from, long to, int hCode) {
+    //
+    // @Override
+    // public int hashCode() {
+    // if (hCode != 0) { return hCode; }
+    // int h = 0;
+    // for (long i = from; i < to; i++) {
+    // h = (h * 31) ^ UNSAFE.getByte(i);
+    // }
+    // return h;
+    // }
+    //
+    // public int length() {
+    // return (int) (to - from);
+    // }
+    //
+    // public byte get(int i) {
+    // return UNSAFE.getByte(from + i);
+    // }
+    //
+    // @Override
+    // public boolean equals(Object o) {
+    // if (o instanceof ByteSlice bs) {
+    // int len = this.length();
+    // if (bs.length() != len) {
+    // return false;
+    // }
+    // for (int i = 0; i < len; i++) {
+    // if (this.get(i) != bs.get(i)) {
+    // return false;
+    // }
+    // }
+    // return true;
+    // } else {
+    // return false;
+    // }
+    // }
+    //
+    // @Override
+    // public String toString() {
+    // byte[] copy = new byte[this.length()];
+    // for (int i = 0; i < copy.length; i++) {
+    // copy[i] = get(i);
+    // }
+    // return new String(copy, StandardCharsets.UTF_8);
+    // }
+    // }
+
+    private static int parseDouble(long startAddress, long endAddress) {
         int normalized = 0;
         boolean sign = true;
-        int index = 0;
-        if (slice.get(index) == '-') {
+        long index = startAddress;
+        if (UNSAFE.getByte(index) == '-') {
             index++;
             sign = false;
         }
-        int length = slice.length();
-        for (; index < length; index++) {
-            byte ch = slice.get(index);
+        for (; index < endAddress; index++) {
+            byte ch = UNSAFE.getByte(index);
             if (ch != '.') {
                 normalized = normalized * 10 + (ch - '0');
             }
@@ -144,16 +171,16 @@ private static int parseDouble(ByteSlice slice) {
         return normalized;
     }
 
-    interface MapReduce<T> {
+    interface MapReduce<I> {
 
-        void process(ByteSlice key, ByteSlice value);
+        void process(long keyStartAddress, long keyEndAddress, int hash, int temperature);
 
-        T result();
+        I result();
     }
 
     private final FileService fileService;
-    private final Supplier<MapReduce<T>> chunkProcessCreator;
-    private final Function<List<T>, T> reducer;
+    private final Supplier<MapReduce<I>> chunkProcessCreator;
+    private final Function<List<I>, T> reducer;
 
     interface FileService {
         long length();
@@ -161,9 +188,9 @@ interface FileService {
         long address();
     }
 
-    public CalculateAverage_vaidhy(FileService fileService,
-                                   Supplier<MapReduce<T>> mapReduce,
-                                   Function<List<T>, T> reducer) {
+    CalculateAverage_vaidhy(FileService fileService,
+                            Supplier<MapReduce<I>> mapReduce,
+                            Function<List<I>, T> reducer) {
         this.fileService = fileService;
         this.chunkProcessCreator = mapReduce;
         this.reducer = reducer;
@@ -181,25 +208,27 @@ static class LineStream {
         private final long chunkEnd;
 
         private long position;
+        private int hash;
 
         public LineStream(FileService fileService, long offset, long chunkSize) {
             long fileStart = fileService.address();
             this.fileEnd = fileStart + fileService.length();
             this.chunkEnd = fileStart + offset + chunkSize;
             this.position = fileStart + offset;
+            this.hash = 0;
         }
 
         public boolean hasNext() {
             return position <= chunkEnd && position < fileEnd;
         }
 
-        public ByteSlice until(byte ch, boolean computeHash) {
+        public long find(byte ch, boolean computeHash) {
             int h = 0;
             byte inCh;
             for (long i = position; i < fileEnd; i++) {
                 if ((inCh = UNSAFE.getByte(i)) == ch) {
                     try {
-                        return new ByteSlice(position, i, h);
+                        return i;
                     }
                     finally {
                         position = i + 1;
@@ -207,11 +236,12 @@ public ByteSlice until(byte ch, boolean computeHash) {
                 }
                 if (computeHash) {
                     h = (h * 31) ^ inCh;
+                    this.hash = h;
                 }
             }
 
             try {
-                return new ByteSlice(position, fileEnd, h);
+                return fileEnd;
             }
             finally {
                 position = fileEnd;
@@ -220,13 +250,13 @@ public ByteSlice until(byte ch, boolean computeHash) {
     }
 
     // Space complexity: O(scanSize) + O(max line length)
-    private void worker(long offset, long chunkSize, MapReduce<T> lineConsumer) {
+    private void worker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
         LineStream lineStream = new LineStream(fileService, offset, chunkSize);
 
         if (offset != 0) {
             if (lineStream.hasNext()) {
                 // Skip the first line.
-                lineStream.until((byte) '\n', false);
+                lineStream.find((byte) '\n', false);
             }
             else {
                 // No lines then do nothing.
@@ -234,9 +264,13 @@ private void worker(long offset, long chunkSize, MapReduce<T> lineConsumer) {
             }
         }
         while (lineStream.hasNext()) {
-            ByteSlice key = lineStream.until((byte) ';', true);
-            ByteSlice value = lineStream.until((byte) '\n', false);
-            lineConsumer.process(key, value);
+            long keyStartAddress = lineStream.position;
+            long keyEndAddress = lineStream.find((byte) ';', true);
+            int keyHash = lineStream.hash;
+            long valueStartAddress = lineStream.position;
+            long valueEndAddress = lineStream.find((byte) '\n', false);
+            int temperature = parseDouble(valueStartAddress, valueEndAddress);
+            lineConsumer.process(keyStartAddress, keyEndAddress, keyHash, temperature);
         }
     }
 
@@ -244,19 +278,19 @@ private void worker(long offset, long chunkSize, MapReduce<T> lineConsumer) {
     // workers space assuming they are running in different hosts.
     public T master(long chunkSize, ExecutorService executor) {
         long len = fileService.length();
-        List<Future<T>> summaries = new ArrayList<>();
+        List<Future<I>> summaries = new ArrayList<>();
 
         for (long offset = 0; offset < len; offset += chunkSize) {
             long workerLength = Math.min(len, offset + chunkSize) - offset;
-            MapReduce<T> mr = chunkProcessCreator.get();
+            MapReduce<I> mr = chunkProcessCreator.get();
             final long transferOffset = offset;
-            Future<T> task = executor.submit(() -> {
+            Future<I> task = executor.submit(() -> {
                 worker(transferOffset, workerLength, mr);
                 return mr.result();
             });
             summaries.add(task);
         }
-        List<T> summariesDone = summaries.stream()
+        List<I> summariesDone = summaries.stream()
                 .map(task -> {
                     try {
                         return task.get();
@@ -295,23 +329,27 @@ public long address() {
         }
     }
 
-    public static class ChunkProcessorImpl implements MapReduce<Map<ByteSlice, IntSummaryStatistics>> {
+    public static class ChunkProcessorImpl implements MapReduce<PrimitiveHashMap> {
 
-        private final Map<ByteSlice, IntSummaryStatistics> statistics = new HashMap<>(10000);
+        private final PrimitiveHashMap statistics = new PrimitiveHashMap(1024);
 
         @Override
-        public void process(ByteSlice station, ByteSlice value) {
-            int temperature = parseDouble(value);
-            IntSummaryStatistics stats = statistics.get(station);
-            if (stats == null) {
-                stats = new IntSummaryStatistics();
-                statistics.put(station, stats);
+        public void process(long keyStartAddress, long keyEndAddress, int hash, int temperature) {
+            HashEntry entry = statistics.find(keyStartAddress, keyStartAddress, hash);
+            if (entry == null) {
+                throw new IllegalStateException("Hash table too small :(");
+            }
+            if (entry.value == null) {
+                entry.startAddress = keyStartAddress;
+                entry.endAddress = keyEndAddress;
+                entry.hash = hash;
+                entry.value = new IntSummaryStatistics();
             }
-            stats.accept(temperature);
+            entry.value.accept(temperature);
         }
 
         @Override
-        public Map<ByteSlice, IntSummaryStatistics> result() {
+        public PrimitiveHashMap result() {
             return statistics;
         }
     }
@@ -319,7 +357,7 @@ public Map<ByteSlice, IntSummaryStatistics> result() {
     public static void main(String[] args) throws IOException {
         DiskFileService diskFileService = new DiskFileService(FILE);
 
-        CalculateAverage_vaidhy<Map<ByteSlice, IntSummaryStatistics>> calculateAverageVaidhy = new CalculateAverage_vaidhy<>(
+        CalculateAverage_vaidhy<PrimitiveHashMap, Map<String, IntSummaryStatistics>> calculateAverageVaidhy = new CalculateAverage_vaidhy<>(
                 diskFileService,
                 ChunkProcessorImpl::new,
                 CalculateAverage_vaidhy::combineOutputs);
@@ -331,40 +369,54 @@ public static void main(String[] args) throws IOException {
         long chunkSize = Math.ceilDiv(fileSize, shards);
 
         ExecutorService executor = Executors.newFixedThreadPool(proc);
-        Map<ByteSlice, IntSummaryStatistics> output = calculateAverageVaidhy.master(chunkSize, executor);
+        Map<String, IntSummaryStatistics> output = calculateAverageVaidhy.master(chunkSize, executor);
         executor.shutdown();
 
         Map<String, String> outputStr = toPrintMap(output);
         System.out.println(outputStr);
     }
 
-    private static Map<String, String> toPrintMap(Map<ByteSlice, IntSummaryStatistics> output) {
+    private static Map<String, String> toPrintMap(Map<String, IntSummaryStatistics> output) {
 
         Map<String, String> outputStr = new TreeMap<>();
-        for (Map.Entry<ByteSlice, IntSummaryStatistics> entry : output.entrySet()) {
+        for (Map.Entry<String, IntSummaryStatistics> entry : output.entrySet()) {
             IntSummaryStatistics stat = entry.getValue();
-            outputStr.put(entry.getKey().toString(),
+            outputStr.put(entry.getKey(),
                     STR."\{stat.getMin() / 10.0}/\{Math.round(stat.getAverage()) / 10.0}/\{stat.getMax() / 10.0}");
         }
         return outputStr;
     }
 
-    private static Map<ByteSlice, IntSummaryStatistics> combineOutputs(List<Map<ByteSlice, IntSummaryStatistics>> list) {
-        Map<ByteSlice, IntSummaryStatistics> output = new HashMap<>(10000);
-        for (Map<ByteSlice, IntSummaryStatistics> map : list) {
-            for (Map.Entry<ByteSlice, IntSummaryStatistics> entry : map.entrySet()) {
-                output.compute(entry.getKey(), (ignore, val) -> {
-                    if (val == null) {
-                        return entry.getValue();
-                    }
-                    else {
-                        val.combine(entry.getValue());
-                        return val;
-                    }
-                });
+    private static Map<String, IntSummaryStatistics> combineOutputs(
+                                                                    List<PrimitiveHashMap> list) {
+
+        Map<String, IntSummaryStatistics> output = new HashMap<>(10000);
+        for (PrimitiveHashMap map : list) {
+            for (HashEntry entry : map.entries) {
+                if (entry.value != null) {
+                    String keyStr = unsafeToString(entry.startAddress, entry.endAddress);
+
+                    output.compute(keyStr, (ignore, val) -> {
+                        if (val == null) {
+                            return entry.value;
+                        }
+                        else {
+                            val.combine(entry.value);
+                            return val;
+                        }
+                    });
+                }
             }
         }
 
         return output;
     }
+
+    private static String unsafeToString(long startAddress, long endAddress) {
+        byte[] keyBytes = new byte[(int) (endAddress - startAddress)];
+        for (int i = 0; i < keyBytes.length; i++) {
+            keyBytes[i] = UNSAFE.getByte(startAddress + i);
+        }
+        return new String(keyBytes, StandardCharsets.UTF_8);
+    }
 }

From ea333e2821ebb5c1d6d71a4e87e569a8f2f8f7f0 Mon Sep 17 00:00:00 2001
From: Vaidhy Gopalan <vaidhy@gmail.com>
Date: Thu, 11 Jan 2024 16:45:18 -0800
Subject: [PATCH 14/64] Micro optimizations to push for optimizations

---
 .../onebrc/CalculateAverage_vaidhy.java       | 70 ++++---------------
 1 file changed, 14 insertions(+), 56 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 6006787b0..ca1a39a80 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -67,7 +67,14 @@ public HashEntry find(long startAddress, long endAddress, long hash) {
                         long entryIndex = entry.startAddress;
                         long lookupIndex = startAddress;
                         boolean found = true;
-                        for (; lookupIndex < endAddress; lookupIndex++) {
+                        for (; lookupIndex < endAddress / 8; lookupIndex += 8) {
+                            if (UNSAFE.getLong(entryIndex) != UNSAFE.getLong(lookupIndex)) {
+                                found = false;
+                                break;
+                            }
+                            entryIndex += 8;
+                        }
+                        for (; lookupIndex < lookupLength; lookupIndex++) {
                             if (UNSAFE.getByte(entryIndex) != UNSAFE.getByte(lookupIndex)) {
                                 found = false;
                                 break;
@@ -102,67 +109,19 @@ private static Unsafe initUnsafe() {
     }
 
     private static final Unsafe UNSAFE = initUnsafe();
-    //
-    // record ByteSlice(long from, long to, int hCode) {
-    //
-    // @Override
-    // public int hashCode() {
-    // if (hCode != 0) { return hCode; }
-    // int h = 0;
-    // for (long i = from; i < to; i++) {
-    // h = (h * 31) ^ UNSAFE.getByte(i);
-    // }
-    // return h;
-    // }
-    //
-    // public int length() {
-    // return (int) (to - from);
-    // }
-    //
-    // public byte get(int i) {
-    // return UNSAFE.getByte(from + i);
-    // }
-    //
-    // @Override
-    // public boolean equals(Object o) {
-    // if (o instanceof ByteSlice bs) {
-    // int len = this.length();
-    // if (bs.length() != len) {
-    // return false;
-    // }
-    // for (int i = 0; i < len; i++) {
-    // if (this.get(i) != bs.get(i)) {
-    // return false;
-    // }
-    // }
-    // return true;
-    // } else {
-    // return false;
-    // }
-    // }
-    //
-    // @Override
-    // public String toString() {
-    // byte[] copy = new byte[this.length()];
-    // for (int i = 0; i < copy.length; i++) {
-    // copy[i] = get(i);
-    // }
-    // return new String(copy, StandardCharsets.UTF_8);
-    // }
-    // }
 
     private static int parseDouble(long startAddress, long endAddress) {
         int normalized = 0;
         boolean sign = true;
         long index = startAddress;
-        if (UNSAFE.getByte(index) == '-') {
+        if (UNSAFE.getByte(index) == 0x2D) {
             index++;
             sign = false;
         }
         for (; index < endAddress; index++) {
             byte ch = UNSAFE.getByte(index);
-            if (ch != '.') {
-                normalized = normalized * 10 + (ch - '0');
+            if (ch != 0x2E) {
+                normalized = (normalized << 3) + (normalized << 1) + (ch ^ 0x30);
             }
         }
         if (!sign) {
@@ -224,9 +183,8 @@ public boolean hasNext() {
 
         public long find(byte ch, boolean computeHash) {
             int h = 0;
-            byte inCh;
             for (long i = position; i < fileEnd; i++) {
-                if ((inCh = UNSAFE.getByte(i)) == ch) {
+                if ((UNSAFE.getByte(i) ^ ch) == 0) {
                     try {
                         return i;
                     }
@@ -235,10 +193,10 @@ public long find(byte ch, boolean computeHash) {
                     }
                 }
                 if (computeHash) {
-                    h = (h * 31) ^ inCh;
-                    this.hash = h;
+                    h = ((h << 5) - h) ^ UNSAFE.getByte(i);
                 }
             }
+            this.hash = h;
 
             try {
                 return fileEnd;

From 44f5169d93f5b3e59f5f251363e36dd3f26f2889 Mon Sep 17 00:00:00 2001
From: Vaidhy Gopalan <vaidhy@gmail.com>
Date: Thu, 11 Jan 2024 17:03:42 -0800
Subject: [PATCH 15/64] Revert "Micro optimizations to push for optimizations"

This reverts commit ea333e2821ebb5c1d6d71a4e87e569a8f2f8f7f0.
---
 .../onebrc/CalculateAverage_vaidhy.java       | 70 +++++++++++++++----
 1 file changed, 56 insertions(+), 14 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index ca1a39a80..6006787b0 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -67,14 +67,7 @@ public HashEntry find(long startAddress, long endAddress, long hash) {
                         long entryIndex = entry.startAddress;
                         long lookupIndex = startAddress;
                         boolean found = true;
-                        for (; lookupIndex < endAddress / 8; lookupIndex += 8) {
-                            if (UNSAFE.getLong(entryIndex) != UNSAFE.getLong(lookupIndex)) {
-                                found = false;
-                                break;
-                            }
-                            entryIndex += 8;
-                        }
-                        for (; lookupIndex < lookupLength; lookupIndex++) {
+                        for (; lookupIndex < endAddress; lookupIndex++) {
                             if (UNSAFE.getByte(entryIndex) != UNSAFE.getByte(lookupIndex)) {
                                 found = false;
                                 break;
@@ -109,19 +102,67 @@ private static Unsafe initUnsafe() {
     }
 
     private static final Unsafe UNSAFE = initUnsafe();
+    //
+    // record ByteSlice(long from, long to, int hCode) {
+    //
+    // @Override
+    // public int hashCode() {
+    // if (hCode != 0) { return hCode; }
+    // int h = 0;
+    // for (long i = from; i < to; i++) {
+    // h = (h * 31) ^ UNSAFE.getByte(i);
+    // }
+    // return h;
+    // }
+    //
+    // public int length() {
+    // return (int) (to - from);
+    // }
+    //
+    // public byte get(int i) {
+    // return UNSAFE.getByte(from + i);
+    // }
+    //
+    // @Override
+    // public boolean equals(Object o) {
+    // if (o instanceof ByteSlice bs) {
+    // int len = this.length();
+    // if (bs.length() != len) {
+    // return false;
+    // }
+    // for (int i = 0; i < len; i++) {
+    // if (this.get(i) != bs.get(i)) {
+    // return false;
+    // }
+    // }
+    // return true;
+    // } else {
+    // return false;
+    // }
+    // }
+    //
+    // @Override
+    // public String toString() {
+    // byte[] copy = new byte[this.length()];
+    // for (int i = 0; i < copy.length; i++) {
+    // copy[i] = get(i);
+    // }
+    // return new String(copy, StandardCharsets.UTF_8);
+    // }
+    // }
 
     private static int parseDouble(long startAddress, long endAddress) {
         int normalized = 0;
         boolean sign = true;
         long index = startAddress;
-        if (UNSAFE.getByte(index) == 0x2D) {
+        if (UNSAFE.getByte(index) == '-') {
             index++;
             sign = false;
         }
         for (; index < endAddress; index++) {
             byte ch = UNSAFE.getByte(index);
-            if (ch != 0x2E) {
-                normalized = (normalized << 3) + (normalized << 1) + (ch ^ 0x30);
+            if (ch != '.') {
+                normalized = normalized * 10 + (ch - '0');
             }
         }
         if (!sign) {
@@ -183,8 +224,9 @@ public boolean hasNext() {
 
         public long find(byte ch, boolean computeHash) {
             int h = 0;
+            byte inCh;
             for (long i = position; i < fileEnd; i++) {
-                if ((UNSAFE.getByte(i) ^ ch) == 0) {
+                if ((inCh = UNSAFE.getByte(i)) == ch) {
                     try {
                         return i;
                     }
@@ -193,10 +235,10 @@ public long find(byte ch, boolean computeHash) {
                     }
                 }
                 if (computeHash) {
-                    h = ((h << 5) - h) ^ UNSAFE.getByte(i);
+                    h = (h * 31) ^ inCh;
+                    this.hash = h;
                 }
             }
-            this.hash = h;
 
             try {
                 return fileEnd;

From 0be1a16ee9166e8553c11c26ec9d5423a2d4db4c Mon Sep 17 00:00:00 2001
From: Vaidhy Gopalan <vaidhy@gmail.com>
Date: Thu, 11 Jan 2024 17:06:33 -0800
Subject: [PATCH 16/64] Micro optimizations to get the juice

---
 src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 6006787b0..a7a3050f4 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -162,7 +162,7 @@ private static int parseDouble(long startAddress, long endAddress) {
         for (; index < endAddress; index++) {
             byte ch = UNSAFE.getByte(index);
             if (ch != '.') {
-                normalized = normalized * 10 + (ch - '0');
+                normalized = (normalized << 3) + (normalized << 1) + (ch - '0');
             }
         }
         if (!sign) {
@@ -235,7 +235,7 @@ public long find(byte ch, boolean computeHash) {
                     }
                 }
                 if (computeHash) {
-                    h = (h * 31) ^ inCh;
+                    h = ((h << 5) - h) ^ inCh;
                     this.hash = h;
                 }
             }

From 2eb87aec718321faea4bb2cf31d5eb2df6ac06e6 Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Thu, 11 Jan 2024 17:20:59 -0800
Subject: [PATCH 17/64] floorMod fixes

---
 .../onebrc/CalculateAverage_vaidhy.java       | 93 +++++--------------
 1 file changed, 22 insertions(+), 71 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index a7a3050f4..56a36a8b5 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -37,24 +37,25 @@ public class CalculateAverage_vaidhy<I, T> {
     private static final class HashEntry {
         private long startAddress;
         private long endAddress;
-        private long hash;
+        private int hash;
         IntSummaryStatistics value;
     }
 
     private static class PrimitiveHashMap {
-        HashEntry[] entries;
+        private final HashEntry[] entries;
+        private final int twoPow;
 
-        PrimitiveHashMap(int capacity) {
-            this.entries = new HashEntry[capacity];
-            for (int i = 0; i < capacity; i++) {
+        PrimitiveHashMap(int twoPow) {
+            this.twoPow = twoPow;
+            this.entries = new HashEntry[1 << twoPow];
+            for (int i = 0; i < entries.length; i++) {
                 this.entries[i] = new HashEntry();
             }
         }
 
-        public HashEntry find(long startAddress, long endAddress, long hash) {
+        public HashEntry find(long startAddress, long endAddress, int hash) {
             int len = entries.length;
-            int i = Math.floorMod(hash, len);
-            long lookupLength = endAddress - startAddress;
+            int i = (hash ^ (hash >> twoPow)) & (len - 1);
 
             do {
                 HashEntry entry = entries[i];
@@ -62,21 +63,18 @@ public HashEntry find(long startAddress, long endAddress, long hash) {
                     return entry;
                 }
                 if (entry.hash == hash) {
-                    long entryLength = endAddress - startAddress;
-                    if (entryLength == lookupLength) {
-                        long entryIndex = entry.startAddress;
-                        long lookupIndex = startAddress;
-                        boolean found = true;
-                        for (; lookupIndex < endAddress; lookupIndex++) {
-                            if (UNSAFE.getByte(entryIndex) != UNSAFE.getByte(lookupIndex)) {
-                                found = false;
-                                break;
-                            }
-                            entryIndex++;
-                        }
-                        if (found) {
-                            return entry;
+                    long entryIndex = entry.startAddress;
+                    long lookupIndex = startAddress;
+                    boolean found = true;
+                    for (; lookupIndex < endAddress; lookupIndex++) {
+                        if (UNSAFE.getByte(entryIndex) != UNSAFE.getByte(lookupIndex)) {
+                            found = false;
+                            break;
                         }
+                        entryIndex++;
+                    }
+                    if (found) {
+                        return entry;
                     }
                 }
                 i++;
@@ -102,54 +100,6 @@ private static Unsafe initUnsafe() {
     }
 
     private static final Unsafe UNSAFE = initUnsafe();
-    //
-    // record ByteSlice(long from, long to, int hCode) {
-    //
-    // @Override
-    // public int hashCode() {
-    // if (hCode != 0) { return hCode; }
-    // int h = 0;
-    // for (long i = from; i < to; i++) {
-    // h = (h * 31) ^ UNSAFE.getByte(i);
-    // }
-    // return h;
-    // }
-    //
-    // public int length() {
-    // return (int) (to - from);
-    // }
-    //
-    // public byte get(int i) {
-    // return UNSAFE.getByte(from + i);
-    // }
-    //
-    // @Override
-    // public boolean equals(Object o) {
-    // if (o instanceof ByteSlice bs) {
-    // int len = this.length();
-    // if (bs.length() != len) {
-    // return false;
-    // }
-    // for (int i = 0; i < len; i++) {
-    // if (this.get(i) != bs.get(i)) {
-    // return false;
-    // }
-    // }
-    // return true;
-    // } else {
-    // return false;
-    // }
-    // }
-    //
-    // @Override
-    // public String toString() {
-    // byte[] copy = new byte[this.length()];
-    // for (int i = 0; i < copy.length; i++) {
-    // copy[i] = get(i);
-    // }
-    // return new String(copy, StandardCharsets.UTF_8);
-    // }
-    // }
 
     private static int parseDouble(long startAddress, long endAddress) {
         int normalized = 0;
@@ -331,7 +281,8 @@ public long address() {
 
     public static class ChunkProcessorImpl implements MapReduce<PrimitiveHashMap> {
 
-        private final PrimitiveHashMap statistics = new PrimitiveHashMap(1024);
+        // 1 << 14 > 10,000 so it works
+        private final PrimitiveHashMap statistics = new PrimitiveHashMap(14);
 
         @Override
         public void process(long keyStartAddress, long keyEndAddress, int hash, int temperature) {

From ccd154a18d15b6939b2dba9ecf084fea0fbd2c6a Mon Sep 17 00:00:00 2001
From: Vaidhy Gopalan <vaidhy@gmail.com>
Date: Thu, 11 Jan 2024 17:41:21 -0800
Subject: [PATCH 18/64] findSemi and findNewLine as separate functions

---
 .../onebrc/CalculateAverage_vaidhy.java       | 35 ++++++++++++++-----
 1 file changed, 27 insertions(+), 8 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 56a36a8b5..a01198751 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -172,11 +172,11 @@ public boolean hasNext() {
             return position <= chunkEnd && position < fileEnd;
         }
 
-        public long find(byte ch, boolean computeHash) {
+        public long findSemi() {
             int h = 0;
             byte inCh;
             for (long i = position; i < fileEnd; i++) {
-                if ((inCh = UNSAFE.getByte(i)) == ch) {
+                if ((inCh = UNSAFE.getByte(i)) == 0x3B) {
                     try {
                         return i;
                     }
@@ -184,9 +184,28 @@ public long find(byte ch, boolean computeHash) {
                         position = i + 1;
                     }
                 }
-                if (computeHash) {
-                    h = ((h << 5) - h) ^ inCh;
-                    this.hash = h;
+                h = ((h << 5) - h) ^ inCh;
+                this.hash = h;
+            }
+
+            try {
+                return fileEnd;
+            }
+            finally {
+                position = fileEnd;
+            }
+        }
+
+        public long findNewLine() {
+            this.hash = 0;
+            for (long i = position; i < fileEnd; i++) {
+                if ((UNSAFE.getByte(i)) == 0x0a) {
+                    try {
+                        return i;
+                    }
+                    finally {
+                        position = i + 1;
+                    }
                 }
             }
 
@@ -206,7 +225,7 @@ private void worker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
         if (offset != 0) {
             if (lineStream.hasNext()) {
                 // Skip the first line.
-                lineStream.find((byte) '\n', false);
+                lineStream.findNewLine();
             }
             else {
                 // No lines then do nothing.
@@ -215,10 +234,10 @@ private void worker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
         }
         while (lineStream.hasNext()) {
             long keyStartAddress = lineStream.position;
-            long keyEndAddress = lineStream.find((byte) ';', true);
+            long keyEndAddress = lineStream.findSemi();
             int keyHash = lineStream.hash;
             long valueStartAddress = lineStream.position;
-            long valueEndAddress = lineStream.find((byte) '\n', false);
+            long valueEndAddress = lineStream.findNewLine();
             int temperature = parseDouble(valueStartAddress, valueEndAddress);
             lineConsumer.process(keyStartAddress, keyEndAddress, keyHash, temperature);
         }

From e338cc5f5a53fafb219336da032902f2c327410a Mon Sep 17 00:00:00 2001
From: Vaidhy Gopalan <vaidhy@gmail.com>
Date: Thu, 11 Jan 2024 18:40:00 -0800
Subject: [PATCH 19/64] Optimized parseDouble

---
 .../onebrc/CalculateAverage_vaidhy.java       | 34 ++++++++++++-------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index a01198751..0b9db2786 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -103,22 +103,32 @@ private static Unsafe initUnsafe() {
 
     private static int parseDouble(long startAddress, long endAddress) {
         int normalized = 0;
-        boolean sign = true;
-        long index = startAddress;
-        if (UNSAFE.getByte(index) == '-') {
-            index++;
-            sign = false;
+        int length = (int) (endAddress - startAddress);
+        if (length == 5) {
+            normalized = (UNSAFE.getByte(startAddress + 1) ^ 0x30);
+            normalized = (normalized << 3) + (normalized << 1) + (UNSAFE.getByte(startAddress + 2) ^ 0x30);
+            normalized = (normalized << 3) + (normalized << 1) + (UNSAFE.getByte(startAddress + 4) ^ 0x30);
+            normalized = -normalized;
+            return normalized;
         }
-        for (; index < endAddress; index++) {
-            byte ch = UNSAFE.getByte(index);
-            if (ch != '.') {
-                normalized = (normalized << 3) + (normalized << 1) + (ch - '0');
-            }
+        if (length == 3) {
+            normalized = (UNSAFE.getByte(startAddress) ^ 0x30);
+            normalized = (normalized << 3) + (normalized << 1) + (UNSAFE.getByte(startAddress + 2) ^ 0x30);
+            return normalized;
         }
-        if (!sign) {
+
+        if (UNSAFE.getByte(startAddress) == '-') {
+            normalized = (UNSAFE.getByte(startAddress + 1) ^ 0x30);
+            normalized = (normalized << 3) + (normalized << 1) + (UNSAFE.getByte(startAddress + 3) ^ 0x30);
             normalized = -normalized;
+            return normalized;
+        }
+        else {
+            normalized = (UNSAFE.getByte(startAddress) ^ 0x30);
+            normalized = (normalized << 3) + (normalized << 1) + (UNSAFE.getByte(startAddress + 1) ^ 0x30);
+            normalized = (normalized << 3) + (normalized << 1) + (UNSAFE.getByte(startAddress + 3) ^ 0x30);
+            return normalized;
         }
-        return normalized;
     }
 
     interface MapReduce<I> {

From a51249d29a7827558a2005dd64be6d8d217d67ee Mon Sep 17 00:00:00 2001
From: Vaidhy Gopalan <vaidhy@gmail.com>
Date: Thu, 11 Jan 2024 18:49:04 -0800
Subject: [PATCH 20/64] More micro changes

---
 .../onebrc/CalculateAverage_vaidhy.java       | 29 +++++--------------
 1 file changed, 8 insertions(+), 21 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 0b9db2786..d6e2f8bdf 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -187,35 +187,23 @@ public long findSemi() {
             byte inCh;
             for (long i = position; i < fileEnd; i++) {
                 if ((inCh = UNSAFE.getByte(i)) == 0x3B) {
-                    try {
-                        return i;
-                    }
-                    finally {
-                        position = i + 1;
-                    }
+                    this.hash = h;
+                    position = i + 1;
+                    return i;
                 }
                 h = ((h << 5) - h) ^ inCh;
-                this.hash = h;
-            }
-
-            try {
-                return fileEnd;
-            }
-            finally {
-                position = fileEnd;
             }
+            this.hash = h;
+            position = fileEnd;
+            return fileEnd;
         }
 
         public long findNewLine() {
             this.hash = 0;
             for (long i = position; i < fileEnd; i++) {
                 if ((UNSAFE.getByte(i)) == 0x0a) {
-                    try {
-                        return i;
-                    }
-                    finally {
-                        position = i + 1;
-                    }
+                    position = i + 1;
+                    return i;
                 }
             }
 
@@ -285,7 +273,6 @@ public T master(long chunkSize, ExecutorService executor) {
     /// SAMPLE CANDIDATE CODE ENDS
 
     static class DiskFileService implements FileService {
-
         private final long fileSize;
         private final long mappedAddress;
 

From 56ddd32d406b17462bd0e01c9bdbcc5be066e81e Mon Sep 17 00:00:00 2001
From: Vaidhy Gopalan <vaidhy@gmail.com>
Date: Thu, 11 Jan 2024 20:21:19 -0800
Subject: [PATCH 21/64] Aligned equal check

---
 .../onebrc/CalculateAverage_vaidhy.java       | 57 ++++++++++++-------
 1 file changed, 38 insertions(+), 19 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index d6e2f8bdf..1628500df 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -37,7 +37,9 @@ public class CalculateAverage_vaidhy<I, T> {
     private static final class HashEntry {
         private long startAddress;
         private long endAddress;
+        private long suffix;
         private int hash;
+
         IntSummaryStatistics value;
     }
 
@@ -53,7 +55,7 @@ private static class PrimitiveHashMap {
             }
         }
 
-        public HashEntry find(long startAddress, long endAddress, int hash) {
+        public HashEntry find(long startAddress, long endAddress, long suffix, int hash) {
             int len = entries.length;
             int i = (hash ^ (hash >> twoPow)) & (len - 1);
 
@@ -63,18 +65,17 @@ public HashEntry find(long startAddress, long endAddress, int hash) {
                     return entry;
                 }
                 if (entry.hash == hash) {
-                    long entryIndex = entry.startAddress;
-                    long lookupIndex = startAddress;
-                    boolean found = true;
-                    for (; lookupIndex < endAddress; lookupIndex++) {
-                        if (UNSAFE.getByte(entryIndex) != UNSAFE.getByte(lookupIndex)) {
-                            found = false;
-                            break;
+                    long entryLength = entry.endAddress - entry.startAddress;
+                    long lookupLength = endAddress - startAddress;
+                    if ((entryLength == lookupLength) && (entry.suffix == suffix)) {
+                        boolean found = compareEntryKeys(startAddress, endAddress, entry);
+
+                        if (found) {
+                            return entry;
                         }
-                        entryIndex++;
                     }
-                    if (found) {
-                        return entry;
+                    else {
+                        System.out.println("Broken");
                     }
                 }
                 i++;
@@ -84,6 +85,19 @@ public HashEntry find(long startAddress, long endAddress, int hash) {
             } while (i != hash);
             return null;
         }
+
+        private static boolean compareEntryKeys(long startAddress, long endAddress, HashEntry entry) {
+            long entryIndex = entry.startAddress;
+            long lookupIndex = startAddress;
+
+            for (; (lookupIndex + 7) < endAddress; lookupIndex += 8) {
+                if (UNSAFE.getLong(entryIndex) != UNSAFE.getLong(lookupIndex)) {
+                    return false;
+                }
+                entryIndex += 8;
+            }
+            return true;
+        }
     }
 
     private static final String FILE = "./measurements.txt";
@@ -206,13 +220,8 @@ public long findNewLine() {
                     return i;
                 }
             }
-
-            try {
-                return fileEnd;
-            }
-            finally {
-                position = fileEnd;
-            }
+            position = fileEnd;
+            return fileEnd;
         }
     }
 
@@ -302,13 +311,23 @@ public static class ChunkProcessorImpl implements MapReduce<PrimitiveHashMap> {
 
         @Override
         public void process(long keyStartAddress, long keyEndAddress, int hash, int temperature) {
-            HashEntry entry = statistics.find(keyStartAddress, keyStartAddress, hash);
+            int length = (int) (keyEndAddress - keyStartAddress);
+            int alignedLength = (length >> 3) << 3;
+            long alignedAddress = keyStartAddress + alignedLength;
+            int tail = length & 7;
+            long suffix = 0;
+            if (tail != 0) {
+                suffix = UNSAFE.getLong(alignedAddress) << ((8 - tail) * 8);
+            }
+
+            HashEntry entry = statistics.find(keyStartAddress, keyEndAddress, suffix, hash);
             if (entry == null) {
                 throw new IllegalStateException("Hash table too small :(");
             }
             if (entry.value == null) {
                 entry.startAddress = keyStartAddress;
                 entry.endAddress = keyEndAddress;
+                entry.suffix = suffix;
                 entry.hash = hash;
                 entry.value = new IntSummaryStatistics();
             }

From f25cf738a6288b44c4a9b7f0c4fec94647984331 Mon Sep 17 00:00:00 2001
From: Vaidhy Gopalan <vaidhy@gmail.com>
Date: Thu, 11 Jan 2024 20:55:10 -0800
Subject: [PATCH 22/64] more small changes

---
 .../onebrc/CalculateAverage_vaidhy.java       | 55 ++++++++++---------
 1 file changed, 29 insertions(+), 26 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 1628500df..13c30b2f1 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -75,7 +75,8 @@ public HashEntry find(long startAddress, long endAddress, long suffix, int hash)
                         }
                     }
                     else {
-                        System.out.println("Broken");
+                        System.out.println("Broken - Entry : " + unsafeToString(entry.startAddress, entry.endAddress) +
+                                " Hash - " + unsafeToString(startAddress, endAddress));
                     }
                 }
                 i++;
@@ -116,7 +117,7 @@ private static Unsafe initUnsafe() {
     private static final Unsafe UNSAFE = initUnsafe();
 
     private static int parseDouble(long startAddress, long endAddress) {
-        int normalized = 0;
+        int normalized;
         int length = (int) (endAddress - startAddress);
         if (length == 5) {
             normalized = (UNSAFE.getByte(startAddress + 1) ^ 0x30);
@@ -147,7 +148,7 @@ private static int parseDouble(long startAddress, long endAddress) {
 
     interface MapReduce<I> {
 
-        void process(long keyStartAddress, long keyEndAddress, int hash, int temperature);
+        void process(long keyStartAddress, long keyEndAddress, int hash, int temperature, long suffix);
 
         I result();
     }
@@ -183,6 +184,7 @@ static class LineStream {
 
         private long position;
         private int hash;
+        private long suffix;
 
         public LineStream(FileService fileService, long offset, long chunkSize) {
             long fileStart = fileService.address();
@@ -198,18 +200,19 @@ public boolean hasNext() {
 
         public long findSemi() {
             int h = 0;
-            byte inCh;
-            for (long i = position; i < fileEnd; i++) {
-                if ((inCh = UNSAFE.getByte(i)) == 0x3B) {
-                    this.hash = h;
-                    position = i + 1;
-                    return i;
-                }
-                h = ((h << 5) - h) ^ inCh;
+            int s = 0;
+            long i = position;
+            for (; i < fileEnd; i++) {
+                byte myCh = UNSAFE.getByte(i);
+                if (myCh != 0x3B) {
+                    h = ((h << 5) - h) ^ myCh;
+                    s = (s << 8) ^ myCh;
+                } else break;
             }
             this.hash = h;
-            position = fileEnd;
-            return fileEnd;
+            this.suffix = s;
+            position = i + 1;
+            return i;
         }
 
         public long findNewLine() {
@@ -242,11 +245,12 @@ private void worker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
         while (lineStream.hasNext()) {
             long keyStartAddress = lineStream.position;
             long keyEndAddress = lineStream.findSemi();
+            long keySuffix = lineStream.suffix;
             int keyHash = lineStream.hash;
             long valueStartAddress = lineStream.position;
             long valueEndAddress = lineStream.findNewLine();
             int temperature = parseDouble(valueStartAddress, valueEndAddress);
-            lineConsumer.process(keyStartAddress, keyEndAddress, keyHash, temperature);
+            lineConsumer.process(keyStartAddress, keyEndAddress, keyHash, temperature, keySuffix);
         }
     }
 
@@ -304,21 +308,21 @@ public long address() {
         }
     }
 
-    public static class ChunkProcessorImpl implements MapReduce<PrimitiveHashMap> {
+    private static class ChunkProcessorImpl implements MapReduce<PrimitiveHashMap> {
 
         // 1 << 14 > 10,000 so it works
         private final PrimitiveHashMap statistics = new PrimitiveHashMap(14);
 
         @Override
-        public void process(long keyStartAddress, long keyEndAddress, int hash, int temperature) {
-            int length = (int) (keyEndAddress - keyStartAddress);
-            int alignedLength = (length >> 3) << 3;
-            long alignedAddress = keyStartAddress + alignedLength;
-            int tail = length & 7;
-            long suffix = 0;
-            if (tail != 0) {
-                suffix = UNSAFE.getLong(alignedAddress) << ((8 - tail) * 8);
-            }
+        public void process(long keyStartAddress, long keyEndAddress, int hash, int temperature, long suffix) {
+            // int length = (int) (keyEndAddress - keyStartAddress);
+            // int alignedLength = (length >> 3) << 3;
+            // long alignedAddress = keyStartAddress + alignedLength;
+            // int tail = length & 7;
+            // long suffix = 0;
+            // if (tail != 0) {
+            // suffix = UNSAFE.getLong(alignedAddress) << ((8 - tail) * 8);
+            // }
 
             HashEntry entry = statistics.find(keyStartAddress, keyEndAddress, suffix, hash);
             if (entry == null) {
@@ -350,9 +354,8 @@ public static void main(String[] args) throws IOException {
 
         int proc = 2 * Runtime.getRuntime().availableProcessors();
 
-        int shards = proc;
         long fileSize = diskFileService.length();
-        long chunkSize = Math.ceilDiv(fileSize, shards);
+        long chunkSize = Math.ceilDiv(fileSize, proc);
 
         ExecutorService executor = Executors.newFixedThreadPool(proc);
         Map<String, IntSummaryStatistics> output = calculateAverageVaidhy.master(chunkSize, executor);

From 71d95024653ba5787d284bc251c59d7eb55f0d71 Mon Sep 17 00:00:00 2001
From: Vaidhy Gopalan <vaidhy@gmail.com>
Date: Thu, 11 Jan 2024 21:14:46 -0800
Subject: [PATCH 23/64] XOR instead of compare

---
 .../java/dev/morling/onebrc/CalculateAverage_vaidhy.java | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 13c30b2f1..e83a35770 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -204,10 +204,12 @@ public long findSemi() {
             long i = position;
             for (; i < fileEnd; i++) {
                 byte myCh = UNSAFE.getByte(i);
-                if (myCh != 0x3B) {
+                if ((myCh ^ 0x3B) == 0) {
                     h = ((h << 5) - h) ^ myCh;
                     s = (s << 8) ^ myCh;
-                } else break;
+                }
+                else
+                    break;
             }
             this.hash = h;
             this.suffix = s;
@@ -218,7 +220,8 @@ public long findSemi() {
         public long findNewLine() {
             this.hash = 0;
             for (long i = position; i < fileEnd; i++) {
-                if ((UNSAFE.getByte(i)) == 0x0a) {
+                byte ch = UNSAFE.getByte(i);
+                if ((ch ^ 0x0a) == 0) {
                     position = i + 1;
                     return i;
                 }

From f7825255a4cfad73298f532d2522648915e11cb1 Mon Sep 17 00:00:00 2001
From: Vaidhy Gopalan <vaidhy@gmail.com>
Date: Thu, 11 Jan 2024 21:17:39 -0800
Subject: [PATCH 24/64] Reduce loop length

---
 src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index e83a35770..f5ee6bafc 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -219,7 +219,7 @@ public long findSemi() {
 
         public long findNewLine() {
             this.hash = 0;
-            for (long i = position; i < fileEnd; i++) {
+            for (long i = position; i < position + 7; i++) {
                 byte ch = UNSAFE.getByte(i);
                 if ((ch ^ 0x0a) == 0) {
                     position = i + 1;

From eedee26fd1757631f5eefcbe17c87328396ed2a4 Mon Sep 17 00:00:00 2001
From: Vaidhy Gopalan <vaidhy@gmail.com>
Date: Thu, 11 Jan 2024 21:21:46 -0800
Subject: [PATCH 25/64] Revert changes

---
 .../dev/morling/onebrc/CalculateAverage_vaidhy.java | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index f5ee6bafc..2c7528304 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -204,12 +204,11 @@ public long findSemi() {
             long i = position;
             for (; i < fileEnd; i++) {
                 byte myCh = UNSAFE.getByte(i);
-                if ((myCh ^ 0x3B) == 0) {
-                    h = ((h << 5) - h) ^ myCh;
-                    s = (s << 8) ^ myCh;
-                }
-                else
+                if (myCh == 0x3B) {
                     break;
+                }
+                h = ((h << 5) - h) ^ myCh;
+                s = (s << 8) ^ myCh;
             }
             this.hash = h;
             this.suffix = s;
@@ -219,9 +218,9 @@ public long findSemi() {
 
         public long findNewLine() {
             this.hash = 0;
-            for (long i = position; i < position + 7; i++) {
+            for (long i = position; i < fileEnd; i++) {
                 byte ch = UNSAFE.getByte(i);
-                if ((ch ^ 0x0a) == 0) {
+                if (ch == 0x0a) {
                     position = i + 1;
                     return i;
                 }

From 310da9f802cc57f4ba6f70362857b9b8e912a79f Mon Sep 17 00:00:00 2001
From: Vaidhy Gopalan <vaidhy@gmail.com>
Date: Thu, 11 Jan 2024 21:49:44 -0800
Subject: [PATCH 26/64] Loop optimization and added native build

---
 additional_build_step_vaidhy.sh               | 21 +++++++++++++++++++
 calculate_average_vaidhy.sh                   | 15 ++++++++-----
 .../onebrc/CalculateAverage_vaidhy.java       | 20 ++++++++++++++----
 3 files changed, 47 insertions(+), 9 deletions(-)
 create mode 100755 additional_build_step_vaidhy.sh

diff --git a/additional_build_step_vaidhy.sh b/additional_build_step_vaidhy.sh
new file mode 100755
index 000000000..e8520df78
--- /dev/null
+++ b/additional_build_step_vaidhy.sh
@@ -0,0 +1,21 @@
+#!/bin/sh
+#
+#  Copyright 2023 The original authors
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+source "$HOME/.sdkman/bin/sdkman-init.sh"
+sdk use java 21.0.1-graal 1>&2
+NATIVE_IMAGE_OPTS="--gc=epsilon -O3 -march=native --enable-preview"
+native-image $NATIVE_IMAGE_OPTS -cp target/average-1.0.0-SNAPSHOT.jar -o image_calculateaverage_vaidhy dev.morling.onebrc.CalculateAverage_vaidhy
diff --git a/calculate_average_vaidhy.sh b/calculate_average_vaidhy.sh
index fdb4b531c..1c51dc3d1 100755
--- a/calculate_average_vaidhy.sh
+++ b/calculate_average_vaidhy.sh
@@ -15,8 +15,13 @@
 #  limitations under the License.
 #
 
-source "$HOME/.sdkman/bin/sdkman-init.sh"
-sdk use java 21.0.1-open 1>&2
-
-JAVA_OPTS="--enable-preview -Djava.util.concurrent.ForkJoinPool.common.parallelism=8"
-time java $JAVA_OPTS --class-path target/average-1.0.0-SNAPSHOT.jar dev.morling.onebrc.CalculateAverage_vaidhy
+if [ -f ./image_calculateaverage_vaidhy ]; then
+    echo "Picking up existing native image, delete the file to select JVM mode." 1>&2
+    time ./image_calculateaverage_vaidhy
+else
+    source "$HOME/.sdkman/bin/sdkman-init.sh"
+    sdk use java 21.0.1-graal 1>&2
+    JAVA_OPTS="--enable-preview"
+    echo "Choosing to run the app in JVM mode as no native image was found, use additional_build_step_vaidhy.sh to generate." 1>&2
+    time java $JAVA_OPTS --class-path target/average-1.0.0-SNAPSHOT.jar dev.morling.onebrc.CalculateAverage_vaidhy
+fi
diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 2c7528304..0ac5497ba 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -216,8 +216,20 @@ public long findSemi() {
             return i;
         }
 
-        public long findNewLine() {
-            this.hash = 0;
+        public long skipLine() {
+            for (long i = position; i < fileEnd; i++) {
+                byte ch = UNSAFE.getByte(i);
+                if (ch == 0x0a) {
+                    position = i + 1;
+                    return i;
+                }
+            }
+            position = fileEnd;
+            return fileEnd;
+        }
+
+        public long findTemperature() {
+            position += 3;
             for (long i = position; i < fileEnd; i++) {
                 byte ch = UNSAFE.getByte(i);
                 if (ch == 0x0a) {
@@ -237,7 +249,7 @@ private void worker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
         if (offset != 0) {
             if (lineStream.hasNext()) {
                 // Skip the first line.
-                lineStream.findNewLine();
+                lineStream.skipLine();
             }
             else {
                 // No lines then do nothing.
@@ -250,7 +262,7 @@ private void worker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
             long keySuffix = lineStream.suffix;
             int keyHash = lineStream.hash;
             long valueStartAddress = lineStream.position;
-            long valueEndAddress = lineStream.findNewLine();
+            long valueEndAddress = lineStream.findTemperature();
             int temperature = parseDouble(valueStartAddress, valueEndAddress);
             lineConsumer.process(keyStartAddress, keyEndAddress, keyHash, temperature, keySuffix);
         }

From e33dc0b6e2fe889f4a2beecde178260231965c32 Mon Sep 17 00:00:00 2001
From: Vaidhy Gopalan <vaidhy@gmail.com>
Date: Thu, 11 Jan 2024 22:56:46 -0800
Subject: [PATCH 27/64] Hand unrolled findSemi loop.

---
 .../onebrc/CalculateAverage_vaidhy.java       | 43 ++++++++++++++-----
 1 file changed, 33 insertions(+), 10 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 0ac5497ba..188e18ec0 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -20,6 +20,7 @@
 import java.io.IOException;
 import java.lang.foreign.Arena;
 import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
@@ -74,10 +75,6 @@ public HashEntry find(long startAddress, long endAddress, long suffix, int hash)
                             return entry;
                         }
                     }
-                    else {
-                        System.out.println("Broken - Entry : " + unsafeToString(entry.startAddress, entry.endAddress) +
-                                " Hash - " + unsafeToString(startAddress, endAddress));
-                    }
                 }
                 i++;
                 if (i == len) {
@@ -185,6 +182,7 @@ static class LineStream {
         private long position;
         private int hash;
         private long suffix;
+        byte[] b = new byte[4];
 
         public LineStream(FileService fileService, long offset, long chunkSize) {
             long fileStart = fileService.address();
@@ -200,16 +198,41 @@ public boolean hasNext() {
 
         public long findSemi() {
             int h = 0;
-            int s = 0;
+            long s = 0;
             long i = position;
-            for (; i < fileEnd; i++) {
-                byte myCh = UNSAFE.getByte(i);
-                if (myCh == 0x3B) {
+            while ((i + 3) < fileEnd) {
+                // Adding 16 as it is the offset for primitive arrays
+                ByteBuffer.wrap(b).putInt(UNSAFE.getInt(i));
+
+                if (b[3] == 0x3B) {
+                    break;
+                }
+                i++;
+                h = ((h << 5) - h) ^ b[3];
+                s = (s << 8) ^ b[3];
+
+                if (b[2] == 0x3B) {
+                    break;
+                }
+                i++;
+                h = ((h << 5) - h) ^ b[2];
+                s = (s << 8) ^ b[2];
+
+                if (b[1] == 0x3B) {
+                    break;
+                }
+                i++;
+                h = ((h << 5) - h) ^ b[1];
+                s = (s << 8) ^ b[1];
+
+                if (b[0] == 0x3B) {
                     break;
                 }
-                h = ((h << 5) - h) ^ myCh;
-                s = (s << 8) ^ myCh;
+                i++;
+                h = ((h << 5) - h) ^ b[0];
+                s = (s << 8) ^ b[0];
             }
+
             this.hash = h;
             this.suffix = s;
             position = i + 1;

From ce39a533fe01756b737474cc7e8a25a55a8a27c7 Mon Sep 17 00:00:00 2001
From: Anita S V <anitasvasu@gmail.com>
Date: Fri, 12 Jan 2024 01:42:52 -0800
Subject: [PATCH 28/64] Remove incorrect comments

---
 .../onebrc/CalculateAverage_vaidhy.java       | 21 -------------------
 1 file changed, 21 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 188e18ec0..5795077b3 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -168,13 +168,6 @@ interface FileService {
         this.reducer = reducer;
     }
 
-    /**
-     * Reads lines from a given character stream, hasNext() is always
-     * efficient, all work is done only in next().
-     */
-    // Space complexity: O(max line length) in next() call, structure is O(1)
-    // not counting charStream as it is only a reference, we will count that
-    // in worker space.
     static class LineStream {
         private final long fileEnd;
         private final long chunkEnd;
@@ -265,7 +258,6 @@ public long findTemperature() {
         }
     }
 
-    // Space complexity: O(scanSize) + O(max line length)
     private void worker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
         LineStream lineStream = new LineStream(fileService, offset, chunkSize);
 
@@ -291,8 +283,6 @@ private void worker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
         }
     }
 
-    // Space complexity: O(number of workers), not counting
-    // workers space assuming they are running in different hosts.
     public T master(long chunkSize, ExecutorService executor) {
         long len = fileService.length();
         List<Future<I>> summaries = new ArrayList<>();
@@ -320,8 +310,6 @@ public T master(long chunkSize, ExecutorService executor) {
         return reducer.apply(summariesDone);
     }
 
-    /// SAMPLE CANDIDATE CODE ENDS
-
     static class DiskFileService implements FileService {
         private final long fileSize;
         private final long mappedAddress;
@@ -352,15 +340,6 @@ private static class ChunkProcessorImpl implements MapReduce<PrimitiveHashMap> {
 
         @Override
         public void process(long keyStartAddress, long keyEndAddress, int hash, int temperature, long suffix) {
-            // int length = (int) (keyEndAddress - keyStartAddress);
-            // int alignedLength = (length >> 3) << 3;
-            // long alignedAddress = keyStartAddress + alignedLength;
-            // int tail = length & 7;
-            // long suffix = 0;
-            // if (tail != 0) {
-            // suffix = UNSAFE.getLong(alignedAddress) << ((8 - tail) * 8);
-            // }
-
             HashEntry entry = statistics.find(keyStartAddress, keyEndAddress, suffix, hash);
             if (entry == null) {
                 throw new IllegalStateException("Hash table too small :(");

From a5f41dba722b54730beb8a24ac0e6434c577bad2 Mon Sep 17 00:00:00 2001
From: Anita S V <anitasvasu@gmail.com>
Date: Fri, 12 Jan 2024 12:44:24 -0800
Subject: [PATCH 29/64] Taking care fo PR comments

---
 additional_build_step_vaidhy.sh | 21 ---------------------
 calculate_average_vaidhy.sh     | 14 ++++----------
 2 files changed, 4 insertions(+), 31 deletions(-)
 delete mode 100755 additional_build_step_vaidhy.sh

diff --git a/additional_build_step_vaidhy.sh b/additional_build_step_vaidhy.sh
deleted file mode 100755
index e8520df78..000000000
--- a/additional_build_step_vaidhy.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/sh
-#
-#  Copyright 2023 The original authors
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-source "$HOME/.sdkman/bin/sdkman-init.sh"
-sdk use java 21.0.1-graal 1>&2
-NATIVE_IMAGE_OPTS="--gc=epsilon -O3 -march=native --enable-preview"
-native-image $NATIVE_IMAGE_OPTS -cp target/average-1.0.0-SNAPSHOT.jar -o image_calculateaverage_vaidhy dev.morling.onebrc.CalculateAverage_vaidhy
diff --git a/calculate_average_vaidhy.sh b/calculate_average_vaidhy.sh
index 1c51dc3d1..969e53506 100755
--- a/calculate_average_vaidhy.sh
+++ b/calculate_average_vaidhy.sh
@@ -15,13 +15,7 @@
 #  limitations under the License.
 #
 
-if [ -f ./image_calculateaverage_vaidhy ]; then
-    echo "Picking up existing native image, delete the file to select JVM mode." 1>&2
-    time ./image_calculateaverage_vaidhy
-else
-    source "$HOME/.sdkman/bin/sdkman-init.sh"
-    sdk use java 21.0.1-graal 1>&2
-    JAVA_OPTS="--enable-preview"
-    echo "Choosing to run the app in JVM mode as no native image was found, use additional_build_step_vaidhy.sh to generate." 1>&2
-    time java $JAVA_OPTS --class-path target/average-1.0.0-SNAPSHOT.jar dev.morling.onebrc.CalculateAverage_vaidhy
-fi
+source "$HOME/.sdkman/bin/sdkman-init.sh"
+sdk use java 21.0.1-graal 1>&2
+JAVA_OPTS="--enable-preview"
+java $JAVA_OPTS --class-path target/average-1.0.0-SNAPSHOT.jar dev.morling.onebrc.CalculateAverage_vaidhy

From 12bee02c05b27ec764cff737e7194ac8320f2e88 Mon Sep 17 00:00:00 2001
From: Anita S V <anitasvasu@gmail.com>
Date: Fri, 12 Jan 2024 13:57:23 -0800
Subject: [PATCH 30/64] Add prepare script

---
 calculate_average_vaidhy.sh |  3 ---
 prepare_vaidhy.sh           | 20 ++++++++++++++++++++
 2 files changed, 20 insertions(+), 3 deletions(-)
 create mode 100755 prepare_vaidhy.sh

diff --git a/calculate_average_vaidhy.sh b/calculate_average_vaidhy.sh
index 969e53506..b220274b7 100755
--- a/calculate_average_vaidhy.sh
+++ b/calculate_average_vaidhy.sh
@@ -14,8 +14,5 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
-
-source "$HOME/.sdkman/bin/sdkman-init.sh"
-sdk use java 21.0.1-graal 1>&2
 JAVA_OPTS="--enable-preview"
 java $JAVA_OPTS --class-path target/average-1.0.0-SNAPSHOT.jar dev.morling.onebrc.CalculateAverage_vaidhy
diff --git a/prepare_vaidhy.sh b/prepare_vaidhy.sh
new file mode 100755
index 000000000..06b81c4dd
--- /dev/null
+++ b/prepare_vaidhy.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+#  Copyright 2023 The original authors
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+# Uncomment below to use sdk
+source "$HOME/.sdkman/bin/sdkman-init.sh"
+sdk use java 21.0.1-graal 1>&2

From 6d3004f31c0f4ff56de9d10f6fe752ae833202e7 Mon Sep 17 00:00:00 2001
From: Anita S V <anitasvasu@gmail.com>
Date: Fri, 12 Jan 2024 13:58:39 -0800
Subject: [PATCH 31/64] Missing header error fix

---
 calculate_average_vaidhy.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/calculate_average_vaidhy.sh b/calculate_average_vaidhy.sh
index b220274b7..ca204f806 100755
--- a/calculate_average_vaidhy.sh
+++ b/calculate_average_vaidhy.sh
@@ -14,5 +14,6 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+
 JAVA_OPTS="--enable-preview"
 java $JAVA_OPTS --class-path target/average-1.0.0-SNAPSHOT.jar dev.morling.onebrc.CalculateAverage_vaidhy

From eed255452c4b4366a2c70459e1a9167bd364d196 Mon Sep 17 00:00:00 2001
From: Anita S V <anitasvasu@gmail.com>
Date: Fri, 12 Jan 2024 13:59:17 -0800
Subject: [PATCH 32/64] remove wrong comment

---
 prepare_vaidhy.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/prepare_vaidhy.sh b/prepare_vaidhy.sh
index 06b81c4dd..f83a3ff69 100755
--- a/prepare_vaidhy.sh
+++ b/prepare_vaidhy.sh
@@ -15,6 +15,5 @@
 #  limitations under the License.
 #
 
-# Uncomment below to use sdk
 source "$HOME/.sdkman/bin/sdkman-init.sh"
 sdk use java 21.0.1-graal 1>&2

From a0f876ecf83e2ab2250d15b269fa6586dd07fc92 Mon Sep 17 00:00:00 2001
From: Anita S V <anitasvasu@gmail.com>
Date: Sat, 13 Jan 2024 13:08:38 -0800
Subject: [PATCH 33/64] Adding Long hashes, but not using

---
 .../onebrc/CalculateAverage_vaidhy.java       | 55 +++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 5795077b3..4d97c0506 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -100,6 +100,61 @@ private static boolean compareEntryKeys(long startAddress, long endAddress, Hash
 
     private static final String FILE = "./measurements.txt";
 
+    private static final long C1 = 0x87c37b91114253d5L;
+    private static final long C2 = 0x4cf5ad432745937fL;
+    private static final int R1 = 31;
+    private static final int R2 = 27;
+    private static final int R3 = 33;
+    private static final int M = 5;
+    private static final int N1 = 0x52dce729;
+
+    private static final long DEFAULT_SEED = 104729;
+
+    /*
+     * @vaidhy
+     *
+     * Powerful Murmur Hash:
+     *
+     * To use full MurMur strength:
+     *
+     * long hash = DEFAULT_SEED
+     * for (long value : list) {
+     * hash = murmurHash(hash, value);
+     * }
+     * hash = murmurHashFinalize(hash, list.size())
+     *
+     * To use a faster hash inspired by murmur:
+     *
+     * long hash = DEFAULT_SEED
+     * for (long value : list) {
+     * hash = simpleHash(hash, value);
+     * }
+     *
+     */
+    private static long murmurMix64(long hash) {
+        hash ^= (hash >>> 33);
+        hash *= 0xff51afd7ed558ccdL;
+        hash ^= (hash >>> 33);
+        hash *= 0xc4ceb9fe1a85ec53L;
+        hash ^= (hash >>> 33);
+        return hash;
+    }
+
+    private static long murmurHash(long hash, long nextData) {
+        hash = hash ^ Long.rotateLeft((nextData * C1), R1) * C2;
+        return Long.rotateLeft(hash, R2) * M + N1;
+    }
+
+    private static long murmurHashFinalize(long hash, int length) {
+        hash ^= length;
+        hash = murmurMix64(hash);
+        return hash;
+    }
+
+    private static long simpleHash(long hash, long nextData) {
+        return (hash ^ Long.rotateLeft((nextData * C1), R1)) * C2;
+    }
+
     private static Unsafe initUnsafe() {
         try {
             Field theUnsafe = Unsafe.class.getDeclaredField("theUnsafe");

From d5062d85319e63b1900d5152819a5f1297e8b80b Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Thu, 25 Jan 2024 01:43:56 -0800
Subject: [PATCH 34/64] Aligned Reads

---
 .../onebrc/CalculateAverage_vaidhy.java       | 407 ++++++++++--------
 1 file changed, 238 insertions(+), 169 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index be815efcb..7c913fcf2 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -21,6 +21,7 @@
 import java.lang.foreign.Arena;
 import java.lang.reflect.Field;
 import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 import java.nio.channels.FileChannel;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
@@ -38,8 +39,7 @@ public class CalculateAverage_vaidhy<I, T> {
     private static final class HashEntry {
         private long startAddress;
         private long endAddress;
-        private long suffix;
-        private int hash;
+        private long hash;
 
         IntSummaryStatistics value;
     }
@@ -48,6 +48,8 @@ private static class PrimitiveHashMap {
         private final HashEntry[] entries;
         private final int twoPow;
 
+        private int size = 0;
+
         PrimitiveHashMap(int twoPow) {
             this.twoPow = twoPow;
             this.entries = new HashEntry[1 << twoPow];
@@ -56,25 +58,31 @@ private static class PrimitiveHashMap {
             }
         }
 
-        public HashEntry find(long startAddress, long endAddress, long suffix, int hash) {
+        public HashEntry find(long startAddress, long endAddress, long hash) {
             int len = entries.length;
-            int i = (hash ^ (hash >> twoPow)) & (len - 1);
+            int h = Long.hashCode(hash);
+            int i = (h ^ (h >> twoPow)) & (len - 1);
 
             do {
                 HashEntry entry = entries[i];
                 if (entry.value == null) {
+                    entry.startAddress = startAddress;
+                    entry.endAddress = endAddress;
+                    entry.hash = hash;
+                    ++size;
                     return entry;
                 }
                 if (entry.hash == hash) {
-                    long entryLength = entry.endAddress - entry.startAddress;
-                    long lookupLength = endAddress - startAddress;
-                    if ((entryLength == lookupLength) && (entry.suffix == suffix)) {
-                        boolean found = compareEntryKeys(startAddress, endAddress, entry);
-
-                        if (found) {
-                            return entry;
-                        }
-                    }
+                    return entry;
+                    // long entryLength = entry.endAddress - entry.startAddress;
+                    // long lookupLength = endAddress - startAddress;
+                    // if ((entryLength == lookupLength)) {
+                    // boolean found = compareEntryKeys(startAddress, endAddress, entry);
+                    //
+                    // if (found) {
+                    // return entry;
+                    // }
+                    // }
                 }
                 i++;
                 if (i == len) {
@@ -200,7 +208,7 @@ private static int parseDouble(long startAddress, long endAddress) {
 
     interface MapReduce<I> {
 
-        void process(long keyStartAddress, long keyEndAddress, int hash, int temperature, long suffix);
+        void process(long keyStartAddress, long keyEndAddress, long hash, int temperature);
 
         I result();
     }
@@ -228,9 +236,11 @@ static class LineStream {
         private final long chunkEnd;
 
         private long position;
-        private int hash;
-        private long suffix;
-        byte[] b = new byte[4];
+        private long hash;
+
+        private final ByteBuffer buf = ByteBuffer
+                .allocate(8)
+                .order(ByteOrder.LITTLE_ENDIAN);
 
         public LineStream(FileService fileService, long offset, long chunkSize) {
             long fileStart = fileService.address();
@@ -245,46 +255,32 @@ public boolean hasNext() {
         }
 
         public long findSemi() {
-            int h = 0;
-            long s = 0;
-            long i = position;
-            while ((i + 3) < fileEnd) {
-                // Adding 16 as it is the offset for primitive arrays
-                ByteBuffer.wrap(b).putInt(UNSAFE.getInt(i));
-
-                if (b[3] == 0x3B) {
-                    break;
-                }
-                i++;
-                h = ((h << 5) - h) ^ b[3];
-                s = (s << 8) ^ b[3];
+            long h = DEFAULT_SEED;
+            buf.rewind();
 
-                if (b[2] == 0x3B) {
-                    break;
+            for (long i = position; i < fileEnd; i++) {
+                byte ch = UNSAFE.getByte(i);
+                if (ch == ';') {
+                    int discard = buf.remaining();
+                    buf.rewind();
+                    long nextData = (buf.getLong() << discard) >>> discard;
+                    hash = simpleHash(h, nextData);
+                    position = i + 1;
+                    return i;
                 }
-                i++;
-                h = ((h << 5) - h) ^ b[2];
-                s = (s << 8) ^ b[2];
-
-                if (b[1] == 0x3B) {
-                    break;
+                if (buf.hasRemaining()) {
+                    buf.put(ch);
                 }
-                i++;
-                h = ((h << 5) - h) ^ b[1];
-                s = (s << 8) ^ b[1];
-
-                if (b[0] == 0x3B) {
-                    break;
+                else {
+                    buf.flip();
+                    long nextData = buf.getLong();
+                    h = simpleHash(h, nextData);
+                    buf.rewind();
                 }
-                i++;
-                h = ((h << 5) - h) ^ b[0];
-                s = (s << 8) ^ b[0];
             }
-
-            this.hash = h;
-            this.suffix = s;
-            position = i + 1;
-            return i;
+            hash = h;
+            position = fileEnd;
+            return fileEnd;
         }
 
         public long skipLine() {
@@ -313,48 +309,73 @@ public long findTemperature() {
         }
     }
 
-    private long findNewLine(long data, int readOffset) {
-        data = data >>> (8 - readOffset);
+    private static final long START_BYTE_INDICATOR = 0x0101_0101_0101_0101L;
+    private static final long END_BYTE_INDICATOR = START_BYTE_INDICATOR << 7;
+
+    private static final long NEW_LINE_DETECTION = START_BYTE_INDICATOR * '\n';
+
+    private static final long SEMI_DETECTION = START_BYTE_INDICATOR * ';';
+
+    private static final long ALL_ONES = 0xffff_ffff_ffff_ffffL;
+
+    private int findByte(long data, long pattern, int readOffset) {
+        data >>>= (readOffset << 3);
+        long match = data ^ pattern;
+        long mask = (match - START_BYTE_INDICATOR) & ((~match) & END_BYTE_INDICATOR);
+
+        if (mask == 0) {
+            // Not Found
+            return -1;
+        }
+        else {
+            // Found
+            return readOffset + (Long.numberOfTrailingZeros(mask) >>> 3);
+        }
 
     }
 
+    private int findSemi(long data, int readOffset) {
+        return findByte(data, SEMI_DETECTION, readOffset);
+    }
 
+    private int findNewLine(long data, int readOffset) {
+        return findByte(data, NEW_LINE_DETECTION, readOffset);
+    }
 
     private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
-        long chunkEnd = offset + chunkSize;
-        long newRecordStart = offset;
-        long position = offset;
+        long fileStart = fileService.address();
+        long chunkEnd = fileStart + offset + chunkSize;
+        long newRecordStart = fileStart + offset;
+        long position = fileStart + offset;
+        long fileEnd = fileStart + fileService.length();
 
-        int nextReadOffset = -1;
-        int excessBytes = 0;
+        int nextReadOffset = 0;
 
-        long data;
-        long prevData = 0;
+        long data = UNSAFE.getLong(position);
 
         if (offset != 0) {
+            boolean foundNewLine = false;
             for (; position < chunkEnd; position += 8) {
                 data = UNSAFE.getLong(position);
-                int newLinePosition = findNewLine(data, 0);
+                int newLinePosition = findNewLine(data, nextReadOffset);
                 if (newLinePosition != -1) {
                     newRecordStart = position + newLinePosition + 1;
                     nextReadOffset = newLinePosition + 1;
 
                     if (nextReadOffset == 8) {
                         position += 8;
-                        prevData = data;
+                        nextReadOffset = 0;
                         data = UNSAFE.getLong(position);
-                    } else {
-                        excessBytes = nextReadOffset;
                     }
+                    foundNewLine = true;
                     break;
                 }
             }
-            if (nextReadOffset == -1) {
+            if (!foundNewLine) {
                 return;
             }
         }
 
-
         boolean newLineToken = false;
         // false means looking for semi Colon
         // true means looking for new line.
@@ -362,8 +383,8 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
         long stationEnd = offset;
 
         long hash = DEFAULT_SEED;
-
-        long suffix = 0;
+        long prevRelevant = 0;
+        int prevBytes = 0;
 
         while (true) {
             if (newLineToken) {
@@ -372,35 +393,34 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
                     nextReadOffset = 0;
                     position += 8;
                     data = UNSAFE.getLong(position);
-                    continue;
-                } else {
-
+                }
+                else {
                     long temperatureEnd = position + newLinePosition;
-
-                    // TODO:
-                    // station = [newRecordStart, stationEnd )
-                    // temperature = [stationEnd + 1, temperatureEnd )
-                    // parseDouble
-                    // insert
                     int temperature = parseDouble(stationEnd + 1, temperatureEnd);
-                    lineConsumer.process(newRecordStart, stationEnd, hash, temperature, suffix);
+                    // System.out.println("Big worker!");
+                    lineConsumer.process(newRecordStart, stationEnd, hash, temperature);
                     newLineToken = false;
 
                     nextReadOffset = newLinePosition + 1;
                     newRecordStart = temperatureEnd + 1;
 
-                    hash = DEFAULT_SEED;
-
-                    if (position >= chunkEnd) {
+                    if (newRecordStart > chunkEnd || newRecordStart >= fileEnd) {
                         break;
                     }
+
+                    hash = DEFAULT_SEED;
+
+                    prevRelevant = 0;
+                    prevBytes = 0;
+
                     if (nextReadOffset == 8) {
                         nextReadOffset = 0;
                         position += 8;
                         data = UNSAFE.getLong(position);
                     }
                 }
-            } else {
+            }
+            else {
                 int semiPosition = findSemi(data, nextReadOffset);
 
                 // excessBytes = 5
@@ -408,40 +428,83 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
 
                 // nextData = (nextData << excessBytes) | (data <<< (8 - excessBytes));
 
-                long prevRelevant = prevData & (0xffff_ffffL << excessBytes);
-                // prevRelevant = aaa0_0000
+                // prevRelevant = 0000_0aaa
+
+                // nextReadOffset = 2 ( 8 - NRO = 6 useful)
+                // prevBytes = 3 (6 useful + 3 available = 9 - meaning 1 extra)
 
                 if (semiPosition == -1) {
-                    // currentData = bbbb_bbbb
-                    long currRelevant = data >>> (8 - excessBytes);
-                    // currRelevant = 000b_bbbb
-                    long toHash = prevRelevant | currRelevant;
-                    // toHash = sssb_bbbb
+                    long currRelevant = data >>> (nextReadOffset << 3);
+                    currRelevant <<= (prevBytes << 3);
+
+                    prevRelevant = prevRelevant | currRelevant;
+                    int newPrevBytes = prevBytes + (8 - nextReadOffset);
 
-                    hash = simpleHash(hash, toHash);
+                    if (newPrevBytes >= 8) {
+                        // System.out.println(unsafeToString(position - prevBytes, position + 8 - prevBytes));
+                        // System.out.println(Long.toHexString(prevRelevant));
+                        hash = simpleHash(hash, prevRelevant);
+
+                        prevBytes = (newPrevBytes - 8);
+                        if (prevBytes != 0) {
+                            prevRelevant = (data >>> ((8 - prevBytes) << 3));
+                        }
+                        else {
+                            prevRelevant = 0;
+                        }
+                    }
+                    else {
+                        prevBytes = newPrevBytes;
+                    }
 
                     nextReadOffset = 0;
                     position += 8;
                     data = UNSAFE.getLong(position);
-                } else {
-                    // currentData = b;xx_xxxx
-                    long currRelevant = data >>> (8 - excessBytes);
-                    // currRelevant = 000b_;xxx
-                    currRelevant &= (0xffff_ffff << (8 - semiPosition));
+                }
+                else {
+                    // currentData = xxxx_x;aaN
+                    if (semiPosition != 0) {
+                        long currRelevant = (data & (ALL_ONES >>> ((8 - semiPosition) << 3))) >>> (nextReadOffset << 3);
+                        // 0000_00aa
 
-                    long toHash = prevRelevant | currRelevant;
-                    // toHash = sssb_bbbb
+                        // 0aaa_0000;
+                        long currUsable = currRelevant << (prevBytes << 3);
 
-                    nextData &= ((0xfffff_ffffL) << (8 - (excessBytes + semiPosition)));
-                    suffix = nextData;
+                        long toHash = prevRelevant | currUsable;
 
-                    hash = simpleHash(hash, nextData);
+                        // System.out.println(unsafeToString(position - prevBytes, position + 8 - prevBytes));
+                        // System.out.println(Long.toHexString(toHash));
+                        //
+                        // 0aaa_bbbb;
 
+                        hash = simpleHash(hash, toHash);
+                        // if (toHash == 0x6f506b696e7661a0L) {
+                        // System.out.println("Debug");
+                        // }
+
+                        int newPrevBytes = prevBytes + (semiPosition - nextReadOffset);
+                        if (newPrevBytes > 8) {
+
+                            long remaining = currRelevant >>> ((8 - prevBytes) << 3);
+                            hash = simpleHash(hash, remaining);
+                        }
+                    }
+                    else {
+                        hash = simpleHash(hash, prevRelevant);
+                    }
+
+                    prevRelevant = 0;
+                    prevBytes = 0;
 
                     stationEnd = position + semiPosition;
                     nextReadOffset = semiPosition + 1;
                     newLineToken = true;
 
+                    // String key = unsafeToString(newRecordStart, stationEnd);
+                    // if (key.equals("id4058")) {
+                    // System.out.println(key + " hash: " + hash);
+                    // }
+
                     if (nextReadOffset == 8) {
                         nextReadOffset = 0;
                         position += 8;
@@ -450,42 +513,36 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
 
                 }
             }
-//
-//            long semiPosition = findSemi(data, newLinePosition);
-//            if (semiPosition != 0) {
-//                pointerEnd = position + semiPosition;
-//
-//                newLinePosition = findNewLine(data, semiPosition);
-//
-//            }
+            //
+            // long semiPosition = findSemi(data, newLinePosition);
+            // if (semiPosition != 0) {
+            // pointerEnd = position + semiPosition;
+            //
+            // newLinePosition = findNewLine(data, semiPosition);
+            //
+            // }
         }
 
-
-
-
-
-
-
-//        if (offset != 0) {
-//            if (lineStream.hasNext()) {
-//                // Skip the first line.
-//                lineStream.skipLine();
-//            }
-//            else {
-//                // No lines then do nothing.
-//                return;
-//            }
-//        }
-//        while (lineStream.hasNext()) {
-//            long keyStartAddress = lineStream.position;
-//            long keyEndAddress = lineStream.findSemi();
-//            long keySuffix = lineStream.suffix;
-//            int keyHash = lineStream.hash;
-//            long valueStartAddress = lineStream.position;
-//            long valueEndAddress = lineStream.findTemperature();
-//            int temperature = parseDouble(valueStartAddress, valueEndAddress);
-//            lineConsumer.process(keyStartAddress, keyEndAddress, keyHash, temperature, keySuffix);
-//        }
+        // if (offset != 0) {
+        // if (lineStream.hasNext()) {
+        // // Skip the first line.
+        // lineStream.skipLine();
+        // }
+        // else {
+        // // No lines then do nothing.
+        // return;
+        // }
+        // }
+        // while (lineStream.hasNext()) {
+        // long keyStartAddress = lineStream.position;
+        // long keyEndAddress = lineStream.findSemi();
+        // long keySuffix = lineStream.suffix;
+        // int keyHash = lineStream.hash;
+        // long valueStartAddress = lineStream.position;
+        // long valueEndAddress = lineStream.findTemperature();
+        // int temperature = parseDouble(valueStartAddress, valueEndAddress);
+        // lineConsumer.process(keyStartAddress, keyEndAddress, keyHash, temperature, keySuffix);
+        // }
 
     }
 
@@ -505,47 +562,56 @@ private void smallWorker(long offset, long chunkSize, MapReduce<I> lineConsumer)
         while (lineStream.hasNext()) {
             long keyStartAddress = lineStream.position;
             long keyEndAddress = lineStream.findSemi();
-            long keySuffix = lineStream.suffix;
-            int keyHash = lineStream.hash;
+            long keyHash = lineStream.hash;
             long valueStartAddress = lineStream.position;
             long valueEndAddress = lineStream.findTemperature();
             int temperature = parseDouble(valueStartAddress, valueEndAddress);
-            lineConsumer.process(keyStartAddress, keyEndAddress, keyHash, temperature, keySuffix);
+            // System.out.println("Small worker!");
+            lineConsumer.process(keyStartAddress, keyEndAddress, keyHash, temperature);
         }
     }
 
+    // file size = 7
+    // (0,0) (0,0) small chunk= (0,7)
+    // a;0.1\n
+
     public T master(int shards, ExecutorService executor) {
+        List<Future<I>> summaries = new ArrayList<>();
         long len = fileService.length();
 
-        long fileEndAligned = (len - 128) & 0xffff_ffff_ffff_fff8L;
-        long bigChunk = Math.floorDiv(fileEndAligned, shards);
-        long bigChunkReAlign = bigChunk & 0xffff_ffff_ffff_fff8L;
-
-        long smallChunkStart = bigChunkReAlign * shards;
-        long smallChunkSize = len - smallChunkStart;
-
-        System.out.println(UNSAFE.addressSize());
-        System.out.println(fileService.address() % UNSAFE.addressSize());
-
-        List<Future<I>> summaries = new ArrayList<>();
+        if (len > 128) {
+            long bigChunk = Math.floorDiv(len, shards);
+            long bigChunkReAlign = bigChunk & 0xffff_ffff_ffff_fff8L;
+
+            long smallChunkStart = bigChunkReAlign * shards;
+            long smallChunkSize = len - smallChunkStart;
+
+            for (long offset = 0; offset < smallChunkStart; offset += bigChunkReAlign) {
+                MapReduce<I> mr = chunkProcessCreator.get();
+                final long transferOffset = offset;
+                Future<I> task = executor.submit(() -> {
+                    bigWorker(transferOffset, bigChunkReAlign, mr);
+                    return mr.result();
+                });
+                summaries.add(task);
+            }
 
-        for (long offset = 0; offset < fileEndAligned; offset += bigChunkReAlign) {
-            MapReduce<I> mr = chunkProcessCreator.get();
-            final long transferOffset = offset;
-            Future<I> task = executor.submit(() -> {
-                bigWorker(transferOffset, bigChunkReAlign, mr);
-                return mr.result();
+            MapReduce<I> mrLast = chunkProcessCreator.get();
+            Future<I> lastTask = executor.submit(() -> {
+                smallWorker(smallChunkStart, smallChunkSize - 1, mrLast);
+                return mrLast.result();
             });
-            summaries.add(task);
+            summaries.add(lastTask);
         }
+        else {
 
-        MapReduce<I> mrLast = chunkProcessCreator.get();
-        Future<I> lastTask = executor.submit(() -> {
-            smallWorker(smallChunkStart, smallChunkSize - 1, mrLast);
-            return mrLast.result();
-        });
-        summaries.add(lastTask);
-
+            MapReduce<I> mrLast = chunkProcessCreator.get();
+            Future<I> lastTask = executor.submit(() -> {
+                smallWorker(0, len - 1, mrLast);
+                return mrLast.result();
+            });
+            summaries.add(lastTask);
+        }
 
         List<I> summariesDone = summaries.stream()
                 .map(task -> {
@@ -589,19 +655,22 @@ private static class ChunkProcessorImpl implements MapReduce<PrimitiveHashMap> {
         private final PrimitiveHashMap statistics = new PrimitiveHashMap(14);
 
         @Override
-        public void process(long keyStartAddress, long keyEndAddress, int hash, int temperature, long suffix) {
-            HashEntry entry = statistics.find(keyStartAddress, keyEndAddress, suffix, hash);
+        public void process(long keyStartAddress, long keyEndAddress, long hash, int temperature) {
+            // System.out.println(unsafeToString(keyStartAddress, keyEndAddress) + " --> " + temperature + " hash: " + hash);
+            HashEntry entry = statistics.find(keyStartAddress, keyEndAddress, hash);
             if (entry == null) {
                 throw new IllegalStateException("Hash table too small :(");
             }
             if (entry.value == null) {
-                entry.startAddress = keyStartAddress;
-                entry.endAddress = keyEndAddress;
-                entry.suffix = suffix;
-                entry.hash = hash;
                 entry.value = new IntSummaryStatistics();
             }
             entry.value.accept(temperature);
+
+            // IntSummaryStatistics stats = verify.computeIfAbsent(key, ignore -> new IntSummaryStatistics());
+            // stats.accept(temperature);
+            // if (stats.getCount() != entry.value.getCount()) {
+            // System.out.println("Trouble");
+            // }
         }
 
         @Override

From 4239be4807ef199deb08c05e65c04e06963d048b Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Thu, 25 Jan 2024 01:49:24 -0800
Subject: [PATCH 35/64] Put the hash equals back

---
 .../onebrc/CalculateAverage_vaidhy.java       | 25 +++++++++++--------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 7c913fcf2..598c9ae45 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -73,16 +73,14 @@ public HashEntry find(long startAddress, long endAddress, long hash) {
                     return entry;
                 }
                 if (entry.hash == hash) {
-                    return entry;
-                    // long entryLength = entry.endAddress - entry.startAddress;
-                    // long lookupLength = endAddress - startAddress;
-                    // if ((entryLength == lookupLength)) {
-                    // boolean found = compareEntryKeys(startAddress, endAddress, entry);
-                    //
-                    // if (found) {
-                    // return entry;
-                    // }
-                    // }
+                    long entryLength = entry.endAddress - entry.startAddress;
+                    long lookupLength = endAddress - startAddress;
+                    if ((entryLength == lookupLength)) {
+                        boolean found = compareEntryKeys(startAddress, endAddress, entry);
+                        if (found) {
+                            return entry;
+                        }
+                    }
                 }
                 i++;
                 if (i == len) {
@@ -102,6 +100,13 @@ private static boolean compareEntryKeys(long startAddress, long endAddress, Hash
                 }
                 entryIndex += 8;
             }
+            for (; lookupIndex < endAddress; lookupIndex++) {
+                if (UNSAFE.getByte(entryIndex) != UNSAFE.getByte(lookupIndex)) {
+                    return false;
+                }
+                entryIndex++;
+            }
+
             return true;
         }
     }

From 6d9dce9ca9ff73ab0cd2ad16f122cfaf8aaa870f Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Thu, 25 Jan 2024 02:07:16 -0800
Subject: [PATCH 36/64] minor improvement to work it more efficiently on small
 files

---
 .../onebrc/CalculateAverage_vaidhy.java       | 45 +++++++++++--------
 1 file changed, 27 insertions(+), 18 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 598c9ae45..8ad9f0f49 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -41,6 +41,7 @@ private static final class HashEntry {
         private long endAddress;
         private long hash;
 
+        private int next;
         IntSummaryStatistics value;
     }
 
@@ -48,7 +49,7 @@ private static class PrimitiveHashMap {
         private final HashEntry[] entries;
         private final int twoPow;
 
-        private int size = 0;
+        private int next = -1;
 
         PrimitiveHashMap(int twoPow) {
             this.twoPow = twoPow;
@@ -69,7 +70,8 @@ public HashEntry find(long startAddress, long endAddress, long hash) {
                     entry.startAddress = startAddress;
                     entry.endAddress = endAddress;
                     entry.hash = hash;
-                    ++size;
+                    entry.next = next;
+                    this.next = i;
                     return entry;
                 }
                 if (entry.hash == hash) {
@@ -109,6 +111,24 @@ private static boolean compareEntryKeys(long startAddress, long endAddress, Hash
 
             return true;
         }
+
+        public Iterable<HashEntry> entrySet() {
+            return () -> new Iterator<>() {
+                int scan = next;
+
+                @Override
+                public boolean hasNext() {
+                    return scan != -1;
+                }
+
+                @Override
+                public HashEntry next() {
+                    HashEntry entry = entries[scan];
+                    scan = entry.next;
+                    return entry;
+                }
+            };
+        }
     }
 
     private static final String FILE = "./measurements.txt";
@@ -165,6 +185,7 @@ private static long murmurHashFinalize(long hash, int length) {
     }
 
     private static long simpleHash(long hash, long nextData) {
+        // return hash ^ nextData;
         return (hash ^ Long.rotateLeft((nextData * C1), R1)) * C2;
     }
 
@@ -391,6 +412,8 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
         long prevRelevant = 0;
         int prevBytes = 0;
 
+        long crossPoint = Math.min(chunkEnd + 1, fileEnd);
+
         while (true) {
             if (newLineToken) {
                 int newLinePosition = findNewLine(data, nextReadOffset);
@@ -409,15 +432,12 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
                     nextReadOffset = newLinePosition + 1;
                     newRecordStart = temperatureEnd + 1;
 
-                    if (newRecordStart > chunkEnd || newRecordStart >= fileEnd) {
+                    if (newRecordStart >= crossPoint) {
                         break;
                     }
 
                     hash = DEFAULT_SEED;
 
-                    prevRelevant = 0;
-                    prevBytes = 0;
-
                     if (nextReadOffset == 8) {
                         nextReadOffset = 0;
                         position += 8;
@@ -428,16 +448,6 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
             else {
                 int semiPosition = findSemi(data, nextReadOffset);
 
-                // excessBytes = 5
-                // prevData = aaax_xxxx
-
-                // nextData = (nextData << excessBytes) | (data <<< (8 - excessBytes));
-
-                // prevRelevant = 0000_0aaa
-
-                // nextReadOffset = 2 ( 8 - NRO = 6 useful)
-                // prevBytes = 3 (6 useful + 3 available = 9 - meaning 1 extra)
-
                 if (semiPosition == -1) {
                     long currRelevant = data >>> (nextReadOffset << 3);
                     currRelevant <<= (prevBytes << 3);
@@ -489,7 +499,6 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
 
                         int newPrevBytes = prevBytes + (semiPosition - nextReadOffset);
                         if (newPrevBytes > 8) {
-
                             long remaining = currRelevant >>> ((8 - prevBytes) << 3);
                             hash = simpleHash(hash, remaining);
                         }
@@ -718,7 +727,7 @@ private static Map<String, IntSummaryStatistics> combineOutputs(
 
         Map<String, IntSummaryStatistics> output = new HashMap<>(10000);
         for (PrimitiveHashMap map : list) {
-            for (HashEntry entry : map.entries) {
+            for (HashEntry entry : map.entrySet()) {
                 if (entry.value != null) {
                     String keyStr = unsafeToString(entry.startAddress, entry.endAddress);
 

From 3cc95556d02e9d645154e9c1c054770d44e4325c Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Thu, 25 Jan 2024 11:25:54 -0800
Subject: [PATCH 37/64] checking why bit conversion is not working

---
 .../onebrc/CalculateAverage_vaidhy.java       | 136 ++++++------------
 1 file changed, 44 insertions(+), 92 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 8ad9f0f49..2a863b07b 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -136,54 +136,8 @@ public HashEntry next() {
     private static final long C1 = 0x87c37b91114253d5L;
     private static final long C2 = 0x4cf5ad432745937fL;
     private static final int R1 = 31;
-    private static final int R2 = 27;
-    private static final int R3 = 33;
-    private static final int M = 5;
-    private static final int N1 = 0x52dce729;
-
     private static final long DEFAULT_SEED = 104729;
 
-    /*
-     * @vaidhy
-     *
-     * Powerful Murmur Hash:
-     *
-     * To use full MurMur strength:
-     *
-     * long hash = DEFAULT_SEED
-     * for (long value : list) {
-     * hash = murmurHash(hash, value);
-     * }
-     * hash = murmurHashFinalize(hash, list.size())
-     *
-     * To use a faster hash inspired by murmur:
-     *
-     * long hash = DEFAULT_SEED
-     * for (long value : list) {
-     * hash = simpleHash(hash, value);
-     * }
-     *
-     */
-    private static long murmurMix64(long hash) {
-        hash ^= (hash >>> 33);
-        hash *= 0xff51afd7ed558ccdL;
-        hash ^= (hash >>> 33);
-        hash *= 0xc4ceb9fe1a85ec53L;
-        hash ^= (hash >>> 33);
-        return hash;
-    }
-
-    private static long murmurHash(long hash, long nextData) {
-        hash = hash ^ Long.rotateLeft((nextData * C1), R1) * C2;
-        return Long.rotateLeft(hash, R2) * M + N1;
-    }
-
-    private static long murmurHashFinalize(long hash, int length) {
-        hash ^= length;
-        hash = murmurMix64(hash);
-        return hash;
-    }
-
     private static long simpleHash(long hash, long nextData) {
         // return hash ^ nextData;
         return (hash ^ Long.rotateLeft((nextData * C1), R1)) * C2;
@@ -344,8 +298,8 @@ public long findTemperature() {
 
     private static final long ALL_ONES = 0xffff_ffff_ffff_ffffL;
 
-    private int findByte(long data, long pattern, int readOffset) {
-        data >>>= (readOffset << 3);
+    private int findByte(long data, long pattern, int readOffsetBits) {
+        data >>>= readOffsetBits;
         long match = data ^ pattern;
         long mask = (match - START_BYTE_INDICATOR) & ((~match) & END_BYTE_INDICATOR);
 
@@ -355,17 +309,16 @@ private int findByte(long data, long pattern, int readOffset) {
         }
         else {
             // Found
-            return readOffset + (Long.numberOfTrailingZeros(mask) >>> 3);
+            return readOffsetBits + (Long.numberOfTrailingZeros(mask) >>> 3);
         }
-
     }
 
-    private int findSemi(long data, int readOffset) {
-        return findByte(data, SEMI_DETECTION, readOffset);
+    private int findSemi(long data, int readOffsetBits) {
+        return findByte(data, SEMI_DETECTION, readOffsetBits);
     }
 
-    private int findNewLine(long data, int readOffset) {
-        return findByte(data, NEW_LINE_DETECTION, readOffset);
+    private int findNewLine(long data, int readOffsetBits) {
+        return findByte(data, NEW_LINE_DETECTION, readOffsetBits);
     }
 
     private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
@@ -375,7 +328,7 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
         long position = fileStart + offset;
         long fileEnd = fileStart + fileService.length();
 
-        int nextReadOffset = 0;
+        int nextReadOffsetBits = 0;
 
         long data = UNSAFE.getLong(position);
 
@@ -383,14 +336,14 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
             boolean foundNewLine = false;
             for (; position < chunkEnd; position += 8) {
                 data = UNSAFE.getLong(position);
-                int newLinePosition = findNewLine(data, nextReadOffset);
-                if (newLinePosition != -1) {
-                    newRecordStart = position + newLinePosition + 1;
-                    nextReadOffset = newLinePosition + 1;
+                int newLinePositionBits = findNewLine(data, nextReadOffsetBits);
+                if (newLinePositionBits != -1) {
+                    nextReadOffsetBits = newLinePositionBits + 8;
+                    newRecordStart = position + (nextReadOffsetBits >>> 3);
 
-                    if (nextReadOffset == 8) {
+                    if (nextReadOffsetBits == 64) {
                         position += 8;
-                        nextReadOffset = 0;
+                        nextReadOffsetBits = 0;
                         data = UNSAFE.getLong(position);
                     }
                     foundNewLine = true;
@@ -410,26 +363,26 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
 
         long hash = DEFAULT_SEED;
         long prevRelevant = 0;
-        int prevBytes = 0;
+        int prevBits = 0;
 
         long crossPoint = Math.min(chunkEnd + 1, fileEnd);
 
         while (true) {
             if (newLineToken) {
-                int newLinePosition = findNewLine(data, nextReadOffset);
-                if (newLinePosition == -1) {
-                    nextReadOffset = 0;
+                int newLinePositionBits = findNewLine(data, nextReadOffsetBits);
+                if (newLinePositionBits == -1) {
+                    nextReadOffsetBits = 0;
                     position += 8;
                     data = UNSAFE.getLong(position);
                 }
                 else {
-                    long temperatureEnd = position + newLinePosition;
+                    long temperatureEnd = position + (newLinePositionBits >>> 3);
                     int temperature = parseDouble(stationEnd + 1, temperatureEnd);
                     // System.out.println("Big worker!");
                     lineConsumer.process(newRecordStart, stationEnd, hash, temperature);
                     newLineToken = false;
 
-                    nextReadOffset = newLinePosition + 1;
+                    nextReadOffsetBits = newLinePositionBits + 8;
                     newRecordStart = temperatureEnd + 1;
 
                     if (newRecordStart >= crossPoint) {
@@ -438,52 +391,52 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
 
                     hash = DEFAULT_SEED;
 
-                    if (nextReadOffset == 8) {
-                        nextReadOffset = 0;
+                    if (nextReadOffsetBits == 64) {
+                        nextReadOffsetBits = 0;
                         position += 8;
                         data = UNSAFE.getLong(position);
                     }
                 }
             }
             else {
-                int semiPosition = findSemi(data, nextReadOffset);
+                int semiPositionBits = findSemi(data, nextReadOffsetBits);
 
-                if (semiPosition == -1) {
-                    long currRelevant = data >>> (nextReadOffset << 3);
-                    currRelevant <<= (prevBytes << 3);
+                if (semiPositionBits == -1) {
+                    long currRelevant = data >>> nextReadOffsetBits;
+                    currRelevant <<= prevBits;
 
                     prevRelevant = prevRelevant | currRelevant;
-                    int newPrevBytes = prevBytes + (8 - nextReadOffset);
+                    int newPrevBits = prevBits + (64 - nextReadOffsetBits);
 
-                    if (newPrevBytes >= 8) {
+                    if (newPrevBits >= 64) {
                         // System.out.println(unsafeToString(position - prevBytes, position + 8 - prevBytes));
                         // System.out.println(Long.toHexString(prevRelevant));
                         hash = simpleHash(hash, prevRelevant);
 
-                        prevBytes = (newPrevBytes - 8);
-                        if (prevBytes != 0) {
-                            prevRelevant = (data >>> ((8 - prevBytes) << 3));
+                        prevBits = (newPrevBits - 64);
+                        if (prevBits != 0) {
+                            prevRelevant = (data >>> (64 - prevBits));
                         }
                         else {
                             prevRelevant = 0;
                         }
                     }
                     else {
-                        prevBytes = newPrevBytes;
+                        prevBits = newPrevBits;
                     }
 
-                    nextReadOffset = 0;
+                    nextReadOffsetBits = 0;
                     position += 8;
                     data = UNSAFE.getLong(position);
                 }
                 else {
                     // currentData = xxxx_x;aaN
-                    if (semiPosition != 0) {
-                        long currRelevant = (data & (ALL_ONES >>> ((8 - semiPosition) << 3))) >>> (nextReadOffset << 3);
+                    if (semiPositionBits != 0) {
+                        long currRelevant = (data & (ALL_ONES >>> (64 - semiPositionBits))) >>> nextReadOffsetBits;
                         // 0000_00aa
 
                         // 0aaa_0000;
-                        long currUsable = currRelevant << (prevBytes << 3);
+                        long currUsable = currRelevant << prevBits;
 
                         long toHash = prevRelevant | currUsable;
 
@@ -497,9 +450,9 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
                         // System.out.println("Debug");
                         // }
 
-                        int newPrevBytes = prevBytes + (semiPosition - nextReadOffset);
-                        if (newPrevBytes > 8) {
-                            long remaining = currRelevant >>> ((8 - prevBytes) << 3);
+                        int newPrevBits = prevBits + (semiPositionBits - nextReadOffsetBits);
+                        if (newPrevBits > 64) {
+                            long remaining = currRelevant >>> (64 - prevBits);
                             hash = simpleHash(hash, remaining);
                         }
                     }
@@ -508,10 +461,10 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
                     }
 
                     prevRelevant = 0;
-                    prevBytes = 0;
+                    prevBits = 0;
 
-                    stationEnd = position + semiPosition;
-                    nextReadOffset = semiPosition + 1;
+                    stationEnd = position + (semiPositionBits >>> 3);
+                    nextReadOffsetBits = semiPositionBits + 8;
                     newLineToken = true;
 
                     // String key = unsafeToString(newRecordStart, stationEnd);
@@ -519,12 +472,11 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
                     // System.out.println(key + " hash: " + hash);
                     // }
 
-                    if (nextReadOffset == 8) {
-                        nextReadOffset = 0;
+                    if (nextReadOffsetBits == 64) {
+                        nextReadOffsetBits = 0;
                         position += 8;
                         data = UNSAFE.getLong(position);
                     }
-
                 }
             }
             //

From e792886ed55f808c42e5e5f4f128975c1b8601ec Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Thu, 25 Jan 2024 11:28:35 -0800
Subject: [PATCH 38/64] one bug fix, still not there

---
 src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 2a863b07b..49ae22922 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -309,7 +309,7 @@ private int findByte(long data, long pattern, int readOffsetBits) {
         }
         else {
             // Found
-            return readOffsetBits + (Long.numberOfTrailingZeros(mask) >>> 3);
+            return readOffsetBits + Long.numberOfTrailingZeros(mask);
         }
     }
 

From 89fdf499394bccb572445c40181d42d8e34dd718 Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Thu, 25 Jan 2024 11:42:26 -0800
Subject: [PATCH 39/64] Ah found the last bug

---
 src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 49ae22922..99206be8f 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -309,7 +309,8 @@ private int findByte(long data, long pattern, int readOffsetBits) {
         }
         else {
             // Found
-            return readOffsetBits + Long.numberOfTrailingZeros(mask);
+            int trailingZeroes = Long.numberOfTrailingZeros(mask) - 7;
+            return readOffsetBits + trailingZeroes;
         }
     }
 

From c6589f79e89708b04c16e7ba2cd0dbaa9618fe23 Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Thu, 25 Jan 2024 11:44:19 -0800
Subject: [PATCH 40/64] remove commented out code

---
 .../onebrc/CalculateAverage_vaidhy.java       | 55 -------------------
 1 file changed, 55 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 99206be8f..ade3fbc34 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -40,7 +40,6 @@ private static final class HashEntry {
         private long startAddress;
         private long endAddress;
         private long hash;
-
         private int next;
         IntSummaryStatistics value;
     }
@@ -48,7 +47,6 @@ private static final class HashEntry {
     private static class PrimitiveHashMap {
         private final HashEntry[] entries;
         private final int twoPow;
-
         private int next = -1;
 
         PrimitiveHashMap(int twoPow) {
@@ -379,7 +377,6 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
                 else {
                     long temperatureEnd = position + (newLinePositionBits >>> 3);
                     int temperature = parseDouble(stationEnd + 1, temperatureEnd);
-                    // System.out.println("Big worker!");
                     lineConsumer.process(newRecordStart, stationEnd, hash, temperature);
                     newLineToken = false;
 
@@ -410,8 +407,6 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
                     int newPrevBits = prevBits + (64 - nextReadOffsetBits);
 
                     if (newPrevBits >= 64) {
-                        // System.out.println(unsafeToString(position - prevBytes, position + 8 - prevBytes));
-                        // System.out.println(Long.toHexString(prevRelevant));
                         hash = simpleHash(hash, prevRelevant);
 
                         prevBits = (newPrevBits - 64);
@@ -441,15 +436,7 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
 
                         long toHash = prevRelevant | currUsable;
 
-                        // System.out.println(unsafeToString(position - prevBytes, position + 8 - prevBytes));
-                        // System.out.println(Long.toHexString(toHash));
-                        //
-                        // 0aaa_bbbb;
-
                         hash = simpleHash(hash, toHash);
-                        // if (toHash == 0x6f506b696e7661a0L) {
-                        // System.out.println("Debug");
-                        // }
 
                         int newPrevBits = prevBits + (semiPositionBits - nextReadOffsetBits);
                         if (newPrevBits > 64) {
@@ -468,11 +455,6 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
                     nextReadOffsetBits = semiPositionBits + 8;
                     newLineToken = true;
 
-                    // String key = unsafeToString(newRecordStart, stationEnd);
-                    // if (key.equals("id4058")) {
-                    // System.out.println(key + " hash: " + hash);
-                    // }
-
                     if (nextReadOffsetBits == 64) {
                         nextReadOffsetBits = 0;
                         position += 8;
@@ -480,37 +462,7 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
                     }
                 }
             }
-            //
-            // long semiPosition = findSemi(data, newLinePosition);
-            // if (semiPosition != 0) {
-            // pointerEnd = position + semiPosition;
-            //
-            // newLinePosition = findNewLine(data, semiPosition);
-            //
-            // }
         }
-
-        // if (offset != 0) {
-        // if (lineStream.hasNext()) {
-        // // Skip the first line.
-        // lineStream.skipLine();
-        // }
-        // else {
-        // // No lines then do nothing.
-        // return;
-        // }
-        // }
-        // while (lineStream.hasNext()) {
-        // long keyStartAddress = lineStream.position;
-        // long keyEndAddress = lineStream.findSemi();
-        // long keySuffix = lineStream.suffix;
-        // int keyHash = lineStream.hash;
-        // long valueStartAddress = lineStream.position;
-        // long valueEndAddress = lineStream.findTemperature();
-        // int temperature = parseDouble(valueStartAddress, valueEndAddress);
-        // lineConsumer.process(keyStartAddress, keyEndAddress, keyHash, temperature, keySuffix);
-        // }
-
     }
 
     private void smallWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
@@ -623,7 +575,6 @@ private static class ChunkProcessorImpl implements MapReduce<PrimitiveHashMap> {
 
         @Override
         public void process(long keyStartAddress, long keyEndAddress, long hash, int temperature) {
-            // System.out.println(unsafeToString(keyStartAddress, keyEndAddress) + " --> " + temperature + " hash: " + hash);
             HashEntry entry = statistics.find(keyStartAddress, keyEndAddress, hash);
             if (entry == null) {
                 throw new IllegalStateException("Hash table too small :(");
@@ -632,12 +583,6 @@ public void process(long keyStartAddress, long keyEndAddress, long hash, int tem
                 entry.value = new IntSummaryStatistics();
             }
             entry.value.accept(temperature);
-
-            // IntSummaryStatistics stats = verify.computeIfAbsent(key, ignore -> new IntSummaryStatistics());
-            // stats.accept(temperature);
-            // if (stats.getCount() != entry.value.getCount()) {
-            // System.out.println("Trouble");
-            // }
         }
 
         @Override

From fadce6d4babe530b84594eee0953cfc7349d3da7 Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Thu, 25 Jan 2024 12:22:05 -0800
Subject: [PATCH 41/64] Re introduce suffix

---
 .../onebrc/CalculateAverage_vaidhy.java       | 48 ++++++++++++-------
 1 file changed, 30 insertions(+), 18 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index ade3fbc34..7923e65bd 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -40,6 +40,7 @@ private static final class HashEntry {
         private long startAddress;
         private long endAddress;
         private long hash;
+        private long suffix;
         private int next;
         IntSummaryStatistics value;
     }
@@ -57,7 +58,7 @@ private static class PrimitiveHashMap {
             }
         }
 
-        public HashEntry find(long startAddress, long endAddress, long hash) {
+        public HashEntry find(long startAddress, long endAddress, long hash, long suffix) {
             int len = entries.length;
             int h = Long.hashCode(hash);
             int i = (h ^ (h >> twoPow)) & (len - 1);
@@ -69,13 +70,14 @@ public HashEntry find(long startAddress, long endAddress, long hash) {
                     entry.endAddress = endAddress;
                     entry.hash = hash;
                     entry.next = next;
+                    entry.suffix = suffix;
                     this.next = i;
                     return entry;
                 }
-                if (entry.hash == hash) {
+                if (entry.hash == hash && entry.suffix == suffix) {
                     long entryLength = entry.endAddress - entry.startAddress;
                     long lookupLength = endAddress - startAddress;
-                    if ((entryLength == lookupLength)) {
+                    if (entryLength == lookupLength) {
                         boolean found = compareEntryKeys(startAddress, endAddress, entry);
                         if (found) {
                             return entry;
@@ -100,12 +102,12 @@ private static boolean compareEntryKeys(long startAddress, long endAddress, Hash
                 }
                 entryIndex += 8;
             }
-            for (; lookupIndex < endAddress; lookupIndex++) {
-                if (UNSAFE.getByte(entryIndex) != UNSAFE.getByte(lookupIndex)) {
-                    return false;
-                }
-                entryIndex++;
-            }
+            // for (; lookupIndex < endAddress; lookupIndex++) {
+            // if (UNSAFE.getByte(entryIndex) != UNSAFE.getByte(lookupIndex)) {
+            // return false;
+            // }
+            // entryIndex++;
+            // }
 
             return true;
         }
@@ -186,7 +188,7 @@ private static int parseDouble(long startAddress, long endAddress) {
 
     interface MapReduce<I> {
 
-        void process(long keyStartAddress, long keyEndAddress, long hash, int temperature);
+        void process(long keyStartAddress, long keyEndAddress, long hash, long suffix, int temperature);
 
         I result();
     }
@@ -216,6 +218,8 @@ static class LineStream {
         private long position;
         private long hash;
 
+        private long suffix;
+
         private final ByteBuffer buf = ByteBuffer
                 .allocate(8)
                 .order(ByteOrder.LITTLE_ENDIAN);
@@ -242,7 +246,8 @@ public long findSemi() {
                     int discard = buf.remaining();
                     buf.rewind();
                     long nextData = (buf.getLong() << discard) >>> discard;
-                    hash = simpleHash(h, nextData);
+                    this.suffix = nextData;
+                    this.hash = simpleHash(h, nextData);
                     position = i + 1;
                     return i;
                 }
@@ -256,7 +261,8 @@ public long findSemi() {
                     buf.rewind();
                 }
             }
-            hash = h;
+            this.hash = h;
+            this.suffix = buf.getLong();
             position = fileEnd;
             return fileEnd;
         }
@@ -363,6 +369,7 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
         long hash = DEFAULT_SEED;
         long prevRelevant = 0;
         int prevBits = 0;
+        long suffix = 0;
 
         long crossPoint = Math.min(chunkEnd + 1, fileEnd);
 
@@ -377,7 +384,7 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
                 else {
                     long temperatureEnd = position + (newLinePositionBits >>> 3);
                     int temperature = parseDouble(stationEnd + 1, temperatureEnd);
-                    lineConsumer.process(newRecordStart, stationEnd, hash, temperature);
+                    lineConsumer.process(newRecordStart, stationEnd, hash, suffix, temperature);
                     newLineToken = false;
 
                     nextReadOffsetBits = newLinePositionBits + 8;
@@ -440,11 +447,15 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
 
                         int newPrevBits = prevBits + (semiPositionBits - nextReadOffsetBits);
                         if (newPrevBits > 64) {
-                            long remaining = currRelevant >>> (64 - prevBits);
-                            hash = simpleHash(hash, remaining);
+                            suffix = currRelevant >>> (64 - prevBits);
+                            hash = simpleHash(hash, suffix);
+                        }
+                        else {
+                            suffix = toHash;
                         }
                     }
                     else {
+                        suffix = prevRelevant;
                         hash = simpleHash(hash, prevRelevant);
                     }
 
@@ -482,11 +493,12 @@ private void smallWorker(long offset, long chunkSize, MapReduce<I> lineConsumer)
             long keyStartAddress = lineStream.position;
             long keyEndAddress = lineStream.findSemi();
             long keyHash = lineStream.hash;
+            long suffix = lineStream.suffix;
             long valueStartAddress = lineStream.position;
             long valueEndAddress = lineStream.findTemperature();
             int temperature = parseDouble(valueStartAddress, valueEndAddress);
             // System.out.println("Small worker!");
-            lineConsumer.process(keyStartAddress, keyEndAddress, keyHash, temperature);
+            lineConsumer.process(keyStartAddress, keyEndAddress, keyHash, suffix, temperature);
         }
     }
 
@@ -574,8 +586,8 @@ private static class ChunkProcessorImpl implements MapReduce<PrimitiveHashMap> {
         private final PrimitiveHashMap statistics = new PrimitiveHashMap(14);
 
         @Override
-        public void process(long keyStartAddress, long keyEndAddress, long hash, int temperature) {
-            HashEntry entry = statistics.find(keyStartAddress, keyEndAddress, hash);
+        public void process(long keyStartAddress, long keyEndAddress, long hash, long suffix, int temperature) {
+            HashEntry entry = statistics.find(keyStartAddress, keyEndAddress, hash, suffix);
             if (entry == null) {
                 throw new IllegalStateException("Hash table too small :(");
             }

From 530e5e9d7f1facc25ef6dbef698bbd31f686ce45 Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Thu, 25 Jan 2024 12:28:58 -0800
Subject: [PATCH 42/64] Bug fix that doesn't appear?

---
 src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 7923e65bd..5bce4a910 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -446,7 +446,7 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
                         hash = simpleHash(hash, toHash);
 
                         int newPrevBits = prevBits + (semiPositionBits - nextReadOffsetBits);
-                        if (newPrevBits > 64) {
+                        if (newPrevBits >= 64) {
                             suffix = currRelevant >>> (64 - prevBits);
                             hash = simpleHash(hash, suffix);
                         }

From e27d93c5d98daa5207a0a682484dec19980f0efb Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Thu, 25 Jan 2024 23:58:21 -0800
Subject: [PATCH 43/64] Try unaligned version

---
 .../onebrc/CalculateAverage_vaidhy.java       | 112 +++++++++++++++++-
 1 file changed, 110 insertions(+), 2 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 5bce4a910..fc2c7d541 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -156,6 +156,24 @@ private static Unsafe initUnsafe() {
 
     private static final Unsafe UNSAFE = initUnsafe();
 
+    private static int parseDoubleFromLong(long data, int len) {
+        int normalized = 0;
+        boolean negative = false;
+        for (int i = 0; i < len; i++) {
+            long ch = data & 0xff;
+            data >>>= 1;
+            if (ch == '-') {
+                negative = true;
+                continue;
+            }
+            if (ch == '.') {
+                continue;
+            }
+            normalized = (normalized * 10) + (normalized ^ 0x30);
+        }
+        return negative ? -normalized : normalized;
+    }
+
     private static int parseDouble(long startAddress, long endAddress) {
         int normalized;
         int length = (int) (endAddress - startAddress);
@@ -302,6 +320,7 @@ public long findTemperature() {
 
     private static final long ALL_ONES = 0xffff_ffff_ffff_ffffL;
 
+    // Influenced by roy's SIMD as a register technique.
     private int findByte(long data, long pattern, int readOffsetBits) {
         data >>>= readOffsetBits;
         long match = data ^ pattern;
@@ -318,6 +337,20 @@ private int findByte(long data, long pattern, int readOffsetBits) {
         }
     }
 
+    private int findByteOctet(long data, long pattern) {
+        long match = data ^ pattern;
+        long mask = (match - START_BYTE_INDICATOR) & ((~match) & END_BYTE_INDICATOR);
+
+        if (mask == 0) {
+            // Not Found
+            return -1;
+        }
+        else {
+            // Found
+            return Long.numberOfTrailingZeros(mask) >>> 3;
+        }
+    }
+
     private int findSemi(long data, int readOffsetBits) {
         return findByte(data, SEMI_DETECTION, readOffsetBits);
     }
@@ -327,6 +360,82 @@ private int findNewLine(long data, int readOffsetBits) {
     }
 
     private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
+        long chunkStart = offset + fileService.address();
+        long chunkEnd = chunkStart + chunkSize;
+        long fileEnd = fileService.address() + fileService.length();
+        long stopPoint = Math.min(chunkEnd + 1, fileEnd);
+
+        boolean skip = offset != 0;
+        for (long position = chunkStart; position < stopPoint;) {
+
+            if (skip) {
+                long data = UNSAFE.getLong(position);
+                int newLinePosition = findByteOctet(data, NEW_LINE_DETECTION);
+                if (newLinePosition != -1) {
+                    skip = false;
+                    position = position + newLinePosition + 1;
+                }
+                else {
+                    position = position + 8;
+                }
+                continue;
+            }
+
+            long stationStart = position;
+            long stationEnd = -1;
+            long hash = DEFAULT_SEED;
+            long suffix = 0;
+            do {
+                long data = UNSAFE.getLong(position);
+                int semiPosition = findByteOctet(data, SEMI_DETECTION);
+                if (semiPosition != -1) {
+                    stationEnd = position + semiPosition;
+                    position = stationEnd + 1;
+
+                    if (semiPosition != 0) {
+                        suffix = data & (ALL_ONES >>> (64 - (semiPosition << 3)));
+                    }
+                    else {
+                        suffix = UNSAFE.getLong(position - 8);
+                    }
+                    hash = simpleHash(hash, suffix);
+                    break;
+                }
+                else {
+                    hash = simpleHash(hash, data);
+                    position = position + 8;
+                }
+            } while (true);
+
+            int temperature = 0;
+            {
+                byte ch = UNSAFE.getByte(position);
+                boolean negative = false;
+                if (ch == '-') {
+                    negative = true;
+                    position++;
+                }
+                while (true) {
+                    position++;
+                    if (ch == '\n') {
+                        break;
+                    }
+                    if (ch != '.') {
+                        temperature *= 10;
+                        temperature += (ch ^ '0');
+                    }
+                    ch = UNSAFE.getByte(position);
+                }
+                if (negative) {
+                    temperature = -temperature;
+                }
+            }
+
+            lineConsumer.process(stationStart, stationEnd, hash, suffix, temperature);
+        }
+    }
+
+    private void bigWorkerAligned(long offset, long chunkSize, MapReduce<I> lineConsumer) {
         long fileStart = fileService.address();
         long chunkEnd = fileStart + offset + chunkSize;
         long newRecordStart = fileStart + offset;
@@ -407,8 +516,7 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
                 int semiPositionBits = findSemi(data, nextReadOffsetBits);
 
                 if (semiPositionBits == -1) {
-                    long currRelevant = data >>> nextReadOffsetBits;
-                    currRelevant <<= prevBits;
+                    long currRelevant = data >>> (nextReadOffsetBits - prevBits);
 
                     prevRelevant = prevRelevant | currRelevant;
                     int newPrevBits = prevBits + (64 - nextReadOffsetBits);

From 7a27bddf53f575aebc39cf910e075dd88528ce5e Mon Sep 17 00:00:00 2001
From: Vaidhy Gopalan <vaidhy@gmail.com>
Date: Fri, 26 Jan 2024 19:39:10 -0800
Subject: [PATCH 44/64] Added prepare to test native

---
 calculate_average_vaidhy.sh | 15 +++++++++++++--
 prepare_vaidhy.sh           |  8 ++++++++
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/calculate_average_vaidhy.sh b/calculate_average_vaidhy.sh
index ca204f806..eedad8d58 100755
--- a/calculate_average_vaidhy.sh
+++ b/calculate_average_vaidhy.sh
@@ -15,5 +15,16 @@
 #  limitations under the License.
 #
 
-JAVA_OPTS="--enable-preview"
-java $JAVA_OPTS --class-path target/average-1.0.0-SNAPSHOT.jar dev.morling.onebrc.CalculateAverage_vaidhy
+if [ -f target/CalculateAverage_vaidhy_image ]; then
+    echo "Picking up existing native image 'target/CalculateAverage_vaidhy_image', delete the file to select JVM mode." 1>&2
+    target/CalculateAverage_vaidhy_image
+else
+    JAVA_OPTS="--enable-preview -XX:+UnlockExperimentalVMOptions -XX:+TrustFinalNonStaticFields -XX:+UseTransparentHugePages"
+    if [[ ! "$(uname -s)" = "Darwin" ]]; then
+        # On OS/X, my machine, this errors:
+        JAVA_OPTS="$JAVA_OPTS -XX:+UseTransparentHugePages"
+    fi
+
+    echo "Chosing to run the app in JVM mode as no native image was found, use prepare_vaidhy.sh to generate." 1>&2
+    java $JAVA_OPTS --class-path target/average-1.0.0-SNAPSHOT.jar dev.morling.onebrc.CalculateAverage_vaidhy
+fi
diff --git a/prepare_vaidhy.sh b/prepare_vaidhy.sh
index f83a3ff69..c5e3268c9 100755
--- a/prepare_vaidhy.sh
+++ b/prepare_vaidhy.sh
@@ -17,3 +17,11 @@
 
 source "$HOME/.sdkman/bin/sdkman-init.sh"
 sdk use java 21.0.1-graal 1>&2
+
+# ./mvnw clean verify removes target/ and will re-trigger native image creation.
+if [ ! -f target/CalculateAverage_vaidhy_image ]; then
+    NATIVE_IMAGE_OPTS="--gc=epsilon -O3 -march=native --enable-preview -XX:+UseTransparentHugePages"
+    # Use -H:MethodFilter=CalculateAverage_vaidhy.* -H:Dump=:2 -H:PrintGraph=Network for IdealGraphVisualizer graph dumping.
+    # native-image $NATIVE_IMAGE_OPTS -cp target/average-1.0.0-SNAPSHOT.jar -o target/CalculateAverage_vaidhy_image dev.morling.onebrc.CalculateAverage_vaidhy
+fi
+

From b5ab10010a6ac2da6353a33837a2115778e1b963 Mon Sep 17 00:00:00 2001
From: Vaidhy Gopalan <vaidhy@gmail.com>
Date: Fri, 26 Jan 2024 21:11:38 -0800
Subject: [PATCH 45/64] Branch miss workaround

---
 calculate_average_vaidhy.sh                   |  7 +--
 .../onebrc/CalculateAverage_vaidhy.java       | 63 +++++++++++++++----
 2 files changed, 51 insertions(+), 19 deletions(-)

diff --git a/calculate_average_vaidhy.sh b/calculate_average_vaidhy.sh
index eedad8d58..3a703fa42 100755
--- a/calculate_average_vaidhy.sh
+++ b/calculate_average_vaidhy.sh
@@ -19,12 +19,7 @@ if [ -f target/CalculateAverage_vaidhy_image ]; then
     echo "Picking up existing native image 'target/CalculateAverage_vaidhy_image', delete the file to select JVM mode." 1>&2
     target/CalculateAverage_vaidhy_image
 else
-    JAVA_OPTS="--enable-preview -XX:+UnlockExperimentalVMOptions -XX:+TrustFinalNonStaticFields -XX:+UseTransparentHugePages"
-    if [[ ! "$(uname -s)" = "Darwin" ]]; then
-        # On OS/X, my machine, this errors:
-        JAVA_OPTS="$JAVA_OPTS -XX:+UseTransparentHugePages"
-    fi
-
+    JAVA_OPTS="--enable-preview"
     echo "Chosing to run the app in JVM mode as no native image was found, use prepare_vaidhy.sh to generate." 1>&2
     java $JAVA_OPTS --class-path target/average-1.0.0-SNAPSHOT.jar dev.morling.onebrc.CalculateAverage_vaidhy
 fi
diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index fc2c7d541..028e2a66b 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -39,7 +39,6 @@ public class CalculateAverage_vaidhy<I, T> {
     private static final class HashEntry {
         private long startAddress;
         private long endAddress;
-        private long hash;
         private long suffix;
         private int next;
         IntSummaryStatistics value;
@@ -47,12 +46,15 @@ private static final class HashEntry {
 
     private static class PrimitiveHashMap {
         private final HashEntry[] entries;
+        private final long[] hashes;
+
         private final int twoPow;
         private int next = -1;
 
         PrimitiveHashMap(int twoPow) {
             this.twoPow = twoPow;
             this.entries = new HashEntry[1 << twoPow];
+            this.hashes = new long[1 << twoPow];
             for (int i = 0; i < entries.length; i++) {
                 this.entries[i] = new HashEntry();
             }
@@ -62,25 +64,60 @@ public HashEntry find(long startAddress, long endAddress, long hash, long suffix
             int len = entries.length;
             int h = Long.hashCode(hash);
             int i = (h ^ (h >> twoPow)) & (len - 1);
+            long lookupLength = endAddress - startAddress;
 
-            do {
+            long hashEntry = hashes[i];
+            if (hashEntry == 0) {
                 HashEntry entry = entries[i];
-                if (entry.value == null) {
+                entry.startAddress = startAddress;
+                entry.endAddress = endAddress;
+                hashes[i] = hash;
+                entry.next = next;
+                entry.suffix = suffix;
+                this.next = i;
+                return entry;
+            }
+
+            if (hashEntry == hash) {
+                HashEntry entry = entries[i];
+                if (entry.suffix == suffix) {
+                    long entryLength = entry.endAddress - entry.startAddress;
+                    if (entryLength == lookupLength) {
+                        boolean found = compareEntryKeys(startAddress, endAddress, entry);
+                        if (found) {
+                            return entry;
+                        }
+                    }
+                }
+            }
+
+            i++;
+            if (i == len) {
+                i = 0;
+            }
+
+            do {
+                hashEntry = hashes[i];
+                if (hashEntry == 0) {
+                    HashEntry entry = entries[i];
                     entry.startAddress = startAddress;
                     entry.endAddress = endAddress;
-                    entry.hash = hash;
+                    hashes[i] = hash;
                     entry.next = next;
                     entry.suffix = suffix;
                     this.next = i;
                     return entry;
                 }
-                if (entry.hash == hash && entry.suffix == suffix) {
-                    long entryLength = entry.endAddress - entry.startAddress;
-                    long lookupLength = endAddress - startAddress;
-                    if (entryLength == lookupLength) {
-                        boolean found = compareEntryKeys(startAddress, endAddress, entry);
-                        if (found) {
-                            return entry;
+
+                if (hashEntry == hash) {
+                    HashEntry entry = entries[i];
+                    if (entry.suffix == suffix) {
+                        long entryLength = entry.endAddress - entry.startAddress;
+                        if (entryLength == lookupLength) {
+                            boolean found = compareEntryKeys(startAddress, endAddress, entry);
+                            if (found) {
+                                return entry;
+                            }
                         }
                     }
                 }
@@ -719,10 +756,10 @@ public static void main(String[] args) throws IOException {
                 ChunkProcessorImpl::new,
                 CalculateAverage_vaidhy::combineOutputs);
 
-        int proc = 2 * Runtime.getRuntime().availableProcessors();
+        int proc = Runtime.getRuntime().availableProcessors();
 
         ExecutorService executor = Executors.newFixedThreadPool(proc);
-        Map<String, IntSummaryStatistics> output = calculateAverageVaidhy.master(proc, executor);
+        Map<String, IntSummaryStatistics> output = calculateAverageVaidhy.master(2 * proc, executor);
         executor.shutdown();
 
         Map<String, String> outputStr = toPrintMap(output);

From 6b863b0f3839d97e6318c71507ef5f6be41021e7 Mon Sep 17 00:00:00 2001
From: Vaidhy Gopalan <vaidhy@gmail.com>
Date: Sat, 27 Jan 2024 11:36:02 -0800
Subject: [PATCH 46/64] Fixed temp bug

---
 .../java/dev/morling/onebrc/CalculateAverage_vaidhy.java   | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 028e2a66b..b3f4383f1 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -446,14 +446,13 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
 
             int temperature = 0;
             {
-                byte ch = UNSAFE.getByte(position);
+                byte ch = UNSAFE.getByte(position++);
                 boolean negative = false;
                 if (ch == '-') {
                     negative = true;
-                    position++;
+                    ch = UNSAFE.getByte(position++);
                 }
                 while (true) {
-                    position++;
                     if (ch == '\n') {
                         break;
                     }
@@ -461,7 +460,7 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
                         temperature *= 10;
                         temperature += (ch ^ '0');
                     }
-                    ch = UNSAFE.getByte(position);
+                    ch = UNSAFE.getByte(position++);
                 }
                 if (negative) {
                     temperature = -temperature;

From bef0f20035fe8fe0227afadc68c2bcdd9f4cb446 Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Sun, 28 Jan 2024 18:39:01 -0800
Subject: [PATCH 47/64] Bug fix, i == initial index not hash

---
 .../java/dev/morling/onebrc/CalculateAverage_vaidhy.java | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index b3f4383f1..8aa95d542 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -63,7 +63,8 @@ private static class PrimitiveHashMap {
         public HashEntry find(long startAddress, long endAddress, long hash, long suffix) {
             int len = entries.length;
             int h = Long.hashCode(hash);
-            int i = (h ^ (h >> twoPow)) & (len - 1);
+            int initialIndex = (h ^ (h >> twoPow)) & (len - 1);
+            int i = initialIndex;
             long lookupLength = endAddress - startAddress;
 
             long hashEntry = hashes[i];
@@ -96,6 +97,10 @@ public HashEntry find(long startAddress, long endAddress, long hash, long suffix
                 i = 0;
             }
 
+            if (i == initialIndex) {
+                return null;
+            }
+
             do {
                 hashEntry = hashes[i];
                 if (hashEntry == 0) {
@@ -125,7 +130,7 @@ public HashEntry find(long startAddress, long endAddress, long hash, long suffix
                 if (i == len) {
                     i = 0;
                 }
-            } while (i != hash);
+            } while (i != initialIndex);
             return null;
         }
 

From 084b7f7136de4cb516051527b41f0f97279800fd Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Sun, 28 Jan 2024 18:41:35 -0800
Subject: [PATCH 48/64] Remove additional null checks

---
 .../onebrc/CalculateAverage_vaidhy.java       | 26 ++++++++-----------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 8aa95d542..3f9e8793a 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -60,7 +60,7 @@ private static class PrimitiveHashMap {
             }
         }
 
-        public HashEntry find(long startAddress, long endAddress, long hash, long suffix) {
+        public IntSummaryStatistics find(long startAddress, long endAddress, long hash, long suffix) {
             int len = entries.length;
             int h = Long.hashCode(hash);
             int initialIndex = (h ^ (h >> twoPow)) & (len - 1);
@@ -73,10 +73,11 @@ public HashEntry find(long startAddress, long endAddress, long hash, long suffix
                 entry.startAddress = startAddress;
                 entry.endAddress = endAddress;
                 hashes[i] = hash;
-                entry.next = next;
                 entry.suffix = suffix;
+                entry.next = next;
                 this.next = i;
-                return entry;
+                entry.value = new IntSummaryStatistics();
+                return entry.value;
             }
 
             if (hashEntry == hash) {
@@ -86,7 +87,7 @@ public HashEntry find(long startAddress, long endAddress, long hash, long suffix
                     if (entryLength == lookupLength) {
                         boolean found = compareEntryKeys(startAddress, endAddress, entry);
                         if (found) {
-                            return entry;
+                            return entry.value;
                         }
                     }
                 }
@@ -108,10 +109,11 @@ public HashEntry find(long startAddress, long endAddress, long hash, long suffix
                     entry.startAddress = startAddress;
                     entry.endAddress = endAddress;
                     hashes[i] = hash;
-                    entry.next = next;
                     entry.suffix = suffix;
+                    entry.next = next;
                     this.next = i;
-                    return entry;
+                    entry.value = new IntSummaryStatistics();
+                    return entry.value;
                 }
 
                 if (hashEntry == hash) {
@@ -121,7 +123,7 @@ public HashEntry find(long startAddress, long endAddress, long hash, long suffix
                         if (entryLength == lookupLength) {
                             boolean found = compareEntryKeys(startAddress, endAddress, entry);
                             if (found) {
-                                return entry;
+                                return entry.value;
                             }
                         }
                     }
@@ -736,14 +738,8 @@ private static class ChunkProcessorImpl implements MapReduce<PrimitiveHashMap> {
 
         @Override
         public void process(long keyStartAddress, long keyEndAddress, long hash, long suffix, int temperature) {
-            HashEntry entry = statistics.find(keyStartAddress, keyEndAddress, hash, suffix);
-            if (entry == null) {
-                throw new IllegalStateException("Hash table too small :(");
-            }
-            if (entry.value == null) {
-                entry.value = new IntSummaryStatistics();
-            }
-            entry.value.accept(temperature);
+            IntSummaryStatistics stats = statistics.find(keyStartAddress, keyEndAddress, hash, suffix);
+            stats.accept(temperature);
         }
 
         @Override

From a71cee1537e7665a02d7446229fd69b9543fc949 Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Sun, 28 Jan 2024 18:46:13 -0800
Subject: [PATCH 49/64] Small strings speedup

---
 src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 3f9e8793a..ce01bdd02 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -85,7 +85,7 @@ public IntSummaryStatistics find(long startAddress, long endAddress, long hash,
                 if (entry.suffix == suffix) {
                     long entryLength = entry.endAddress - entry.startAddress;
                     if (entryLength == lookupLength) {
-                        boolean found = compareEntryKeys(startAddress, endAddress, entry);
+                        boolean found = lookupLength <= 8 || compareEntryKeys(startAddress, endAddress, entry);
                         if (found) {
                             return entry.value;
                         }
@@ -121,7 +121,7 @@ public IntSummaryStatistics find(long startAddress, long endAddress, long hash,
                     if (entry.suffix == suffix) {
                         long entryLength = entry.endAddress - entry.startAddress;
                         if (entryLength == lookupLength) {
-                            boolean found = compareEntryKeys(startAddress, endAddress, entry);
+                            boolean found = lookupLength <= 8 || compareEntryKeys(startAddress, endAddress, entry);
                             if (found) {
                                 return entry.value;
                             }

From 5fac5b1ae92664d77e29b497744234996355a10c Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Sun, 28 Jan 2024 18:47:34 -0800
Subject: [PATCH 50/64] Use faster hash funciton

---
 src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index ce01bdd02..4a88c952a 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -183,8 +183,8 @@ public HashEntry next() {
     private static final long DEFAULT_SEED = 104729;
 
     private static long simpleHash(long hash, long nextData) {
-        // return hash ^ nextData;
-        return (hash ^ Long.rotateLeft((nextData * C1), R1)) * C2;
+        return hash ^ nextData;
+        // return (hash ^ Long.rotateLeft((nextData * C1), R1)) * C2;
     }
 
     private static Unsafe initUnsafe() {

From 99e41dd48fab8c900f6e930fd1d88eadfcceb03d Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Sun, 28 Jan 2024 19:04:38 -0800
Subject: [PATCH 51/64] Remove aligned code

---
 .../onebrc/CalculateAverage_vaidhy.java       | 180 +-----------------
 1 file changed, 8 insertions(+), 172 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 4a88c952a..944fae030 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -381,26 +381,9 @@ private int findByte(long data, long pattern, int readOffsetBits) {
         }
     }
 
-    private int findByteOctet(long data, long pattern) {
+    private long findByteOctet(long data, long pattern) {
         long match = data ^ pattern;
-        long mask = (match - START_BYTE_INDICATOR) & ((~match) & END_BYTE_INDICATOR);
-
-        if (mask == 0) {
-            // Not Found
-            return -1;
-        }
-        else {
-            // Found
-            return Long.numberOfTrailingZeros(mask) >>> 3;
-        }
-    }
-
-    private int findSemi(long data, int readOffsetBits) {
-        return findByte(data, SEMI_DETECTION, readOffsetBits);
-    }
-
-    private int findNewLine(long data, int readOffsetBits) {
-        return findByte(data, NEW_LINE_DETECTION, readOffsetBits);
+        return (match - START_BYTE_INDICATOR) & ((~match) & END_BYTE_INDICATOR);
     }
 
     private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
@@ -414,8 +397,9 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
 
             if (skip) {
                 long data = UNSAFE.getLong(position);
-                int newLinePosition = findByteOctet(data, NEW_LINE_DETECTION);
-                if (newLinePosition != -1) {
+                long newLineMask = findByteOctet(data, NEW_LINE_DETECTION);
+                if (newLineMask != 0) {
+                    int newLinePosition = Long.numberOfTrailingZeros(newLineMask) >>> 3;
                     skip = false;
                     position = position + newLinePosition + 1;
                 }
@@ -431,8 +415,9 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
             long suffix = 0;
             do {
                 long data = UNSAFE.getLong(position);
-                int semiPosition = findByteOctet(data, SEMI_DETECTION);
-                if (semiPosition != -1) {
+                long semiMask = findByteOctet(data, SEMI_DETECTION);
+                if (semiMask != 0) {
+                    int semiPosition = Long.numberOfTrailingZeros(semiMask) >>> 3;
                     stationEnd = position + semiPosition;
                     position = stationEnd + 1;
 
@@ -478,155 +463,6 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
         }
     }
 
-    private void bigWorkerAligned(long offset, long chunkSize, MapReduce<I> lineConsumer) {
-        long fileStart = fileService.address();
-        long chunkEnd = fileStart + offset + chunkSize;
-        long newRecordStart = fileStart + offset;
-        long position = fileStart + offset;
-        long fileEnd = fileStart + fileService.length();
-
-        int nextReadOffsetBits = 0;
-
-        long data = UNSAFE.getLong(position);
-
-        if (offset != 0) {
-            boolean foundNewLine = false;
-            for (; position < chunkEnd; position += 8) {
-                data = UNSAFE.getLong(position);
-                int newLinePositionBits = findNewLine(data, nextReadOffsetBits);
-                if (newLinePositionBits != -1) {
-                    nextReadOffsetBits = newLinePositionBits + 8;
-                    newRecordStart = position + (nextReadOffsetBits >>> 3);
-
-                    if (nextReadOffsetBits == 64) {
-                        position += 8;
-                        nextReadOffsetBits = 0;
-                        data = UNSAFE.getLong(position);
-                    }
-                    foundNewLine = true;
-                    break;
-                }
-            }
-            if (!foundNewLine) {
-                return;
-            }
-        }
-
-        boolean newLineToken = false;
-        // false means looking for semi Colon
-        // true means looking for new line.
-
-        long stationEnd = offset;
-
-        long hash = DEFAULT_SEED;
-        long prevRelevant = 0;
-        int prevBits = 0;
-        long suffix = 0;
-
-        long crossPoint = Math.min(chunkEnd + 1, fileEnd);
-
-        while (true) {
-            if (newLineToken) {
-                int newLinePositionBits = findNewLine(data, nextReadOffsetBits);
-                if (newLinePositionBits == -1) {
-                    nextReadOffsetBits = 0;
-                    position += 8;
-                    data = UNSAFE.getLong(position);
-                }
-                else {
-                    long temperatureEnd = position + (newLinePositionBits >>> 3);
-                    int temperature = parseDouble(stationEnd + 1, temperatureEnd);
-                    lineConsumer.process(newRecordStart, stationEnd, hash, suffix, temperature);
-                    newLineToken = false;
-
-                    nextReadOffsetBits = newLinePositionBits + 8;
-                    newRecordStart = temperatureEnd + 1;
-
-                    if (newRecordStart >= crossPoint) {
-                        break;
-                    }
-
-                    hash = DEFAULT_SEED;
-
-                    if (nextReadOffsetBits == 64) {
-                        nextReadOffsetBits = 0;
-                        position += 8;
-                        data = UNSAFE.getLong(position);
-                    }
-                }
-            }
-            else {
-                int semiPositionBits = findSemi(data, nextReadOffsetBits);
-
-                if (semiPositionBits == -1) {
-                    long currRelevant = data >>> (nextReadOffsetBits - prevBits);
-
-                    prevRelevant = prevRelevant | currRelevant;
-                    int newPrevBits = prevBits + (64 - nextReadOffsetBits);
-
-                    if (newPrevBits >= 64) {
-                        hash = simpleHash(hash, prevRelevant);
-
-                        prevBits = (newPrevBits - 64);
-                        if (prevBits != 0) {
-                            prevRelevant = (data >>> (64 - prevBits));
-                        }
-                        else {
-                            prevRelevant = 0;
-                        }
-                    }
-                    else {
-                        prevBits = newPrevBits;
-                    }
-
-                    nextReadOffsetBits = 0;
-                    position += 8;
-                    data = UNSAFE.getLong(position);
-                }
-                else {
-                    // currentData = xxxx_x;aaN
-                    if (semiPositionBits != 0) {
-                        long currRelevant = (data & (ALL_ONES >>> (64 - semiPositionBits))) >>> nextReadOffsetBits;
-                        // 0000_00aa
-
-                        // 0aaa_0000;
-                        long currUsable = currRelevant << prevBits;
-
-                        long toHash = prevRelevant | currUsable;
-
-                        hash = simpleHash(hash, toHash);
-
-                        int newPrevBits = prevBits + (semiPositionBits - nextReadOffsetBits);
-                        if (newPrevBits >= 64) {
-                            suffix = currRelevant >>> (64 - prevBits);
-                            hash = simpleHash(hash, suffix);
-                        }
-                        else {
-                            suffix = toHash;
-                        }
-                    }
-                    else {
-                        suffix = prevRelevant;
-                        hash = simpleHash(hash, prevRelevant);
-                    }
-
-                    prevRelevant = 0;
-                    prevBits = 0;
-
-                    stationEnd = position + (semiPositionBits >>> 3);
-                    nextReadOffsetBits = semiPositionBits + 8;
-                    newLineToken = true;
-
-                    if (nextReadOffsetBits == 64) {
-                        nextReadOffsetBits = 0;
-                        position += 8;
-                        data = UNSAFE.getLong(position);
-                    }
-                }
-            }
-        }
-    }
-
     private void smallWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
         LineStream lineStream = new LineStream(fileService, offset, chunkSize);
 

From b011b19d86a5ded4d53bd63333d4eb7e1d543570 Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Sun, 28 Jan 2024 19:05:19 -0800
Subject: [PATCH 52/64] Remove more unused code

---
 .../morling/onebrc/CalculateAverage_vaidhy.java | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 944fae030..2dc405395 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -364,23 +364,6 @@ public long findTemperature() {
 
     private static final long ALL_ONES = 0xffff_ffff_ffff_ffffL;
 
-    // Influenced by roy's SIMD as a register technique.
-    private int findByte(long data, long pattern, int readOffsetBits) {
-        data >>>= readOffsetBits;
-        long match = data ^ pattern;
-        long mask = (match - START_BYTE_INDICATOR) & ((~match) & END_BYTE_INDICATOR);
-
-        if (mask == 0) {
-            // Not Found
-            return -1;
-        }
-        else {
-            // Found
-            int trailingZeroes = Long.numberOfTrailingZeros(mask) - 7;
-            return readOffsetBits + trailingZeroes;
-        }
-    }
-
     private long findByteOctet(long data, long pattern) {
         long match = data ^ pattern;
         return (match - START_BYTE_INDICATOR) & ((~match) & END_BYTE_INDICATOR);

From b23c446da67464ea60c2f2b45f5ffaa2f7ec2dd7 Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Sun, 28 Jan 2024 19:10:17 -0800
Subject: [PATCH 53/64] first ch won't be newline in temperature

---
 .../java/dev/morling/onebrc/CalculateAverage_vaidhy.java   | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 2dc405395..6610f1211 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -427,16 +427,13 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
                     negative = true;
                     ch = UNSAFE.getByte(position++);
                 }
-                while (true) {
-                    if (ch == '\n') {
-                        break;
-                    }
+                do {
                     if (ch != '.') {
                         temperature *= 10;
                         temperature += (ch ^ '0');
                     }
                     ch = UNSAFE.getByte(position++);
-                }
+                } while (ch != '\n');
                 if (negative) {
                     temperature = -temperature;
                 }

From 7fd6a19306e15a5b6b305c1b7ce4287474686217 Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Sun, 28 Jan 2024 19:25:10 -0800
Subject: [PATCH 54/64] Optimize compare loop

---
 src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 6610f1211..b9d0d70c8 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -139,8 +139,9 @@ public IntSummaryStatistics find(long startAddress, long endAddress, long hash,
         private static boolean compareEntryKeys(long startAddress, long endAddress, HashEntry entry) {
             long entryIndex = entry.startAddress;
             long lookupIndex = startAddress;
+            long endAddressStop = endAddress - 7;
 
-            for (; (lookupIndex + 7) < endAddress; lookupIndex += 8) {
+            for (; lookupIndex < endAddressStop; lookupIndex += 8) {
                 if (UNSAFE.getLong(entryIndex) != UNSAFE.getLong(lookupIndex)) {
                     return false;
                 }
@@ -377,7 +378,6 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
 
         boolean skip = offset != 0;
         for (long position = chunkStart; position < stopPoint;) {
-
             if (skip) {
                 long data = UNSAFE.getLong(position);
                 long newLineMask = findByteOctet(data, NEW_LINE_DETECTION);

From 8b7344eb73ea91ace74786d4b698c680a2e12680 Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Sun, 28 Jan 2024 19:27:17 -0800
Subject: [PATCH 55/64] Move hit case first

---
 .../onebrc/CalculateAverage_vaidhy.java       | 23 ++++++++++---------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index b9d0d70c8..4abf8ef72 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -68,17 +68,6 @@ public IntSummaryStatistics find(long startAddress, long endAddress, long hash,
             long lookupLength = endAddress - startAddress;
 
             long hashEntry = hashes[i];
-            if (hashEntry == 0) {
-                HashEntry entry = entries[i];
-                entry.startAddress = startAddress;
-                entry.endAddress = endAddress;
-                hashes[i] = hash;
-                entry.suffix = suffix;
-                entry.next = next;
-                this.next = i;
-                entry.value = new IntSummaryStatistics();
-                return entry.value;
-            }
 
             if (hashEntry == hash) {
                 HashEntry entry = entries[i];
@@ -93,6 +82,18 @@ public IntSummaryStatistics find(long startAddress, long endAddress, long hash,
                 }
             }
 
+            if (hashEntry == 0) {
+                HashEntry entry = entries[i];
+                entry.startAddress = startAddress;
+                entry.endAddress = endAddress;
+                hashes[i] = hash;
+                entry.suffix = suffix;
+                entry.next = next;
+                this.next = i;
+                entry.value = new IntSummaryStatistics();
+                return entry.value;
+            }
+
             i++;
             if (i == len) {
                 i = 0;

From 3abe5011a72aa310e4be1af89107e8de82cb9043 Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Sun, 28 Jan 2024 19:30:01 -0800
Subject: [PATCH 56/64] store key length instead of end address

---
 .../onebrc/CalculateAverage_vaidhy.java       | 35 +++++++++----------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 4abf8ef72..30bb6b355 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -38,7 +38,7 @@ public class CalculateAverage_vaidhy<I, T> {
 
     private static final class HashEntry {
         private long startAddress;
-        private long endAddress;
+        private long keyLength;
         private long suffix;
         private int next;
         IntSummaryStatistics value;
@@ -72,8 +72,7 @@ public IntSummaryStatistics find(long startAddress, long endAddress, long hash,
             if (hashEntry == hash) {
                 HashEntry entry = entries[i];
                 if (entry.suffix == suffix) {
-                    long entryLength = entry.endAddress - entry.startAddress;
-                    if (entryLength == lookupLength) {
+                    if (entry.keyLength == lookupLength) {
                         boolean found = lookupLength <= 8 || compareEntryKeys(startAddress, endAddress, entry);
                         if (found) {
                             return entry.value;
@@ -85,7 +84,7 @@ public IntSummaryStatistics find(long startAddress, long endAddress, long hash,
             if (hashEntry == 0) {
                 HashEntry entry = entries[i];
                 entry.startAddress = startAddress;
-                entry.endAddress = endAddress;
+                entry.keyLength = lookupLength;
                 hashes[i] = hash;
                 entry.suffix = suffix;
                 entry.next = next;
@@ -105,10 +104,21 @@ public IntSummaryStatistics find(long startAddress, long endAddress, long hash,
 
             do {
                 hashEntry = hashes[i];
+                if (hashEntry == hash) {
+                    HashEntry entry = entries[i];
+                    if (entry.suffix == suffix) {
+                        if (entry.keyLength == lookupLength) {
+                            boolean found = lookupLength <= 8 || compareEntryKeys(startAddress, endAddress, entry);
+                            if (found) {
+                                return entry.value;
+                            }
+                        }
+                    }
+                }
                 if (hashEntry == 0) {
                     HashEntry entry = entries[i];
                     entry.startAddress = startAddress;
-                    entry.endAddress = endAddress;
+                    entry.keyLength = lookupLength;
                     hashes[i] = hash;
                     entry.suffix = suffix;
                     entry.next = next;
@@ -117,18 +127,6 @@ public IntSummaryStatistics find(long startAddress, long endAddress, long hash,
                     return entry.value;
                 }
 
-                if (hashEntry == hash) {
-                    HashEntry entry = entries[i];
-                    if (entry.suffix == suffix) {
-                        long entryLength = entry.endAddress - entry.startAddress;
-                        if (entryLength == lookupLength) {
-                            boolean found = lookupLength <= 8 || compareEntryKeys(startAddress, endAddress, entry);
-                            if (found) {
-                                return entry.value;
-                            }
-                        }
-                    }
-                }
                 i++;
                 if (i == len) {
                     i = 0;
@@ -601,7 +599,8 @@ private static Map<String, IntSummaryStatistics> combineOutputs(
         for (PrimitiveHashMap map : list) {
             for (HashEntry entry : map.entrySet()) {
                 if (entry.value != null) {
-                    String keyStr = unsafeToString(entry.startAddress, entry.endAddress);
+                    String keyStr = unsafeToString(entry.startAddress,
+                            entry.startAddress + entry.keyLength);
 
                     output.compute(keyStr, (ignore, val) -> {
                         if (val == null) {

From dec19106d58795b21d323c38b42540425b08101d Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Sun, 28 Jan 2024 20:40:36 -0800
Subject: [PATCH 57/64] Remove unused code

---
 .../onebrc/CalculateAverage_vaidhy.java       | 27 -------------------
 1 file changed, 27 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 30bb6b355..ebd53dd40 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -146,12 +146,6 @@ private static boolean compareEntryKeys(long startAddress, long endAddress, Hash
                 }
                 entryIndex += 8;
             }
-            // for (; lookupIndex < endAddress; lookupIndex++) {
-            // if (UNSAFE.getByte(entryIndex) != UNSAFE.getByte(lookupIndex)) {
-            // return false;
-            // }
-            // entryIndex++;
-            // }
 
             return true;
         }
@@ -177,9 +171,6 @@ public HashEntry next() {
 
     private static final String FILE = "./measurements.txt";
 
-    private static final long C1 = 0x87c37b91114253d5L;
-    private static final long C2 = 0x4cf5ad432745937fL;
-    private static final int R1 = 31;
     private static final long DEFAULT_SEED = 104729;
 
     private static long simpleHash(long hash, long nextData) {
@@ -200,24 +191,6 @@ private static Unsafe initUnsafe() {
 
     private static final Unsafe UNSAFE = initUnsafe();
 
-    private static int parseDoubleFromLong(long data, int len) {
-        int normalized = 0;
-        boolean negative = false;
-        for (int i = 0; i < len; i++) {
-            long ch = data & 0xff;
-            data >>>= 1;
-            if (ch == '-') {
-                negative = true;
-                continue;
-            }
-            if (ch == '.') {
-                continue;
-            }
-            normalized = (normalized * 10) + (normalized ^ 0x30);
-        }
-        return negative ? -normalized : normalized;
-    }
-
     private static int parseDouble(long startAddress, long endAddress) {
         int normalized;
         int length = (int) (endAddress - startAddress);

From dc62d9143a2e84831248e6baef61cdf6b5e0866b Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Sun, 28 Jan 2024 21:01:21 -0800
Subject: [PATCH 58/64] Since hash=suffix when len <=8 we have one more
 optimization

---
 .../dev/morling/onebrc/CalculateAverage_vaidhy.java   | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index ebd53dd40..bac8e229d 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -71,12 +71,11 @@ public IntSummaryStatistics find(long startAddress, long endAddress, long hash,
 
             if (hashEntry == hash) {
                 HashEntry entry = entries[i];
-                if (entry.suffix == suffix) {
-                    if (entry.keyLength == lookupLength) {
-                        boolean found = lookupLength <= 8 || compareEntryKeys(startAddress, endAddress, entry);
-                        if (found) {
-                            return entry.value;
-                        }
+                if (entry.keyLength == lookupLength) {
+                    boolean found = lookupLength <= 8 ||
+                            (entry.suffix == suffix && compareEntryKeys(startAddress, endAddress, entry));
+                    if (found) {
+                        return entry.value;
                     }
                 }
             }

From 3f91daff4c32e4d61f0a7d86888166fb66c4d809 Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Sun, 28 Jan 2024 21:03:49 -0800
Subject: [PATCH 59/64] If length < 7, then due to nulls in paddin we can avoid
 checking entry length

---
 .../dev/morling/onebrc/CalculateAverage_vaidhy.java  | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index bac8e229d..25e26c2a4 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -71,12 +71,12 @@ public IntSummaryStatistics find(long startAddress, long endAddress, long hash,
 
             if (hashEntry == hash) {
                 HashEntry entry = entries[i];
-                if (entry.keyLength == lookupLength) {
-                    boolean found = lookupLength <= 8 ||
-                            (entry.suffix == suffix && compareEntryKeys(startAddress, endAddress, entry));
-                    if (found) {
-                        return entry.value;
-                    }
+                if (lookupLength <= 7) {
+                    return entry.value;
+                }
+                boolean found = (entry.suffix == suffix && compareEntryKeys(startAddress, endAddress, entry));
+                if (found) {
+                    return entry.value;
                 }
             }
 

From 16aa77f98dcb2007ab159dea884939a3d6e06d3a Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Tue, 30 Jan 2024 07:03:20 -0800
Subject: [PATCH 60/64] Skipping even more

---
 .../onebrc/CalculateAverage_vaidhy.java       | 26 ++++++++++++-------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 25e26c2a4..6bda1f0f1 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -72,9 +72,15 @@ public IntSummaryStatistics find(long startAddress, long endAddress, long hash,
             if (hashEntry == hash) {
                 HashEntry entry = entries[i];
                 if (lookupLength <= 7) {
+                    // This works because
+                    // hash = suffix , when simpleHash is just xor.
+                    // Since length is not 8, suffix will have a 0 at the end.
+                    // Since utf-8 strings can't have 0 in middle of a string this means
+                    // we can stop here.
                     return entry.value;
                 }
-                boolean found = (entry.suffix == suffix && compareEntryKeys(startAddress, endAddress, entry));
+                boolean found = (entry.suffix == suffix &&
+                        compareEntryKeys(startAddress, endAddress, entry.startAddress));
                 if (found) {
                     return entry.value;
                 }
@@ -105,13 +111,13 @@ public IntSummaryStatistics find(long startAddress, long endAddress, long hash,
                 hashEntry = hashes[i];
                 if (hashEntry == hash) {
                     HashEntry entry = entries[i];
-                    if (entry.suffix == suffix) {
-                        if (entry.keyLength == lookupLength) {
-                            boolean found = lookupLength <= 8 || compareEntryKeys(startAddress, endAddress, entry);
-                            if (found) {
-                                return entry.value;
-                            }
-                        }
+                    if (lookupLength <= 7) {
+                        return entry.value;
+                    }
+                    boolean found = (entry.suffix == suffix &&
+                            compareEntryKeys(startAddress, endAddress, entry.startAddress));
+                    if (found) {
+                        return entry.value;
                     }
                 }
                 if (hashEntry == 0) {
@@ -134,8 +140,8 @@ public IntSummaryStatistics find(long startAddress, long endAddress, long hash,
             return null;
         }
 
-        private static boolean compareEntryKeys(long startAddress, long endAddress, HashEntry entry) {
-            long entryIndex = entry.startAddress;
+        private static boolean compareEntryKeys(long startAddress, long endAddress, long entryStartAddress) {
+            long entryIndex = entryStartAddress;
             long lookupIndex = startAddress;
             long endAddressStop = endAddress - 7;
 

From 67d1bb1f33cb6c151a773b144948fb1151d9936a Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Wed, 31 Jan 2024 12:20:44 -0800
Subject: [PATCH 61/64] Remove native image from vaidhy

---
 calculate_average_vaidhy.sh | 10 ++--------
 prepare_vaidhy.sh           |  7 -------
 2 files changed, 2 insertions(+), 15 deletions(-)

diff --git a/calculate_average_vaidhy.sh b/calculate_average_vaidhy.sh
index 3a703fa42..ca204f806 100755
--- a/calculate_average_vaidhy.sh
+++ b/calculate_average_vaidhy.sh
@@ -15,11 +15,5 @@
 #  limitations under the License.
 #
 
-if [ -f target/CalculateAverage_vaidhy_image ]; then
-    echo "Picking up existing native image 'target/CalculateAverage_vaidhy_image', delete the file to select JVM mode." 1>&2
-    target/CalculateAverage_vaidhy_image
-else
-    JAVA_OPTS="--enable-preview"
-    echo "Chosing to run the app in JVM mode as no native image was found, use prepare_vaidhy.sh to generate." 1>&2
-    java $JAVA_OPTS --class-path target/average-1.0.0-SNAPSHOT.jar dev.morling.onebrc.CalculateAverage_vaidhy
-fi
+JAVA_OPTS="--enable-preview"
+java $JAVA_OPTS --class-path target/average-1.0.0-SNAPSHOT.jar dev.morling.onebrc.CalculateAverage_vaidhy
diff --git a/prepare_vaidhy.sh b/prepare_vaidhy.sh
index c5e3268c9..58dbc240f 100755
--- a/prepare_vaidhy.sh
+++ b/prepare_vaidhy.sh
@@ -18,10 +18,3 @@
 source "$HOME/.sdkman/bin/sdkman-init.sh"
 sdk use java 21.0.1-graal 1>&2
 
-# ./mvnw clean verify removes target/ and will re-trigger native image creation.
-if [ ! -f target/CalculateAverage_vaidhy_image ]; then
-    NATIVE_IMAGE_OPTS="--gc=epsilon -O3 -march=native --enable-preview -XX:+UseTransparentHugePages"
-    # Use -H:MethodFilter=CalculateAverage_vaidhy.* -H:Dump=:2 -H:PrintGraph=Network for IdealGraphVisualizer graph dumping.
-    # native-image $NATIVE_IMAGE_OPTS -cp target/average-1.0.0-SNAPSHOT.jar -o target/CalculateAverage_vaidhy_image dev.morling.onebrc.CalculateAverage_vaidhy
-fi
-

From 04e8de2cf07e3eb22fe32d14b16f36e0bdcd8f5a Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Wed, 31 Jan 2024 12:38:54 -0800
Subject: [PATCH 62/64] increase hash map size to speed up 10K data set

---
 src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index 6bda1f0f1..b06cffe95 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -527,7 +527,7 @@ public long address() {
     private static class ChunkProcessorImpl implements MapReduce<PrimitiveHashMap> {
 
         // 1 << 14 > 10,000 so it works
-        private final PrimitiveHashMap statistics = new PrimitiveHashMap(14);
+        private final PrimitiveHashMap statistics = new PrimitiveHashMap(15);
 
         @Override
         public void process(long keyStartAddress, long keyEndAddress, long hash, long suffix, int temperature) {
@@ -573,7 +573,7 @@ private static Map<String, String> toPrintMap(Map<String, IntSummaryStatistics>
     private static Map<String, IntSummaryStatistics> combineOutputs(
                                                                     List<PrimitiveHashMap> list) {
 
-        Map<String, IntSummaryStatistics> output = new HashMap<>(10000);
+        Map<String, IntSummaryStatistics> output = HashMap.newHashMap(10000);
         for (PrimitiveHashMap map : list) {
             for (HashEntry entry : map.entrySet()) {
                 if (entry.value != null) {

From 8c58897f03e6539ea63b75832e2cf15ce6d3afa6 Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Wed, 31 Jan 2024 12:40:33 -0800
Subject: [PATCH 63/64] remoe extra line in prepare script

---
 prepare_vaidhy.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/prepare_vaidhy.sh b/prepare_vaidhy.sh
index 58dbc240f..f83a3ff69 100755
--- a/prepare_vaidhy.sh
+++ b/prepare_vaidhy.sh
@@ -17,4 +17,3 @@
 
 source "$HOME/.sdkman/bin/sdkman-init.sh"
 sdk use java 21.0.1-graal 1>&2
-

From 4a6961b25e0f442d88f9c8fef9a5c31962c50893 Mon Sep 17 00:00:00 2001
From: Anita SV <anitasvasu@gmail.com>
Date: Wed, 31 Jan 2024 12:50:55 -0800
Subject: [PATCH 64/64] Bug fix initial hash

---
 .../java/dev/morling/onebrc/CalculateAverage_vaidhy.java    | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
index b06cffe95..f63374a10 100644
--- a/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
+++ b/src/main/java/dev/morling/onebrc/CalculateAverage_vaidhy.java
@@ -176,8 +176,6 @@ public HashEntry next() {
 
     private static final String FILE = "./measurements.txt";
 
-    private static final long DEFAULT_SEED = 104729;
-
     private static long simpleHash(long hash, long nextData) {
         return hash ^ nextData;
         // return (hash ^ Long.rotateLeft((nextData * C1), R1)) * C2;
@@ -277,7 +275,7 @@ public boolean hasNext() {
         }
 
         public long findSemi() {
-            long h = DEFAULT_SEED;
+            long h = 0;
             buf.rewind();
 
             for (long i = position; i < fileEnd; i++) {
@@ -371,7 +369,7 @@ private void bigWorker(long offset, long chunkSize, MapReduce<I> lineConsumer) {
 
             long stationStart = position;
             long stationEnd = -1;
-            long hash = DEFAULT_SEED;
+            long hash = 0;
             long suffix = 0;
             do {
                 long data = UNSAFE.getLong(position);