GH-43902: [Java] Support for Long memory addresses

java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationReservation.java

@@ -35,7 +35,7 @@ public interface AllocationReservation extends AutoCloseable {
    * @return true if the addition is possible, false otherwise
    * @throws IllegalStateException if called after buffer() is used to allocate the reservation
    */
-  boolean add(int nBytes);
+  boolean add(long nBytes);
 
   /**
    * Requests a reservation of additional space.
@@ -45,7 +45,7 @@ public interface AllocationReservation extends AutoCloseable {
    * @param nBytes the amount to reserve
    * @return true if the reservation can be satisfied, false otherwise
    */
-  boolean reserve(int nBytes);
+  boolean reserve(long nBytes);
 
   /**
    * Allocate a buffer whose size is the total of all the add()s made.
@@ -63,7 +63,7 @@ public interface AllocationReservation extends AutoCloseable {
    *
    * @return size of the current reservation
    */
-  int getSize();
+  long getSize();
 
   /**
    * Return whether or not the reservation has been used.

java/memory/memory-core/src/main/java/org/apache/arrow/memory/BaseAllocator.java

@@ -889,7 +889,7 @@ public Reservation() {
     }
 
     @Override
-    public boolean add(final int nBytes) {
+    public boolean add(final long nBytes) {
       assertOpen();
 
       Preconditions.checkArgument(nBytes >= 0, "nBytes(%d) < 0", nBytes);
@@ -906,7 +906,7 @@ public boolean add(final int nBytes) {
       // modifying this behavior so that we maintain what we reserve and what the user asked for
       // and make sure to only
       // round to power of two as necessary.
-      final int nBytesTwo = CommonUtil.nextPowerOfTwo(nBytes);
+      final int nBytesTwo = (int) CommonUtil.nextPowerOfTwo(nBytes);
       if (!reserve(nBytesTwo)) {
         return false;
       }
@@ -928,7 +928,7 @@ public ArrowBuf allocateBuffer() {
     }
 
     @Override
-    public int getSize() {
+    public long getSize() {
       return nBytes;
     }
 
@@ -979,7 +979,7 @@ public void close() {
     }
 
     @Override
-    public boolean reserve(int nBytes) {
+    public boolean reserve(long nBytes) {
       assertOpen();
 
       final AllocationOutcome outcome = BaseAllocator.this.allocateBytes(nBytes);

java/memory/memory-core/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java

@@ -38,18 +38,18 @@ public class LowCostIdentityHashMap<K, V extends ValueWithKeyIncluded<K>> {
   private @Nullable Object[] elementData; // elementData[index] = null;
 
   /* Actual number of values. */
-  private int size;
+  private long size;
 
   /*
    * maximum number of elements that can be put in this map before having to
    * rehash.
    */
-  private int threshold;
+  private long threshold;
 
-  private static final int DEFAULT_MIN_SIZE = 1;
+  private static final long DEFAULT_MIN_SIZE = 1;
 
   /* Default load factor of 0.75; */
-  private static final int LOAD_FACTOR = 7500;
+  private static final long LOAD_FACTOR = 7500;
 
   /** Creates a Map with default expected maximum size. */
   public LowCostIdentityHashMap() {
@@ -61,24 +61,24 @@ public LowCostIdentityHashMap() {
    *
    * @param maxSize The estimated maximum number of entries that will be put in this map.
    */
-  public LowCostIdentityHashMap(int maxSize) {
+  public LowCostIdentityHashMap(long maxSize) {
     if (maxSize >= 0) {
       this.size = 0;
       threshold = getThreshold(maxSize);
-      elementData = newElementArrayUnderInitialized(computeElementArraySize());
+      elementData = newElementArrayUnderInitialized((int) computeElementArraySize());
     } else {
       throw new IllegalArgumentException();
     }
   }
 
-  private int getThreshold(@UnderInitialization LowCostIdentityHashMap<K, V> this, int maxSize) {
+  private long getThreshold(@UnderInitialization LowCostIdentityHashMap<K, V> this, long maxSize) {
     // assign the threshold to maxSize initially, this will change to a
     // higher value if rehashing occurs.
     return maxSize > 2 ? maxSize : 2;
   }
 
-  private int computeElementArraySize(@UnderInitialization LowCostIdentityHashMap<K, V> this) {
-    int arraySize = (int) (((long) threshold * 10000) / LOAD_FACTOR);
+  private long computeElementArraySize(@UnderInitialization LowCostIdentityHashMap<K, V> this) {
+    long arraySize = (threshold * 10000) / LOAD_FACTOR;
     // ensure arraySize is positive, the above cast from long to int type
     // leads to overflow and negative arraySize if threshold is too big
     return arraySize < 0 ? -arraySize : arraySize;
@@ -317,7 +317,7 @@ public boolean isEmpty() {
    * @return the number of mappings in this Map.
    */
   public int size() {
-    return size;
+    return (int) size;
   }
 
   /**

java/memory/memory-core/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java

@@ -34,20 +34,20 @@ public class DefaultRoundingPolicy implements RoundingPolicy {
    *
    * <p>It was copied from {@link io.netty.buffer.PooledByteBufAllocator}.
    */
-  private static final int MIN_PAGE_SIZE = 4096;
+  private static final long MIN_PAGE_SIZE = 4096;
 
-  private static final int MAX_CHUNK_SIZE = (int) (((long) Integer.MAX_VALUE + 1) / 2);
+  private static final long MAX_CHUNK_SIZE = Long.MAX_VALUE / 2;
   private static final long DEFAULT_CHUNK_SIZE;
 
   static {
-    int defaultPageSize = Integer.getInteger("org.apache.memory.allocator.pageSize", 8192);
+    long defaultPageSize = Long.getLong("org.apache.memory.allocator.pageSize", 8192);
     try {
       validateAndCalculatePageShifts(defaultPageSize);
     } catch (Throwable t) {
       defaultPageSize = 8192;
     }
 
-    int defaultMaxOrder = Integer.getInteger("org.apache.memory.allocator.maxOrder", 11);
+    long defaultMaxOrder = Long.getLong("org.apache.memory.allocator.maxOrder", 11);
     try {
       validateAndCalculateChunkSize(defaultPageSize, defaultMaxOrder);
     } catch (Throwable t) {
@@ -60,7 +60,7 @@ public class DefaultRoundingPolicy implements RoundingPolicy {
     }
   }
 
-  private static int validateAndCalculatePageShifts(int pageSize) {
+  private static long validateAndCalculatePageShifts(long pageSize) {
     if (pageSize < MIN_PAGE_SIZE) {
       throw new IllegalArgumentException(
           "pageSize: " + pageSize + " (expected: " + MIN_PAGE_SIZE + ")");
@@ -71,17 +71,17 @@ private static int validateAndCalculatePageShifts(int pageSize) {
     }
 
     // Logarithm base 2. At this point we know that pageSize is a power of two.
-    return Integer.SIZE - 1 - Integer.numberOfLeadingZeros(pageSize);
+    return Long.SIZE - 1 - Long.numberOfLeadingZeros(pageSize);
   }
 
-  private static int validateAndCalculateChunkSize(int pageSize, int maxOrder) {
+  private static long validateAndCalculateChunkSize(long pageSize, long maxOrder) {
     if (maxOrder > 14) {
       throw new IllegalArgumentException("maxOrder: " + maxOrder + " (expected: 0-14)");
     }
 
     // Ensure the resulting chunkSize does not overflow.
-    int chunkSize = pageSize;
-    for (int i = maxOrder; i > 0; i--) {
+    long chunkSize = pageSize;
+    for (long i = maxOrder; i > 0; i--) {
       if (chunkSize > MAX_CHUNK_SIZE / 2) {
         throw new IllegalArgumentException(
             String.format(

java/memory/memory-core/src/main/java/org/apache/arrow/memory/rounding/SegmentRoundingPolicy.java

@@ -28,7 +28,7 @@ public class SegmentRoundingPolicy implements RoundingPolicy {
    * The segment size. It must be at least {@link SegmentRoundingPolicy#MIN_SEGMENT_SIZE}, and be a
    * power of 2.
    */
-  private int segmentSize;
+  private long segmentSize;
 
   /**
    * Constructor for the segment rounding policy.
@@ -37,7 +37,7 @@ public class SegmentRoundingPolicy implements RoundingPolicy {
    * @throws IllegalArgumentException if the segment size is smaller than {@link
    *     SegmentRoundingPolicy#MIN_SEGMENT_SIZE}, or is not a power of 2.
    */
-  public SegmentRoundingPolicy(int segmentSize) {
+  public SegmentRoundingPolicy(long segmentSize) {
     Preconditions.checkArgument(
         segmentSize >= MIN_SEGMENT_SIZE,
         "The segment size cannot be smaller than %s",
@@ -52,7 +52,7 @@ public long getRoundedSize(long requestSize) {
     return (requestSize + (segmentSize - 1)) / segmentSize * segmentSize;
   }
 
-  public int getSegmentSize() {
+  public long getSegmentSize() {
     return segmentSize;
   }
 }

java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ArrowBufPointer.java

@@ -157,7 +157,7 @@ public int hashCode() {
     if (buf == null) {
       hashCode = NULL_HASH_CODE;
     } else {
-      hashCode = hasher.hashCode(buf, offset, length);
+      hashCode = (int) hasher.hashCode(buf, offset, length);
     }
 
     hashCodeChanged = false;

java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/ByteFunctionHelpers.java

@@ -295,25 +295,25 @@ private static int memcmp(
   }
 
   /** Compute hashCode with the given {@link ArrowBuf} and start/end index. */
-  public static int hash(final ArrowBuf buf, long start, long end) {
+  public static long hash(final ArrowBuf buf, long start, long end) {
 
     return hash(SimpleHasher.INSTANCE, buf, start, end);
   }
 
   /**
    * Compute hashCode with the given {@link ArrowBufHasher}, {@link ArrowBuf} and start/end index.
    */
-  public static final int hash(ArrowBufHasher hasher, final ArrowBuf buf, long start, long end) {
+  public static final long hash(ArrowBufHasher hasher, final ArrowBuf buf, long start, long end) {
 
     if (hasher == null) {
       hasher = SimpleHasher.INSTANCE;
     }
 
-    return hasher.hashCode(buf, start, end - start);
+    return (int) hasher.hashCode(buf, start, end - start);
   }
 
   /** Generate a new hashCode with the given current hashCode and new hashCode. */
-  public static int combineHash(int currentHash, int newHash) {
+  public static long combineHash(long currentHash, long newHash) {
     return currentHash * 31 + newHash;
   }
 }

java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/HistoricalLog.java

@@ -32,7 +32,7 @@ public class HistoricalLog {
 
   private final Deque<Event> history = new ArrayDeque<>();
   private final String idString; // the formatted id string
-  private final int limit; // the limit on the number of events kept
+  private final long limit; // the limit on the number of events kept
   private @Nullable Event firstEvent; // the first stack trace recorded
 
   /**
@@ -67,7 +67,8 @@ public HistoricalLog(@FormatString final String idStringFormat, Object... args)
    * @param args for the format string, or nothing if none are required
    */
   @FormatMethod
-  public HistoricalLog(final int limit, @FormatString final String idStringFormat, Object... args) {
+  public HistoricalLog(
+      final long limit, @FormatString final String idStringFormat, Object... args) {
     this.limit = limit;
     this.idString = String.format(idStringFormat, args);
     this.firstEvent = null;

java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/ArrowBufHasher.java

@@ -33,7 +33,7 @@ public interface ArrowBufHasher {
    * @param length length of the memory region.
    * @return the hash code.
    */
-  int hashCode(long address, long length);
+  long hashCode(long address, long length);
 
   /**
    * Calculates the hash code for a memory region.
@@ -43,5 +43,5 @@ public interface ArrowBufHasher {
    * @param length length of the memory region.
    * @return the hash code.
    */
-  int hashCode(ArrowBuf buf, long offset, long length);
+  long hashCode(ArrowBuf buf, long offset, long length);
 }

java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/MurmurHasher.java

@@ -50,12 +50,12 @@ public MurmurHasher(int seed) {
   }
 
   @Override
-  public int hashCode(long address, long length) {
+  public long hashCode(long address, long length) {
     return hashCode(address, length, seed);
   }
 
   @Override
-  public int hashCode(ArrowBuf buf, long offset, long length) {
+  public long hashCode(ArrowBuf buf, long offset, long length) {
     buf.checkBytes(offset, offset + length);
     return hashCode(buf.memoryAddress() + offset, length);
   }
@@ -69,7 +69,7 @@ public int hashCode(ArrowBuf buf, long offset, long length) {
    * @param seed the seed.
    * @return the hash code.
    */
-  public static int hashCode(ArrowBuf buf, long offset, long length, int seed) {
+  public static long hashCode(ArrowBuf buf, long offset, long length, int seed) {
     buf.checkBytes(offset, offset + length);
     return hashCode(buf.memoryAddress() + offset, length, seed);
   }
@@ -82,7 +82,7 @@ public static int hashCode(ArrowBuf buf, long offset, long length, int seed) {
    * @param seed the seed.
    * @return the hash code.
    */
-  public static int hashCode(long address, long length, int seed) {
+  public static long hashCode(long address, long length, int seed) {
     int index = 0;
     int hash = seed;
     while (index + 4 <= length) {

java/memory/memory-core/src/main/java/org/apache/arrow/memory/util/hash/SimpleHasher.java

@@ -48,12 +48,12 @@ protected SimpleHasher() {}
    * @return the hash code.
    */
   @Override
-  public int hashCode(long address, long length) {
-    int hashValue = 0;
+  public long hashCode(long address, long length) {
+    long hashValue = 0;
     int index = 0;
     while (index + 8 <= length) {
       long longValue = MemoryUtil.getLong(address + index);
-      int longHash = getLongHashCode(longValue);
+      long longHash = getLongHashCode(longValue);
       hashValue = combineHashCode(hashValue, longHash);
       index += 8;
     }
@@ -84,20 +84,20 @@ public int hashCode(long address, long length) {
    * @return the hash code.
    */
   @Override
-  public int hashCode(ArrowBuf buf, long offset, long length) {
+  public long hashCode(ArrowBuf buf, long offset, long length) {
     buf.checkBytes(offset, offset + length);
     return hashCode(buf.memoryAddress() + offset, length);
   }
 
-  protected int combineHashCode(int currentHashCode, int newHashCode) {
+  protected long combineHashCode(long currentHashCode, long newHashCode) {
     return currentHashCode * 37 + newHashCode;
   }
 
-  protected int getLongHashCode(long longValue) {
+  protected long getLongHashCode(long longValue) {
     return Long.hashCode(longValue);
   }
 
-  protected int finalizeHashCode(int hashCode) {
+  protected long finalizeHashCode(long hashCode) {
     return hashCode;
   }
 

java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/TestArrowBufPointer.java

@@ -190,13 +190,13 @@ static class CounterHasher implements ArrowBufHasher {
     protected int counter = 0;
 
     @Override
-    public int hashCode(long address, long length) {
+    public long hashCode(long address, long length) {
       counter += 1;
       return SimpleHasher.INSTANCE.hashCode(address, length);
     }
 
     @Override
-    public int hashCode(ArrowBuf buf, long offset, long length) {
+    public long hashCode(ArrowBuf buf, long offset, long length) {
       counter += 1;
       return SimpleHasher.INSTANCE.hashCode(buf, offset, length);
     }