GH-43902: [Java] Support for Long memory addresses

java/memory/memory-core/src/main/java/org/apache/arrow/memory/AllocationReservation.java

@@ -35,7 +35,7 @@ public interface AllocationReservation extends AutoCloseable {
    * @return true if the addition is possible, false otherwise
    * @throws IllegalStateException if called after buffer() is used to allocate the reservation
    */
-  boolean add(int nBytes);
+  boolean add(long nBytes);
 
   /**
    * Requests a reservation of additional space.
@@ -45,7 +45,7 @@ public interface AllocationReservation extends AutoCloseable {
    * @param nBytes the amount to reserve
    * @return true if the reservation can be satisfied, false otherwise
    */
-  boolean reserve(int nBytes);
+  boolean reserve(long nBytes);
 
   /**
    * Allocate a buffer whose size is the total of all the add()s made.
@@ -63,7 +63,7 @@ public interface AllocationReservation extends AutoCloseable {
    *
    * @return size of the current reservation
    */
-  int getSize();
+  long getSize();
 
   /**
    * Return whether or not the reservation has been used.

java/memory/memory-core/src/main/java/org/apache/arrow/memory/BaseAllocator.java

@@ -889,7 +889,7 @@ public Reservation() {
     }
 
     @Override
-    public boolean add(final int nBytes) {
+    public boolean add(final long nBytes) {
       assertOpen();
 
       Preconditions.checkArgument(nBytes >= 0, "nBytes(%d) < 0", nBytes);
@@ -906,7 +906,7 @@ public boolean add(final int nBytes) {
       // modifying this behavior so that we maintain what we reserve and what the user asked for
       // and make sure to only
       // round to power of two as necessary.
-      final int nBytesTwo = CommonUtil.nextPowerOfTwo(nBytes);
+      final int nBytesTwo = (int) CommonUtil.nextPowerOfTwo(nBytes);
       if (!reserve(nBytesTwo)) {
         return false;
       }
@@ -928,7 +928,7 @@ public ArrowBuf allocateBuffer() {
     }
 
     @Override
-    public int getSize() {
+    public long getSize() {
       return nBytes;
     }
 
@@ -979,7 +979,7 @@ public void close() {
     }
 
     @Override
-    public boolean reserve(int nBytes) {
+    public boolean reserve(long nBytes) {
       assertOpen();
 
       final AllocationOutcome outcome = BaseAllocator.this.allocateBytes(nBytes);

java/memory/memory-core/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java

@@ -38,18 +38,18 @@ public class LowCostIdentityHashMap<K, V extends ValueWithKeyIncluded<K>> {
   private @Nullable Object[] elementData; // elementData[index] = null;
 
   /* Actual number of values. */
-  private int size;
+  private long size;
 
   /*
    * maximum number of elements that can be put in this map before having to
    * rehash.
    */
-  private int threshold;
+  private long threshold;
 
-  private static final int DEFAULT_MIN_SIZE = 1;
+  private static final long DEFAULT_MIN_SIZE = 1;
 
   /* Default load factor of 0.75; */
-  private static final int LOAD_FACTOR = 7500;
+  private static final long LOAD_FACTOR = 7500;
 
   /** Creates a Map with default expected maximum size. */
   public LowCostIdentityHashMap() {
@@ -61,24 +61,24 @@ public LowCostIdentityHashMap() {
    *
    * @param maxSize The estimated maximum number of entries that will be put in this map.
    */
-  public LowCostIdentityHashMap(int maxSize) {
+  public LowCostIdentityHashMap(long maxSize) {
     if (maxSize >= 0) {
       this.size = 0;
       threshold = getThreshold(maxSize);
-      elementData = newElementArrayUnderInitialized(computeElementArraySize());
+      elementData = newElementArrayUnderInitialized((int) computeElementArraySize());
     } else {
       throw new IllegalArgumentException();
     }
   }
 
-  private int getThreshold(@UnderInitialization LowCostIdentityHashMap<K, V> this, int maxSize) {
+  private long getThreshold(@UnderInitialization LowCostIdentityHashMap<K, V> this, long maxSize) {
     // assign the threshold to maxSize initially, this will change to a
     // higher value if rehashing occurs.
     return maxSize > 2 ? maxSize : 2;
   }
 
-  private int computeElementArraySize(@UnderInitialization LowCostIdentityHashMap<K, V> this) {
-    int arraySize = (int) (((long) threshold * 10000) / LOAD_FACTOR);
+  private long computeElementArraySize(@UnderInitialization LowCostIdentityHashMap<K, V> this) {
+    long arraySize = (threshold * 10000) / LOAD_FACTOR;
     // ensure arraySize is positive, the above cast from long to int type
     // leads to overflow and negative arraySize if threshold is too big
     return arraySize < 0 ? -arraySize : arraySize;
@@ -317,7 +317,7 @@ public boolean isEmpty() {
    * @return the number of mappings in this Map.
    */
   public int size() {
-    return size;
+    return (int) size;
   }
 
   /**

java/memory/memory-core/src/main/java/org/apache/arrow/memory/rounding/DefaultRoundingPolicy.java

@@ -34,20 +34,20 @@ public class DefaultRoundingPolicy implements RoundingPolicy {
    *
    * <p>It was copied from {@link io.netty.buffer.PooledByteBufAllocator}.
    */
-  private static final int MIN_PAGE_SIZE = 4096;
+  private static final long MIN_PAGE_SIZE = 4096;
 
-  private static final int MAX_CHUNK_SIZE = (int) (((long) Integer.MAX_VALUE + 1) / 2);
+  private static final long MAX_CHUNK_SIZE = Long.MAX_VALUE / 2;
   private static final long DEFAULT_CHUNK_SIZE;
 
   static {
-    int defaultPageSize = Integer.getInteger("org.apache.memory.allocator.pageSize", 8192);
+    long defaultPageSize = Long.getLong("org.apache.memory.allocator.pageSize", 8192);
     try {
       validateAndCalculatePageShifts(defaultPageSize);
     } catch (Throwable t) {
       defaultPageSize = 8192;
     }
 
-    int defaultMaxOrder = Integer.getInteger("org.apache.memory.allocator.maxOrder", 11);
+    long defaultMaxOrder = Long.getLong("org.apache.memory.allocator.maxOrder", 11);
     try {
       validateAndCalculateChunkSize(defaultPageSize, defaultMaxOrder);
     } catch (Throwable t) {
@@ -60,7 +60,7 @@ public class DefaultRoundingPolicy implements RoundingPolicy {
     }
   }
 
-  private static int validateAndCalculatePageShifts(int pageSize) {
+  private static long validateAndCalculatePageShifts(long pageSize) {
     if (pageSize < MIN_PAGE_SIZE) {
       throw new IllegalArgumentException(
           "pageSize: " + pageSize + " (expected: " + MIN_PAGE_SIZE + ")");
@@ -71,17 +71,17 @@ private static int validateAndCalculatePageShifts(int pageSize) {
     }
 
     // Logarithm base 2. At this point we know that pageSize is a power of two.
-    return Integer.SIZE - 1 - Integer.numberOfLeadingZeros(pageSize);
+    return Long.SIZE - 1 - Long.numberOfLeadingZeros(pageSize);
   }
 
-  private static int validateAndCalculateChunkSize(int pageSize, int maxOrder) {
+  private static long validateAndCalculateChunkSize(long pageSize, long maxOrder) {
     if (maxOrder > 14) {
       throw new IllegalArgumentException("maxOrder: " + maxOrder + " (expected: 0-14)");
     }
 
     // Ensure the resulting chunkSize does not overflow.
-    int chunkSize = pageSize;
-    for (int i = maxOrder; i > 0; i--) {
+    long chunkSize = pageSize;
+    for (long i = maxOrder; i > 0; i--) {
       if (chunkSize > MAX_CHUNK_SIZE / 2) {
         throw new IllegalArgumentException(
             String.format(

java/memory/memory-core/src/main/java/org/apache/arrow/memory/rounding/SegmentRoundingPolicy.java

@@ -28,7 +28,7 @@ public class SegmentRoundingPolicy implements RoundingPolicy {
    * The segment size. It must be at least {@link SegmentRoundingPolicy#MIN_SEGMENT_SIZE}, and be a
    * power of 2.
    */
-  private int segmentSize;
+  private long segmentSize;
 
   /**
    * Constructor for the segment rounding policy.
@@ -37,7 +37,7 @@ public class SegmentRoundingPolicy implements RoundingPolicy {
    * @throws IllegalArgumentException if the segment size is smaller than {@link
    *     SegmentRoundingPolicy#MIN_SEGMENT_SIZE}, or is not a power of 2.
    */
-  public SegmentRoundingPolicy(int segmentSize) {
+  public SegmentRoundingPolicy(long segmentSize) {
     Preconditions.checkArgument(
         segmentSize >= MIN_SEGMENT_SIZE,
         "The segment size cannot be smaller than %s",
@@ -52,7 +52,7 @@ public long getRoundedSize(long requestSize) {
     return (requestSize + (segmentSize - 1)) / segmentSize * segmentSize;
   }
 
-  public int getSegmentSize() {
+  public long getSegmentSize() {
     return segmentSize;
   }
 }

java/memory/memory-core/src/test/java/org/apache/arrow/memory/util/hash/TestArrowBufHasher.java

@@ -82,6 +82,7 @@ private void verifyHashCodesEqual(
       int length2) {
     int hashCode1 = hasher.hashCode(buf1, offset1, length1);
     int hashCode2 = hasher.hashCode(buf2, offset2, length2);
+
     assertEquals(hashCode1, hashCode2);
   }
 

java/memory/memory-netty-buffer-patch/src/main/java/io/netty/buffer/NettyArrowBuf.java

@@ -38,7 +38,7 @@ public class NettyArrowBuf extends AbstractByteBuf implements AutoCloseable {
 
   private final ArrowBuf arrowBuf;
   private final ArrowByteBufAllocator arrowByteBufAllocator;
-  private int length;
+  private long length;
   private final long address;
 
   /**
@@ -49,8 +49,8 @@ public class NettyArrowBuf extends AbstractByteBuf implements AutoCloseable {
    * @param length The length of this buffer.
    */
   public NettyArrowBuf(
-      final ArrowBuf arrowBuf, final BufferAllocator bufferAllocator, final int length) {
-    super(length);
+      final ArrowBuf arrowBuf, final BufferAllocator bufferAllocator, final long length) {
+    super((int) length);
     this.arrowBuf = arrowBuf;
     this.arrowByteBufAllocator = new ArrowByteBufAllocator(bufferAllocator);
     this.length = length;