[Improvement]: Upgrade to iceberg 1.6.1 (#3084)

apache · Jan 14, 2025 · 23c33cf · 23c33cf
1 parent c72f121
commit 23c33cf
Show file tree

Hide file tree

Showing 10 changed files with 40 additions and 21 deletions.
diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/IcebergRewriteExecutor.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/IcebergRewriteExecutor.java
@@ -24,7 +24,6 @@
 import org.apache.amoro.table.MixedTable;
 import org.apache.amoro.utils.map.StructLikeCollections;
 import org.apache.iceberg.PartitionSpec;
-import org.apache.iceberg.StructLike;
 import org.apache.iceberg.TableProperties;
 import org.apache.iceberg.data.GenericAppenderFactory;
 import org.apache.iceberg.data.IdentityPartitionConverters;
@@ -46,18 +45,6 @@ public IcebergRewriteExecutor(
     super(input, table, structLikeCollections);
   }
 
-  // TODO We can remove this override method after upgrading Iceberg version to 1.5+.
-  @Override
-  protected StructLike partition() {
-    StructLike partitionData = super.partition();
-    if (partitionData != null && partitionData.size() == 0) {
-      // Cast empty partition data to NULL to avoid creating empty partition directory.
-      return null;
-    } else {
-      return partitionData;
-    }
-  }
-
   @Override
   protected OptimizingDataReader dataReader() {
     return new GenericCombinedIcebergDataReader(

diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/BasicUnkeyedTable.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/BasicUnkeyedTable.java
@@ -65,6 +65,7 @@
 
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 
 /** Basic implementation of {@link UnkeyedTable}, wrapping a {@link Table}. */
 public class BasicUnkeyedTable implements UnkeyedTable, HasTableOperations {
@@ -318,4 +319,9 @@ public StructLikeMap<Map<String, String>> partitionProperty() {
   public UpdatePartitionProperties updatePartitionProperties(Transaction transaction) {
     return new PartitionPropertiesUpdate(this, transaction);
   }
+
+  @Override
+  public UUID uuid() {
+    return UUID.fromString(this.operations().current().uuid());
+  }
 }
diff --git a/amoro-format-iceberg/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java b/amoro-format-iceberg/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java
@@ -25,24 +25,44 @@
 import org.apache.iceberg.expressions.Expressions;
 import org.apache.iceberg.types.Conversions;
 import org.apache.iceberg.types.Types;
+import org.junit.After;
 import org.junit.Assert;
 import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import java.util.Arrays;
+import java.util.List;
 import java.util.Set;
 
 @RunWith(Parameterized.class)
-public class TestIcebergFindFiles extends TableTestBase {
+public class TestIcebergFindFiles extends TestBase {
+
   @Parameterized.Parameters(name = "formatVersion = {0}")
-  public static Object[] parameters() {
-    return new Object[] {1, 2};
+  public static List<Object> parameters() {
+    return Arrays.asList(1, 2);
   }
 
+  @Rule public TemporaryFolder tempDir = new TemporaryFolder();
+
   public TestIcebergFindFiles(int formatVersion) {
-    super(formatVersion);
+    this.formatVersion = formatVersion;
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    this.tableDir = tempDir.newFolder();
+    this.tableDir.delete();
+    super.setupTable();
+  }
+
+  @After
+  public void cleanUp() throws Exception {
+    super.cleanupTables();
   }
 
   @Test

diff --git a/amoro-format-mixed/amoro-mixed-flink/v1.15/amoro-mixed-flink-1.15/pom.xml b/amoro-format-mixed/amoro-mixed-flink/v1.15/amoro-mixed-flink-1.15/pom.xml
@@ -34,6 +34,7 @@
     <packaging>jar</packaging>
 
     <properties>
+        <iceberg.version>1.4.3</iceberg.version>
         <kafka.version>2.8.1</kafka.version>
         <assertj.version>3.21.0</assertj.version>
         <testcontainers.version>1.17.2</testcontainers.version>

diff --git a/amoro-format-mixed/amoro-mixed-flink/v1.16/amoro-mixed-flink-1.16/pom.xml b/amoro-format-mixed/amoro-mixed-flink/v1.16/amoro-mixed-flink-1.16/pom.xml
@@ -34,6 +34,7 @@
     <packaging>jar</packaging>
 
     <properties>
+        <iceberg.version>1.4.3</iceberg.version>
         <kafka.version>3.2.3</kafka.version>
         <assertj.version>3.21.0</assertj.version>
         <testcontainers.version>1.17.2</testcontainers.version>

diff --git a/amoro-format-mixed/amoro-mixed-flink/v1.17/amoro-mixed-flink-1.17/pom.xml b/amoro-format-mixed/amoro-mixed-flink/v1.17/amoro-mixed-flink-1.17/pom.xml
@@ -34,6 +34,7 @@
     <packaging>jar</packaging>
 
     <properties>
+        <iceberg.version>1.4.3</iceberg.version>
         <kafka.version>3.2.3</kafka.version>
         <assertj.version>3.21.0</assertj.version>
         <testcontainers.version>1.17.2</testcontainers.version>

diff --git a/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/pom.xml b/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/pom.xml
@@ -33,6 +33,7 @@
     <url>https://amoro.apache.org</url>
 
     <properties>
+        <iceberg.version>1.4.3</iceberg.version>
         <spark.version>3.2.4</spark.version>
         <scala.version>2.12.15</scala.version>
     </properties>
@@ -144,7 +145,7 @@
 
         <dependency>
             <groupId>org.apache.iceberg</groupId>
-            <artifactId>iceberg-spark-3.2_2.12</artifactId>
+            <artifactId>iceberg-spark-3.3_2.12</artifactId>
             <version>${iceberg.version}</version>
             <scope>provided</scope>
             <exclusions>

diff --git a/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-3.2/pom.xml b/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-3.2/pom.xml
@@ -34,6 +34,7 @@
     <url>https://amoro.apache.org</url>
 
     <properties>
+        <iceberg.version>1.4.3</iceberg.version>
         <hive.version>2.3.9</hive.version>
         <spark.version>3.2.4</spark.version>
         <scala.version>2.12.15</scala.version>

diff --git a/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml b/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml
@@ -34,6 +34,7 @@
     <url>https://amoro.apache.org</url>
 
     <properties>
+        <iceberg.version>1.4.3</iceberg.version>
         <hive.version>2.3.9</hive.version>
         <spark.version>3.3.2</spark.version>
         <scala.version>2.12.15</scala.version>

diff --git a/pom.xml b/pom.xml
@@ -101,7 +101,7 @@
         <maven-clean-plugin.version>3.3.2</maven-clean-plugin.version>
         <maven-checkstyle-plugin.version>3.3.1</maven-checkstyle-plugin.version>
 
-        <iceberg.version>1.4.3</iceberg.version>
+        <iceberg.version>1.6.1</iceberg.version>
         <paimon.version>0.9.0</paimon.version>
         <hive.version>3.1.1</hive.version>
         <hadoop.version>3.4.0</hadoop.version>
@@ -1356,8 +1356,8 @@
                 <guava.version>14.0.1</guava.version>
                 <hive.version>2.3.9</hive.version>
                 <hadoop.version>2.10.2</hadoop.version>
-                <terminal.spark.version>3.2.4</terminal.spark.version>
-                <terminal.spark.major.version>3.2</terminal.spark.major.version>
+                <terminal.spark.version>3.3.3</terminal.spark.version>
+                <terminal.spark.major.version>3.3</terminal.spark.major.version>
             </properties>
         </profile>
         <profile>