[refactor](script) optimize build scala version (#198)

apache · May 8, 2024 · f0e4f73 · f0e4f73
1 parent a3cf8a6
commit f0e4f73
Show file tree

Hide file tree

Showing 3 changed files with 68 additions and 26 deletions.
diff --git a/.github/workflows/build-extension.yml b/.github/workflows/build-extension.yml
@@ -46,47 +46,54 @@ jobs:
       run: |
         cd spark-doris-connector && mvn clean package ${MVN_OPT} \
           -Dspark.version=2.3.4 \
-          -Dscala.version=2.11 \
-          -Dspark.major.version=2.3
+          -Dscala.version=2.11.8 \
+          -Dspark.major.version=2.3 \
+          -Dscala.major.version=2.11
 
     - name: Build spark connector 2.4 2.11
       run: |
         cd spark-doris-connector && mvn clean package ${MVN_OPT} \
           -Dspark.version=2.4.0 \
-          -Dscala.version=2.11 \
-          -Dspark.major.version=2.4    
+          -Dscala.version=2.11.8 \
+          -Dspark.major.version=2.4 \
+          -Dscala.major.version=2.11  
 
     - name: Build spark connector 2.4 2.12
       run: |
         cd spark-doris-connector && mvn clean package ${MVN_OPT} \
           -Dspark.version=2.4.0 \
-          -Dscala.version=2.12 \
-          -Dspark.major.version=2.4        
+          -Dscala.version=2.12.10 \
+          -Dspark.major.version=2.4 \
+          -Dscala.major.version=2.12       
 
     - name: Build spark connector 3.1
       run: |
         cd spark-doris-connector && mvn clean package ${MVN_OPT} \
           -Dspark.version=3.1.2 \
-          -Dscala.version=2.12 \
-          -Dspark.major.version=3.1
+          -Dscala.version=2.12.10 \
+          -Dspark.major.version=3.1 \
+          -Dscala.major.version=2.12
 
     - name: Build spark connector 3.2
       run: |
         cd spark-doris-connector && mvn clean package ${MVN_OPT} \
           -Dspark.version=3.2.0 \
-          -Dscala.version=2.12 \
-          -Dspark.major.version=3.2
+          -Dscala.version=2.12.10 \
+          -Dspark.major.version=3.2 \
+          -Dscala.major.version=2.12
 
     - name: Build spark connector 3.3
       run: |
         cd spark-doris-connector && mvn clean package ${MVN_OPT} \
           -Dspark.version=3.3.0 \
-          -Dscala.version=2.12 \
-          -Dspark.major.version=3.3
+          -Dscala.version=2.12.10 \
+          -Dspark.major.version=3.3 \
+          -Dscala.major.version=2.12
 
     - name: Build spark connector 3.4
       run: |
         cd spark-doris-connector && mvn clean package ${MVN_OPT} \
           -Dspark.version=3.4.0 \
-          -Dscala.version=2.12 \
-          -Dspark.major.version=3.4  
+          -Dscala.version=2.12.10 \
+          -Dspark.major.version=3.4 \
+          -Dscala.major.version=2.12
diff --git a/spark-doris-connector/build.sh b/spark-doris-connector/build.sh
@@ -170,9 +170,15 @@ selectSpark() {
   done
 }
 
+SCALA_VERSION=0
 selectScala
 ScalaVer=$?
-SCALA_VERSION="2.1${ScalaVer}"
+if [ ${ScalaVer} -eq 1 ]; then
+    SCALA_VERSION="2.11.8"
+elif [ ${ScalaVer} -eq 2 ]; then
+    SCALA_VERSION="2.12.10"
+fi
+
 
 SPARK_VERSION=0
 selectSpark
@@ -196,27 +202,30 @@ elif [ ${SparkVer} -eq 7 ]; then
     SPARK_VERSION=$ver
 fi
 
-if [[ $SPARK_VERSION =~ ^2.3 && $SCALA_VERSION == "2.12"  ]]; then
+if [[ $SPARK_VERSION =~ ^2.3 && $SCALA_VERSION == "2.12.10"  ]]; then
   echo_r "Spark 2.3 is not compatible with scala 2.12, will exit."
   exit 1
-elif [[ $SPARK_VERSION =~ ^3.* && $SCALA_VERSION == "2.11" ]]; then
+elif [[ $SPARK_VERSION =~ ^3.* && $SCALA_VERSION == "2.11.8" ]]; then
   echo_r "Spark 3.x is not compatible with scala 2.11, will exit."
   exit 1
 fi
 
 # extract major version:
 # eg: 3.1.2 -> 3.1
+SCALA_MAJOR_VERSION=0
+[ ${SCALA_VERSION} != 0 ] && SCALA_MAJOR_VERSION=${SCALA_VERSION%.*}
 SPARK_MAJOR_VERSION=0
 [ ${SPARK_VERSION} != 0 ] && SPARK_MAJOR_VERSION=${SPARK_VERSION%.*}
 
-echo_g " scala version: ${SCALA_VERSION}"
+echo_g " scala version: ${SCALA_VERSION}, major version: ${SCALA_MAJOR_VERSION}"
 echo_g " spark version: ${SPARK_VERSION}, major version: ${SPARK_MAJOR_VERSION}"
 echo_g " build starting..."
 
 ${MVN_BIN} clean package \
   -Dspark.version=${SPARK_VERSION} \
   -Dscala.version=${SCALA_VERSION} \
-  -Dspark.major.version=${SPARK_MAJOR_VERSION} "$@"
+  -Dspark.major.version=${SPARK_MAJOR_VERSION} \
+  -Dscala.major.version=${SCALA_MAJOR_VERSION} "$@"
 
 EXIT_CODE=$?
 if [ $EXIT_CODE -eq 0 ]; then

diff --git a/spark-doris-connector/pom.xml b/spark-doris-connector/pom.xml
@@ -26,7 +26,7 @@
         <version>29</version>
     </parent>
     <groupId>org.apache.doris</groupId>
-    <artifactId>spark-doris-connector-${spark.major.version}_${scala.version}</artifactId>
+    <artifactId>spark-doris-connector-${spark.major.version}_${scala.major.version}</artifactId>
     <version>${revision}</version>
     <name>Spark Doris Connector</name>
     <url>https://doris.apache.org/</url>
@@ -70,7 +70,8 @@
         <revision>1.4.0-SNAPSHOT</revision>
         <spark.version>3.1.2</spark.version>
         <spark.major.version>3.1</spark.major.version>
-        <scala.version>2.12</scala.version>
+        <scala.version>2.12.10</scala.version>
+        <scala.major.version>2.12</scala.major.version>
         <libthrift.version>0.16.0</libthrift.version>
         <arrow.version>13.0.0</arrow.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -86,6 +87,12 @@
             <groupId>org.apache.doris</groupId>
             <artifactId>thrift-service</artifactId>
             <version>${thrift-service.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.httpcomponents</groupId>
+                    <artifactId>httpclient</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>io.netty</groupId>
@@ -96,14 +103,14 @@
 
         <dependency>
             <groupId>org.apache.spark</groupId>
-            <artifactId>spark-core_${scala.version}</artifactId>
+            <artifactId>spark-core_${scala.major.version}</artifactId>
             <version>${spark.version}</version>
             <scope>provided</scope>
         </dependency>
 
         <dependency>
             <groupId>org.apache.spark</groupId>
-            <artifactId>spark-sql_${scala.version}</artifactId>
+            <artifactId>spark-sql_${scala.major.version}</artifactId>
             <version>${spark.version}</version>
             <scope>provided</scope>
         </dependency>
@@ -151,7 +158,7 @@
 
         <dependency>
             <groupId>org.mockito</groupId>
-            <artifactId>mockito-scala_${scala.version}</artifactId>
+            <artifactId>mockito-scala_${scala.major.version}</artifactId>
             <version>1.4.7</version>
             <exclusions>
                 <exclusion>
@@ -177,7 +184,7 @@
 
         <dependency>
             <groupId>org.apache.spark</groupId>
-            <artifactId>spark-sql-kafka-0-10_${scala.version}</artifactId>
+            <artifactId>spark-sql-kafka-0-10_${scala.major.version}</artifactId>
             <version>${spark.version}</version>
             <scope>test</scope>
         </dependency>
@@ -202,7 +209,7 @@
 
         <dependency>
             <groupId>com.fasterxml.jackson.module</groupId>
-            <artifactId>jackson-module-scala_${scala.version}</artifactId>
+            <artifactId>jackson-module-scala_${scala.major.version}</artifactId>
             <version>${fasterxml.jackson.version}</version>
         </dependency>
 
@@ -226,6 +233,21 @@
             <version>4.2.0</version>
             <scope>test</scope>
         </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.scala-lang/scala-library -->
+        <dependency>
+            <groupId>org.scala-lang</groupId>
+            <artifactId>scala-library</artifactId>
+            <version>${scala.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <version>4.5.13</version>
+        </dependency>
+
     </dependencies>
 
     <build>
@@ -329,6 +351,10 @@
                                 <pattern>org.apache.thrift</pattern>
                                 <shadedPattern>org.apache.doris.shaded.org.apache.thrift</shadedPattern>
                             </relocation>
+                            <relocation>
+                                <pattern>org.apache.http</pattern>
+                                <shadedPattern>org.apache.doris.shaded.org.apache.http</shadedPattern>
+                            </relocation>
                         </relocations>
                     </configuration>
                     <executions>