diff --git a/.github/workflows/build-extension.yml b/.github/workflows/build-extension.yml index bbc68f75..5651c5a0 100644 --- a/.github/workflows/build-extension.yml +++ b/.github/workflows/build-extension.yml @@ -46,47 +46,54 @@ jobs: run: | cd spark-doris-connector && mvn clean package ${MVN_OPT} \ -Dspark.version=2.3.4 \ - -Dscala.version=2.11 \ - -Dspark.major.version=2.3 + -Dscala.version=2.11.8 \ + -Dspark.major.version=2.3 \ + -Dscala.major.version=2.11 - name: Build spark connector 2.4 2.11 run: | cd spark-doris-connector && mvn clean package ${MVN_OPT} \ -Dspark.version=2.4.0 \ - -Dscala.version=2.11 \ - -Dspark.major.version=2.4 + -Dscala.version=2.11.8 \ + -Dspark.major.version=2.4 \ + -Dscala.major.version=2.11 - name: Build spark connector 2.4 2.12 run: | cd spark-doris-connector && mvn clean package ${MVN_OPT} \ -Dspark.version=2.4.0 \ - -Dscala.version=2.12 \ - -Dspark.major.version=2.4 + -Dscala.version=2.12.10 \ + -Dspark.major.version=2.4 \ + -Dscala.major.version=2.12 - name: Build spark connector 3.1 run: | cd spark-doris-connector && mvn clean package ${MVN_OPT} \ -Dspark.version=3.1.2 \ - -Dscala.version=2.12 \ - -Dspark.major.version=3.1 + -Dscala.version=2.12.10 \ + -Dspark.major.version=3.1 \ + -Dscala.major.version=2.12 - name: Build spark connector 3.2 run: | cd spark-doris-connector && mvn clean package ${MVN_OPT} \ -Dspark.version=3.2.0 \ - -Dscala.version=2.12 \ - -Dspark.major.version=3.2 + -Dscala.version=2.12.10 \ + -Dspark.major.version=3.2 \ + -Dscala.major.version=2.12 - name: Build spark connector 3.3 run: | cd spark-doris-connector && mvn clean package ${MVN_OPT} \ -Dspark.version=3.3.0 \ - -Dscala.version=2.12 \ - -Dspark.major.version=3.3 + -Dscala.version=2.12.10 \ + -Dspark.major.version=3.3 \ + -Dscala.major.version=2.12 - name: Build spark connector 3.4 run: | cd spark-doris-connector && mvn clean package ${MVN_OPT} \ -Dspark.version=3.4.0 \ - -Dscala.version=2.12 \ - -Dspark.major.version=3.4 + -Dscala.version=2.12.10 \ + -Dspark.major.version=3.4 \ + -Dscala.major.version=2.12 diff --git a/spark-doris-connector/build.sh b/spark-doris-connector/build.sh index ab2f770a..3c125e88 100755 --- a/spark-doris-connector/build.sh +++ b/spark-doris-connector/build.sh @@ -170,9 +170,15 @@ selectSpark() { done } +SCALA_VERSION=0 selectScala ScalaVer=$? -SCALA_VERSION="2.1${ScalaVer}" +if [ ${ScalaVer} -eq 1 ]; then + SCALA_VERSION="2.11.8" +elif [ ${ScalaVer} -eq 2 ]; then + SCALA_VERSION="2.12.10" +fi + SPARK_VERSION=0 selectSpark @@ -196,27 +202,30 @@ elif [ ${SparkVer} -eq 7 ]; then SPARK_VERSION=$ver fi -if [[ $SPARK_VERSION =~ ^2.3 && $SCALA_VERSION == "2.12" ]]; then +if [[ $SPARK_VERSION =~ ^2.3 && $SCALA_VERSION == "2.12.10" ]]; then echo_r "Spark 2.3 is not compatible with scala 2.12, will exit." exit 1 -elif [[ $SPARK_VERSION =~ ^3.* && $SCALA_VERSION == "2.11" ]]; then +elif [[ $SPARK_VERSION =~ ^3.* && $SCALA_VERSION == "2.11.8" ]]; then echo_r "Spark 3.x is not compatible with scala 2.11, will exit." exit 1 fi # extract major version: # eg: 3.1.2 -> 3.1 +SCALA_MAJOR_VERSION=0 +[ ${SCALA_VERSION} != 0 ] && SCALA_MAJOR_VERSION=${SCALA_VERSION%.*} SPARK_MAJOR_VERSION=0 [ ${SPARK_VERSION} != 0 ] && SPARK_MAJOR_VERSION=${SPARK_VERSION%.*} -echo_g " scala version: ${SCALA_VERSION}" +echo_g " scala version: ${SCALA_VERSION}, major version: ${SCALA_MAJOR_VERSION}" echo_g " spark version: ${SPARK_VERSION}, major version: ${SPARK_MAJOR_VERSION}" echo_g " build starting..." ${MVN_BIN} clean package \ -Dspark.version=${SPARK_VERSION} \ -Dscala.version=${SCALA_VERSION} \ - -Dspark.major.version=${SPARK_MAJOR_VERSION} "$@" + -Dspark.major.version=${SPARK_MAJOR_VERSION} \ + -Dscala.major.version=${SCALA_MAJOR_VERSION} "$@" EXIT_CODE=$? if [ $EXIT_CODE -eq 0 ]; then diff --git a/spark-doris-connector/pom.xml b/spark-doris-connector/pom.xml index fef519d6..a9fd1809 100644 --- a/spark-doris-connector/pom.xml +++ b/spark-doris-connector/pom.xml @@ -26,7 +26,7 @@ 29 org.apache.doris - spark-doris-connector-${spark.major.version}_${scala.version} + spark-doris-connector-${spark.major.version}_${scala.major.version} ${revision} Spark Doris Connector https://doris.apache.org/ @@ -70,7 +70,8 @@ 1.4.0-SNAPSHOT 3.1.2 3.1 - 2.12 + 2.12.10 + 2.12 0.16.0 13.0.0 UTF-8 @@ -86,6 +87,12 @@ org.apache.doris thrift-service ${thrift-service.version} + + + org.apache.httpcomponents + httpclient + + io.netty @@ -96,14 +103,14 @@ org.apache.spark - spark-core_${scala.version} + spark-core_${scala.major.version} ${spark.version} provided org.apache.spark - spark-sql_${scala.version} + spark-sql_${scala.major.version} ${spark.version} provided @@ -151,7 +158,7 @@ org.mockito - mockito-scala_${scala.version} + mockito-scala_${scala.major.version} 1.4.7 @@ -177,7 +184,7 @@ org.apache.spark - spark-sql-kafka-0-10_${scala.version} + spark-sql-kafka-0-10_${scala.major.version} ${spark.version} test @@ -202,7 +209,7 @@ com.fasterxml.jackson.module - jackson-module-scala_${scala.version} + jackson-module-scala_${scala.major.version} ${fasterxml.jackson.version} @@ -226,6 +233,21 @@ 4.2.0 test + + + + org.scala-lang + scala-library + ${scala.version} + provided + + + + org.apache.httpcomponents + httpclient + 4.5.13 + + @@ -329,6 +351,10 @@ org.apache.thrift org.apache.doris.shaded.org.apache.thrift + + org.apache.http + org.apache.doris.shaded.org.apache.http +