diff --git a/.github/workflows/build-extension.yml b/.github/workflows/build-extension.yml
index bbc68f75..5651c5a0 100644
--- a/.github/workflows/build-extension.yml
+++ b/.github/workflows/build-extension.yml
@@ -46,47 +46,54 @@ jobs:
run: |
cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-Dspark.version=2.3.4 \
- -Dscala.version=2.11 \
- -Dspark.major.version=2.3
+ -Dscala.version=2.11.8 \
+ -Dspark.major.version=2.3 \
+ -Dscala.major.version=2.11
- name: Build spark connector 2.4 2.11
run: |
cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-Dspark.version=2.4.0 \
- -Dscala.version=2.11 \
- -Dspark.major.version=2.4
+ -Dscala.version=2.11.8 \
+ -Dspark.major.version=2.4 \
+ -Dscala.major.version=2.11
- name: Build spark connector 2.4 2.12
run: |
cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-Dspark.version=2.4.0 \
- -Dscala.version=2.12 \
- -Dspark.major.version=2.4
+ -Dscala.version=2.12.10 \
+ -Dspark.major.version=2.4 \
+ -Dscala.major.version=2.12
- name: Build spark connector 3.1
run: |
cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-Dspark.version=3.1.2 \
- -Dscala.version=2.12 \
- -Dspark.major.version=3.1
+ -Dscala.version=2.12.10 \
+ -Dspark.major.version=3.1 \
+ -Dscala.major.version=2.12
- name: Build spark connector 3.2
run: |
cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-Dspark.version=3.2.0 \
- -Dscala.version=2.12 \
- -Dspark.major.version=3.2
+ -Dscala.version=2.12.10 \
+ -Dspark.major.version=3.2 \
+ -Dscala.major.version=2.12
- name: Build spark connector 3.3
run: |
cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-Dspark.version=3.3.0 \
- -Dscala.version=2.12 \
- -Dspark.major.version=3.3
+ -Dscala.version=2.12.10 \
+ -Dspark.major.version=3.3 \
+ -Dscala.major.version=2.12
- name: Build spark connector 3.4
run: |
cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-Dspark.version=3.4.0 \
- -Dscala.version=2.12 \
- -Dspark.major.version=3.4
+ -Dscala.version=2.12.10 \
+ -Dspark.major.version=3.4 \
+ -Dscala.major.version=2.12
diff --git a/spark-doris-connector/build.sh b/spark-doris-connector/build.sh
index ab2f770a..3c125e88 100755
--- a/spark-doris-connector/build.sh
+++ b/spark-doris-connector/build.sh
@@ -170,9 +170,15 @@ selectSpark() {
done
}
+SCALA_VERSION=0
selectScala
ScalaVer=$?
-SCALA_VERSION="2.1${ScalaVer}"
+if [ ${ScalaVer} -eq 1 ]; then
+ SCALA_VERSION="2.11.8"
+elif [ ${ScalaVer} -eq 2 ]; then
+ SCALA_VERSION="2.12.10"
+fi
+
SPARK_VERSION=0
selectSpark
@@ -196,27 +202,30 @@ elif [ ${SparkVer} -eq 7 ]; then
SPARK_VERSION=$ver
fi
-if [[ $SPARK_VERSION =~ ^2.3 && $SCALA_VERSION == "2.12" ]]; then
+if [[ $SPARK_VERSION =~ ^2.3 && $SCALA_VERSION == "2.12.10" ]]; then
echo_r "Spark 2.3 is not compatible with scala 2.12, will exit."
exit 1
-elif [[ $SPARK_VERSION =~ ^3.* && $SCALA_VERSION == "2.11" ]]; then
+elif [[ $SPARK_VERSION =~ ^3.* && $SCALA_VERSION == "2.11.8" ]]; then
echo_r "Spark 3.x is not compatible with scala 2.11, will exit."
exit 1
fi
# extract major version:
# eg: 3.1.2 -> 3.1
+SCALA_MAJOR_VERSION=0
+[ ${SCALA_VERSION} != 0 ] && SCALA_MAJOR_VERSION=${SCALA_VERSION%.*}
SPARK_MAJOR_VERSION=0
[ ${SPARK_VERSION} != 0 ] && SPARK_MAJOR_VERSION=${SPARK_VERSION%.*}
-echo_g " scala version: ${SCALA_VERSION}"
+echo_g " scala version: ${SCALA_VERSION}, major version: ${SCALA_MAJOR_VERSION}"
echo_g " spark version: ${SPARK_VERSION}, major version: ${SPARK_MAJOR_VERSION}"
echo_g " build starting..."
${MVN_BIN} clean package \
-Dspark.version=${SPARK_VERSION} \
-Dscala.version=${SCALA_VERSION} \
- -Dspark.major.version=${SPARK_MAJOR_VERSION} "$@"
+ -Dspark.major.version=${SPARK_MAJOR_VERSION} \
+ -Dscala.major.version=${SCALA_MAJOR_VERSION} "$@"
EXIT_CODE=$?
if [ $EXIT_CODE -eq 0 ]; then
diff --git a/spark-doris-connector/pom.xml b/spark-doris-connector/pom.xml
index fef519d6..a9fd1809 100644
--- a/spark-doris-connector/pom.xml
+++ b/spark-doris-connector/pom.xml
@@ -26,7 +26,7 @@
29
org.apache.doris
- spark-doris-connector-${spark.major.version}_${scala.version}
+ spark-doris-connector-${spark.major.version}_${scala.major.version}
${revision}
Spark Doris Connector
https://doris.apache.org/
@@ -70,7 +70,8 @@
1.4.0-SNAPSHOT
3.1.2
3.1
- 2.12
+ 2.12.10
+ 2.12
0.16.0
13.0.0
UTF-8
@@ -86,6 +87,12 @@
org.apache.doris
thrift-service
${thrift-service.version}
+
+
+ org.apache.httpcomponents
+ httpclient
+
+
io.netty
@@ -96,14 +103,14 @@
org.apache.spark
- spark-core_${scala.version}
+ spark-core_${scala.major.version}
${spark.version}
provided
org.apache.spark
- spark-sql_${scala.version}
+ spark-sql_${scala.major.version}
${spark.version}
provided
@@ -151,7 +158,7 @@
org.mockito
- mockito-scala_${scala.version}
+ mockito-scala_${scala.major.version}
1.4.7
@@ -177,7 +184,7 @@
org.apache.spark
- spark-sql-kafka-0-10_${scala.version}
+ spark-sql-kafka-0-10_${scala.major.version}
${spark.version}
test
@@ -202,7 +209,7 @@
com.fasterxml.jackson.module
- jackson-module-scala_${scala.version}
+ jackson-module-scala_${scala.major.version}
${fasterxml.jackson.version}
@@ -226,6 +233,21 @@
4.2.0
test
+
+
+
+ org.scala-lang
+ scala-library
+ ${scala.version}
+ provided
+
+
+
+ org.apache.httpcomponents
+ httpclient
+ 4.5.13
+
+
@@ -329,6 +351,10 @@
org.apache.thrift
org.apache.doris.shaded.org.apache.thrift
+
+ org.apache.http
+ org.apache.doris.shaded.org.apache.http
+