Skip to content

Commit

Permalink
[Project] Add shaded jar for common and io (#455)
Browse files Browse the repository at this point in the history 
* add shaded jar

Signed-off-by: chenxu <[email protected]>

* free github action runner disk space

Signed-off-by: chenxu <[email protected]>

* refine docker image cache

Signed-off-by: chenxu <[email protected]>

---------

Signed-off-by: chenxu <[email protected]>
Co-authored-by: chenxu <[email protected]>
  • Loading branch information
@xuchen-plus @dmetasoul01
xuchen-plus and dmetasoul01 committed Mar 28, 2024
1 parent 36ae7be commit ae23229
Show file tree
Hide file tree
Showing 13 changed files with 418 additions and 149 deletions.
4 changes: 4 additions & 0 deletions .github/workflows/deployment.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,10 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
workspaces: "./rust -> target"
- name: Cache Docker images
uses: ScribeMD/[email protected]
with:
key: docker-${{ runner.os }}-${{ hashFiles('rust/Cross.toml') }}
- uses: actions-rs/cargo@v1
with:
use-cross: true
Expand Down
14 changes: 14 additions & 0 deletions .github/workflows/flink-cdc-hdfs-test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,16 @@ jobs:
runs-on: ubuntu-latest

steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: false
docker-images: true
swap-storage: true
- uses: actions/checkout@v4
- name: Set up JDK 8
uses: actions/setup-java@v4
Expand Down Expand Up @@ -64,6 +74,10 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
workspaces: "./rust -> target"
- name: Cache Docker images
uses: ScribeMD/[email protected]
with:
key: docker-${{ runner.os }}-${{ hashFiles('rust/Cross.toml', 'docker/lakesoul-docker-compose-env/docker-compose.yml') }}
- name: Pull images
run: |
docker pull -q bitnami/spark:3.3.1
Expand Down
14 changes: 14 additions & 0 deletions .github/workflows/flink-cdc-test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,16 @@ jobs:
runs-on: ubuntu-latest

steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: false
docker-images: true
swap-storage: true
- uses: actions/checkout@v4
- name: Set up JDK 8
uses: actions/setup-java@v4
Expand Down Expand Up @@ -66,6 +76,10 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
workspaces: "./rust -> target"
- name: Cache Docker images
uses: ScribeMD/[email protected]
with:
key: docker-${{ runner.os }}-${{ hashFiles('rust/Cross.toml', 'docker/lakesoul-docker-compose-env/docker-compose.yml') }}
- name: Pull images
run: |
docker pull -q bitnami/spark:3.3.1
Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/maven-test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
workspaces: "./rust -> target"
- name: Cache Docker images
uses: ScribeMD/[email protected]
with:
key: docker-${{ runner.os }}-${{ hashFiles('rust/Cross.toml') }}
- uses: actions-rs/cargo@v1
with:
use-cross: true
Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/native-build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@ jobs:
with:
workspaces: "./rust -> target"
env-vars: "JAVA_HOME"
- name: Cache Docker images
uses: ScribeMD/[email protected]
with:
key: docker-${{ runner.os }}-${{ hashFiles('rust/Cross.toml') }}
- uses: actions-rs/cargo@v1
with:
use-cross: true
Expand Down
14 changes: 14 additions & 0 deletions .github/workflows/presto-cdc-test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,16 @@ jobs:
runs-on: ubuntu-latest

steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: false
docker-images: true
swap-storage: true
- uses: actions/checkout@v4
- name: Set up JDK 8
uses: actions/setup-java@v4
Expand Down Expand Up @@ -66,6 +76,10 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
workspaces: "./rust -> target"
- name: Cache Docker images
uses: ScribeMD/[email protected]
with:
key: docker-${{ runner.os }}-${{ hashFiles('rust/Cross.toml', 'docker/lakesoul-docker-compose-env/docker-compose.yml') }}
- name: Pull images
run: |
docker pull -q bitnami/spark:3.3.1
Expand Down
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,5 @@ __pycache__/
/python/lakesoul.egg-info/
/python/*.whl
/wheelhouse/
/rust/.idea
.flattened-pom.xml
168 changes: 123 additions & 45 deletions lakesoul-common/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,6 @@ SPDX-License-Identifier: Apache-2.0
<version>${revision}</version>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.10.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<!-- ajc -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
Expand Down Expand Up @@ -93,7 +84,7 @@ SPDX-License-Identifier: Apache-2.0
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>4.6.3</version>
<version>4.8.1</version>
<executions>
<execution>
<id>scala-compile-first</id>
Expand All @@ -112,11 +103,107 @@ SPDX-License-Identifier: Apache-2.0
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.13.0</version>
<executions>
<execution>
<phase>compile</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.5.2</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
<configuration>
<shadedArtifactAttached>true</shadedArtifactAttached>
<shadedClassifierName>shaded</shadedClassifierName>
<createDependencyReducedPom>true</createDependencyReducedPom>
<useDependencyReducedPomInJar>true</useDependencyReducedPomInJar>
<artifactSet>
<includes>
<include>**</include>
</includes>
</artifactSet>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>META-INF/versions/**</exclude>
</excludes>
</filter>
</filters>
<relocations>
<relocation>
<pattern>com.zaxxer.hikari</pattern>
<shadedPattern>com.lakesoul.shaded.com.zaxxer.hikari</shadedPattern>
</relocation>
<relocation>
<pattern>org.postgresql</pattern>
<shadedPattern>com.lakesoul.shaded.org.postgresql</shadedPattern>
</relocation>
<relocation>
<pattern>com.alibaba.fastjson</pattern>
<shadedPattern>com.lakesoul.shaded.com.alibaba.fastjson</shadedPattern>
</relocation>
<relocation>
<pattern>org.ow2.asm</pattern>
<shadedPattern>com.lakesoul.shaded.org.ow2.asm</shadedPattern>
</relocation>
<relocation>
<pattern>org.objectweb.asm</pattern>
<shadedPattern>com.lakesoul.shaded.org.objectweb.asm</shadedPattern>
</relocation>
</relocations>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>flatten-maven-plugin</artifactId>
<!--<version>1.6.0</version>-->
<configuration>
<flattenMode>oss</flattenMode>
</configuration>
<executions>
<!-- enable flattening -->
<execution>
<id>flatten</id>
<phase>process-resources</phase>
<goals>
<goal>flatten</goal>
</goals>
</execution>
<!-- ensure proper cleanup -->
<execution>
<id>flatten.clean</id>
<phase>clean</phase>
<goals>
<goal>clean</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>

<properties>
<postgresql.version>42.5.5</postgresql.version>
<postgresql.version>42.7.3</postgresql.version>
<local.scope>provided</local.scope>
<protobuf.plugin.version>0.6.1</protobuf.plugin.version>
<protoc.version>3.5.1</protoc.version>
Expand All @@ -132,12 +219,23 @@ SPDX-License-Identifier: Apache-2.0
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
<version>4.0.3</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.36</version>
<scope>${local.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
Expand All @@ -149,7 +247,6 @@ SPDX-License-Identifier: Apache-2.0
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
<scope>${local.scope}</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
Expand All @@ -161,7 +258,6 @@ SPDX-License-Identifier: Apache-2.0
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
<version>${scala.version}</version>
<scope>${local.scope}</scope>
</dependency>
<!-- casbin -->
<dependency>
Expand All @@ -181,6 +277,10 @@ SPDX-License-Identifier: Apache-2.0
<groupId>com.microsoft.sqlserver</groupId>
<artifactId>mssql-jdbc</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- aspectJ -->
Expand Down Expand Up @@ -209,32 +309,12 @@ SPDX-License-Identifier: Apache-2.0
<dependency>
<groupId>com.github.jnr</groupId>
<artifactId>jnr-ffi</artifactId>
<version>2.2.13</version>
<exclusions>
<exclusion>
<groupId>com.github.jnr</groupId>
<artifactId>jffi</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- https://github.com/jnr/jffi/issues/138 -->
<!-- upgrade jffi version to make it work on CentOS 7 -->
<dependency>
<groupId>com.github.jnr</groupId>
<artifactId>jffi</artifactId>
<version>1.3.11</version>
</dependency>
<dependency>
<groupId>com.github.jnr</groupId>
<artifactId>jffi</artifactId>
<version>1.3.11</version>
<scope>runtime</scope>
<classifier>native</classifier>
<version>2.2.16</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-api</artifactId>
<version>3.3.2</version>
<version>3.3.6</version>
<scope>${local.scope}</scope>
</dependency>

Expand All @@ -251,9 +331,9 @@ SPDX-License-Identifier: Apache-2.0
<resource>
<directory>${basedir}/../rust/target/release/</directory>
<includes>
<include>*.dylib</include>
<include>*.so</include>
<include>*.dll</include>
<include>liblakesoul_metadata_c.dylib</include>
<include>liblakesoul_metadata_c.so</include>
<include>lakesoul_metadata_c.dll</include>
</includes>
<excludes>
<exclude>deps</exclude>
Expand Down Expand Up @@ -313,7 +393,7 @@ SPDX-License-Identifier: Apache-2.0
<resource>
<directory>${basedir}/../rust/target/x86_64-unknown-linux-gnu/release/</directory>
<includes>
<include>*.so</include>
<include>liblakesoul_metadata_c.so</include>
</includes>
<excludes>
<exclude>deps</exclude>
Expand Down Expand Up @@ -374,9 +454,9 @@ SPDX-License-Identifier: Apache-2.0
<resource>
<directory>${basedir}/../rust/target/release/</directory>
<includes>
<include>*.dylib</include>
<include>*.so</include>
<include>*.dll</include>
<include>liblakesoul_metadata_c.dylib</include>
<include>liblakesoul_metadata_c.so</include>
<include>lakesoul_metadata_c.dll</include>
</includes>
<excludes>
<exclude>deps</exclude>
Expand All @@ -390,6 +470,4 @@ SPDX-License-Identifier: Apache-2.0
</build>
</profile>
</profiles>


</project>
Loading

0 comments on commit ae23229

Please sign in to comment.