Main #28
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'Main' | |
| on: | |
| push: | |
| pull_request: | |
| create: | |
| tags: | |
| - v* | |
| workflow_dispatch: | |
| env: | |
| DUCKDB_JSON: 1 | |
| jobs: | |
| clang_format: | |
| name: Clang-Format | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: 'recursive' | |
| - name: Lint ./lib | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| python3 ./scripts/run_clang_format.py \ | |
| --exclude ./lib/build \ | |
| --exclude ./lib/third_party \ | |
| -r ./lib/ | |
| eslint: | |
| name: ESLint | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: 'recursive' | |
| - name: Cache node_modules | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ./node_modules | |
| ./packages/benchmarks/node_modules | |
| ./packages/duckdb-wasm/node_modules | |
| ./packages/duckdb-wasm-shell/node_modules | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('./yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn- | |
| - name: Prepare repository | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| yarn install --frozen-lockfile --prefer-offline | |
| - name: Lint @duckdb/duckdb-wasm | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| yarn workspace @duckdb/duckdb-wasm run lint | |
| - name: Lint @duckdb/duckdb-wasm-shell | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| yarn workspace @duckdb/duckdb-wasm-shell run lint | |
| - name: Lint @duckdb/benchmarks | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| yarn workspace @duckdb/benchmarks run lint | |
| tpchgen: | |
| name: TPCH Generator | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: 'recursive' | |
| - name: Cache generator | |
| uses: actions/cache@v3 | |
| id: cache-generator | |
| with: | |
| path: ./submodules/tpch-dbgen/dbgen/dbgen | |
| key: ${{ runner.os }}-tpch-dbgen | |
| - name: Build generator | |
| if: steps.cache-generator.outputs.cache-hit != 'true' | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| make -C ./submodules/tpch-dbgen/dbgen/ dbgen | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: tpch-dbgen | |
| path: | | |
| ./submodules/tpch-dbgen/dbgen/dbgen | |
| retention-days: 1 | |
| dataprep: | |
| name: Dataprep | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: 'recursive' | |
| - name: Cache rust build | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ./.cargo/bin/ | |
| ./.cargo/registry/index/ | |
| ./.cargo/registry/cache/ | |
| ./.cargo/git/db/ | |
| ./target | |
| key: ${{ runner.os }}-dataprep-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./tools/dataprep/src/*.rs') }} | |
| restore-keys: | | |
| ${{ runner.os }}-dataprep- | |
| - name: Build generator | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| cargo build --manifest-path=./Cargo.toml --release -p dataprep | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: dataprep | |
| path: | | |
| ./target/release/dataprep | |
| retention-days: 1 | |
| duckdb_shell: | |
| name: DuckDB Shell | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: 'recursive' | |
| - name: Git submodule status | |
| run: | | |
| git submodule status ./submodules/duckdb > git_submodule_status.txt | |
| - name: Cache ccache | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ./.ccache | |
| key: ${{ runner.os }}-duckdb-${{ hashFiles('git_submodule_status.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-duckdb- | |
| - name: Build DuckDB shell | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| ccache -z | |
| ./scripts/build_duckdb_shell.sh | |
| ccache -s | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: duckdb-shell | |
| path: ./submodules/duckdb/build/Release/duckdb | |
| retention-days: 1 | |
| native_debug: | |
| name: Native / Debug | |
| runs-on: ubuntu-latest | |
| needs: | |
| - dataprep | |
| - tpchgen | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: 'recursive' | |
| fetch-depth: 0 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: dataprep | |
| path: ./target/release/ | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: tpch-dbgen | |
| path: ./submodules/tpch-dbgen/dbgen/ | |
| - name: Git submodule status | |
| run: | | |
| git submodule status > git_submodule_status.txt | |
| - name: Cache native build | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ./.cargo/bin/ | |
| ./.cargo/registry/index/ | |
| ./.cargo/registry/cache/ | |
| ./.cargo/git/db/ | |
| ./.ccache | |
| ./.emscripten_cache | |
| key: ${{ runner.os }}-native2-debug-${{ hashFiles('git_submodule_status.txt') }}-${{ hashFiles('lib/src/**') }}-${{ hashFiles('lib/include/**') }} | |
| restore-keys: | | |
| ${{ runner.os }}-native2-debug-${{ hashFiles('git_submodule_status.txt') }} | |
| ${{ runner.os }}-native2-debug- | |
| ${{ runner.os }}-native2- | |
| - name: Prepare repository | |
| run: | | |
| chown -R $(id -u):$(id -g) $PWD | |
| (cd ./submodules/duckdb && git fetch --all --tags) | |
| cd submodules/duckdb && git apply ../../duckdb.patch | |
| - name: Prepare environment | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| mkdir -p ./lib/build/debug ./reports | |
| ./scripts/generate_tpch_tbl.sh 0.01 | |
| ./scripts/generate_tpch_arrow.sh 0.01 | |
| ./scripts/generate_uni.sh | |
| ccache --max-size 200M | |
| ccache -s | |
| - name: Build project | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| mkdir -p ./.ccache/extension | |
| touch ./.ccache/extension/json | |
| cmake \ | |
| -S./lib/ \ | |
| -B./lib/build/debug \ | |
| -DCODE_COVERAGE=1 \ | |
| -DCMAKE_C_COMPILER_LAUNCHER=ccache \ | |
| -DCMAKE_C_COMPILER=clang \ | |
| -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ | |
| -DCMAKE_CXX_COMPILER=clang++ \ | |
| -DCMAKE_BUILD_TYPE=Debug | |
| ccache -z | |
| make -C./lib/build/debug -j`nproc` | |
| ccache -s | |
| - name: Test project | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| export LLVM_PROFILE_FILE=./reports/lib.profraw | |
| ./lib/build/debug/tester \ | |
| --source_dir=./lib/ \ | |
| --gtest_output=xml:./reports/tests_lib_debug.xml | |
| - name: Code Coverage | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| llvm-profdata merge \ | |
| -sparse ./reports/lib.profraw \ | |
| -o ./reports/lib.profdata | |
| llvm-cov show ./lib/build/debug/tester \ | |
| --instr-profile ./reports/lib.profdata \ | |
| -ignore-filename-regex submodules \ | |
| > ./reports/coverage_lib.txt | |
| native_release: | |
| name: Native / Release | |
| runs-on: ubuntu-latest | |
| needs: | |
| - dataprep | |
| - tpchgen | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: 'recursive' | |
| fetch-depth: 0 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: dataprep | |
| path: ./target/release/ | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: tpch-dbgen | |
| path: ./submodules/tpch-dbgen/dbgen/ | |
| - name: Git submodule status | |
| run: | | |
| git submodule status > git_submodule_status.txt | |
| - name: Cache native build | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ./.cargo/bin/ | |
| ./.cargo/registry/index/ | |
| ./.cargo/registry/cache/ | |
| ./.cargo/git/db/ | |
| ./.ccache | |
| ./.emscripten_cache | |
| key: ${{ runner.os }}-native2-release-${{ hashFiles('git_submodule_status.txt') }}-${{ hashFiles('lib/src/**') }}-${{ hashFiles('lib/include/**') }} | |
| restore-keys: | | |
| ${{ runner.os }}-native2-release-${{ hashFiles('git_submodule_status.txt') }} | |
| ${{ runner.os }}-native2-release- | |
| ${{ runner.os }}-native2- | |
| - name: Prepare repository | |
| run: | | |
| chown -R $(id -u):$(id -g) $PWD | |
| (cd ./submodules/duckdb && git fetch --all --tags) | |
| cd submodules/duckdb && git apply ../../duckdb.patch | |
| - name: Prepare environment | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| mkdir -p ./lib/build/release ./reports | |
| ./scripts/generate_tpch_tbl.sh 0.01 | |
| ./scripts/generate_tpch_arrow.sh 0.01 | |
| ./scripts/generate_uni.sh | |
| ccache --max-size 200M | |
| ccache -s | |
| - name: Build project | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| mkdir -p ./.ccache/extension | |
| touch ./.ccache/extension/json | |
| cmake \ | |
| -S./lib/ \ | |
| -B./lib/build/release \ | |
| -DCODE_COVERAGE=1 \ | |
| -DCMAKE_C_COMPILER_LAUNCHER=ccache \ | |
| -DCMAKE_C_COMPILER=clang \ | |
| -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ | |
| -DCMAKE_CXX_COMPILER=clang++ \ | |
| -DCMAKE_BUILD_TYPE=Release | |
| ccache -z | |
| make -C./lib/build/release -j`nproc` | |
| ccache -s | |
| - name: Test project | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| export LLVM_PROFILE_FILE=./reports/lib.profraw | |
| ./lib/build/release/tester \ | |
| --source_dir=./lib/ \ | |
| --gtest_output=xml:./reports/tests_lib_release.xml | |
| - name: Code Coverage | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| llvm-profdata merge \ | |
| -sparse ./reports/lib.profraw \ | |
| -o ./reports/lib.profdata | |
| llvm-cov show ./lib/build/release/tester \ | |
| --instr-profile ./reports/lib.profdata \ | |
| -ignore-filename-regex submodules \ | |
| > ./reports/coverage_lib.txt | |
| wasm_mvp: | |
| name: Wasm / mvp | |
| runs-on: ubuntu-latest | |
| needs: | |
| - dataprep | |
| - tpchgen | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: 'recursive' | |
| fetch-depth: 0 | |
| - uses: mymindstorm/setup-emsdk@v12 | |
| with: | |
| version: 3.1.24 | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| - name: Git submodule status | |
| run: | | |
| git submodule status > git_submodule_status.txt | |
| - name: Build Wasm module | |
| run: | | |
| ./scripts/wasm_build_lib.sh relperf mvp | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wasm-mvp | |
| path: | | |
| ./packages/duckdb-wasm/src/bindings/duckdb-mvp.js | |
| ./packages/duckdb-wasm/src/bindings/duckdb-mvp.wasm | |
| retention-days: 1 | |
| wasm_eh: | |
| name: Wasm / eh | |
| runs-on: ubuntu-latest | |
| needs: | |
| - dataprep | |
| - tpchgen | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: 'recursive' | |
| fetch-depth: 0 | |
| - uses: mymindstorm/setup-emsdk@v12 | |
| with: | |
| version: 3.1.24 | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| - name: Git submodule status | |
| run: | | |
| git submodule status > git_submodule_status.txt | |
| - name: Build Wasm module | |
| run: | | |
| ./scripts/wasm_build_lib.sh relperf eh | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wasm-eh | |
| path: | | |
| ./packages/duckdb-wasm/src/bindings/duckdb-eh.js | |
| ./packages/duckdb-wasm/src/bindings/duckdb-eh.wasm | |
| retention-days: 1 | |
| wasm_coi: | |
| name: Wasm / coi | |
| runs-on: ubuntu-latest | |
| needs: | |
| - dataprep | |
| - tpchgen | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: 'recursive' | |
| fetch-depth: 0 | |
| - uses: mymindstorm/setup-emsdk@v12 | |
| with: | |
| version: 3.1.24 | |
| - name: Setup Ccache | |
| uses: hendrikmuhs/ccache-action@main | |
| with: | |
| key: ${{ github.job }} | |
| - name: Git submodule status | |
| run: | | |
| git submodule status > git_submodule_status.txt | |
| - name: Build Wasm module | |
| run: | | |
| ./scripts/wasm_build_lib.sh relperf coi | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wasm-coi | |
| path: | | |
| ./packages/duckdb-wasm/src/bindings/duckdb-coi.js | |
| ./packages/duckdb-wasm/src/bindings/duckdb-coi.pthread.js | |
| ./packages/duckdb-wasm/src/bindings/duckdb-coi.wasm | |
| retention-days: 1 | |
| js_libs: | |
| name: Js / Libraries | |
| runs-on: ubuntu-latest | |
| needs: | |
| - native_debug | |
| - native_release | |
| - wasm_mvp | |
| - wasm_eh | |
| - wasm_coi | |
| - duckdb_shell | |
| - clang_format | |
| - eslint | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: 'recursive' | |
| fetch-depth: 0 | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: '16.x' | |
| registry-url: 'https://registry.npmjs.org' | |
| - name: Git submodule status | |
| run: | | |
| git submodule status > git_submodule_status.txt | |
| - name: Cache rust build | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ./.cargo/bin/ | |
| ./.cargo/registry/index/ | |
| ./.cargo/registry/cache/ | |
| ./.cargo/git/db/ | |
| ./target | |
| key: ${{ runner.os }}-shell-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./packages/duckdb-wasm-shell/crate/src/**') }} | |
| restore-keys: | | |
| ${{ runner.os }}-shell- | |
| - name: Cache node_modules | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ./node_modules | |
| ./packages/benchmarks/node_modules | |
| ./packages/duckdb-wasm/node_modules | |
| ./packages/duckdb-wasm-shell/node_modules | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('./yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn- | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: dataprep | |
| path: ./target/release/ | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: tpch-dbgen | |
| path: ./submodules/tpch-dbgen/dbgen/ | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: duckdb-shell | |
| path: ./submodules/duckdb/build/Release/ | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: wasm-mvp | |
| path: ./packages/duckdb-wasm/src/bindings/ | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: wasm-eh | |
| path: ./packages/duckdb-wasm/src/bindings/ | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: wasm-coi | |
| path: ./packages/duckdb-wasm/src/bindings/ | |
| - name: Prepare repository | |
| run: | | |
| chown -R $(id -u):$(id -g) $PWD | |
| git fetch --tags --no-recurse-submodules -f | |
| (cd ./submodules/duckdb && git fetch --all --tags) | |
| - name: Prepare environment | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| chown -R $(id -u):$(id -g) $PWD | |
| git config --global --add safe.directory '*' | |
| mkdir -p ./lib/build/wasm/release ./reports | |
| yarn install --frozen-lockfile | |
| ./scripts/generate_tpch_tbl.sh 0.01 | |
| ./scripts/generate_tpch_arrow.sh 0.01 | |
| ./scripts/generate_tpch_duckdb.sh 0.01 | |
| ./scripts/generate_tpch_sqlite.sh 0.01 | |
| ./scripts/generate_uni.sh | |
| ./scripts/npm_version.sh | |
| - name: Build @duckdb/duckdb-wasm | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| rm -rf ./packages/duckdb-wasm/dist/ | |
| yarn workspace @duckdb/duckdb-wasm build:release | |
| yarn workspace @duckdb/duckdb-wasm docs | |
| - name: Build @duckdb/duckdb-wasm-shell | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| rm -rf ./packages/duckdb-wasm-shell/dist/ | |
| yarn workspace @duckdb/duckdb-wasm-shell install:wasmpack | |
| yarn workspace @duckdb/duckdb-wasm-shell build:release | |
| - name: Build @duckdb/duckdb-wasm-app | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| rm -rf ./packages/duckdb-wasm-app/build/ | |
| yarn workspace @duckdb/duckdb-wasm-app build:release | |
| - name: Test @duckdb/duckdb-wasm on Chrome | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome | |
| # - name: Test @duckdb/duckdb-wasm on Firefox | |
| # uses: duckdb/[email protected] | |
| # with: | |
| # script: |- | |
| # yarn workspace @duckdb/duckdb-wasm test:firefox | |
| - name: Test @duckdb/duckdb-wasm on Node.js | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| yarn workspace @duckdb/duckdb-wasm test:node | |
| - name: Coverage @duckdb/duckdb-wasm | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome:coverage | |
| - name: Build examples | |
| uses: duckdb/[email protected] | |
| with: | |
| script: |- | |
| yarn install | |
| yarn workspace @duckdb/duckdb-wasm-examples-bare-node test | |
| yarn workspace @duckdb/duckdb-wasm-examples-bare-browser build | |
| yarn workspace @duckdb/duckdb-wasm-examples-esbuild-node build | |
| yarn workspace @duckdb/duckdb-wasm-examples-esbuild-node test | |
| yarn workspace @duckdb/duckdb-wasm-examples-esbuild-browser build | |
| - name: Publish to NPM | |
| env: | |
| NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} | |
| if: env.NODE_AUTH_TOKEN != null && (startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch') | |
| run: | | |
| ./scripts/npm_publish_lib.sh | |
| # ./scripts/npm_publish_shell.sh | |
| # ./scripts/npm_publish_react.sh | |
| - name: Deploy GitHub Page | |
| if: github.ref == 'refs/heads/master' | |
| run: | | |
| git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY | |
| git fetch origin gh-pages | |
| ./scripts/deploy_pages.sh | |
| - name: Package | |
| shell: bash | |
| run: zip -r duckdb-wasm-packages.zip ./packages | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: duckdb-wasm-packages | |
| path: duckdb-wasm-packages.zip |