diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aebb95f..08d78bd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,11 +12,11 @@ jobs: fail-fast: false matrix: sqlfluff: - - "2.0.7" + - "3.0.7" extra_requirements_txt: - - "testdata/test_failed_dbt/extra_requirements-1.3.txt" - - "testdata/test_failed_dbt/extra_requirements-1.4.txt" - - "testdata/test_failed_dbt/extra_requirements-1.5.txt" + - "testdata/test_failed_dbt/extra_requirements-1.6.txt" + - "testdata/test_failed_dbt/extra_requirements-1.7.txt" + - "testdata/test_failed_dbt/extra_requirements-1.8.txt" config: - ".sqlfluff.bigquery" - ".sqlfluff.postgres" @@ -29,7 +29,7 @@ jobs: id: lint-sql with: github_token: ${{ secrets.github_token }} - reporter: github-check + reporter: github-pr-review sqlfluff_version: ${{ matrix.sqlfluff }} sqlfluff_command: lint templater: jinja @@ -57,7 +57,6 @@ jobs: fail-fast: false matrix: sqlfluff: - - "2.3.0" - "3.0.7" extra_requirements_txt: - "testdata/test_failed_dbt/extra_requirements-1.6.txt" @@ -75,7 +74,7 @@ jobs: id: lint-sql with: github_token: ${{ secrets.github_token }} - reporter: github-check + reporter: github-pr-review sqlfluff_version: ${{ matrix.sqlfluff }} sqlfluff_command: fix templater: jinja diff --git a/Makefile b/Makefile index a4372f5..7812be0 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ setup-dev: - SQLFLUFF_VERSION=1.2.0 pip install --force-reinstall -r requirements/requirements.txt + SQLFLUFF_VERSION=3.0.7 python -m pip install --force-reinstall -r requirements/requirements.txt pre-commit install lint: lint-shell lint-json lint-docker diff --git a/entrypoint.sh b/entrypoint.sh index 6907941..68bc6bc 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -19,16 +19,16 @@ git fetch --prune --unshallow --no-tags SQL_FILE_PATTERN="${FILE_PATTERN:?}" SOURCE_REFERENCE="origin/${GITHUB_PULL_REQUEST_BASE_REF:?}" changed_files=$(git diff --name-only --no-color "$SOURCE_REFERENCE" "HEAD" -- "${SQLFLUFF_PATHS:?}" | - grep -e "${SQL_FILE_PATTERN:?}" | - xargs -I% bash -c 'if [[ -f "%" ]] ; then echo "%"; fi' || :) + grep -e "${SQL_FILE_PATTERN:?}" | + xargs -I% bash -c 'if [[ -f "%" ]] ; then echo "%"; fi' || :) echo "Changed files:" echo "$changed_files" # Halt the job -if [[ "${changed_files}" == "" ]]; then - echo "There is no changed files. The action doesn't scan files." - echo "name=sqlfluff-exit-code::0" >> $GITHUB_OUTPUT - echo "name=reviewdog-return-code::0" >> $GITHUB_OUTPUT - exit 0 +if [[ ${changed_files} == "" ]]; then + echo "There is no changed files. The action doesn't scan files." + echo "name=sqlfluff-exit-code::0" >>$GITHUB_OUTPUT + echo "name=reviewdog-return-code::0" >>$GITHUB_OUTPUT + exit 0 fi echo '::endgroup::' @@ -42,127 +42,127 @@ echo '::endgroup::' # Install extra python modules echo '::group:: Installing extra python modules' if [[ "x${EXTRA_REQUIREMENTS_TXT}" != "x" ]]; then - pip install --no-cache-dir -r "${EXTRA_REQUIREMENTS_TXT}" --use-deprecated=legacy-resolver - # Make sure the installed modules - pip list + pip install --no-cache-dir -r "${EXTRA_REQUIREMENTS_TXT}" --use-deprecated=legacy-resolver + # Make sure the installed modules + pip list fi echo '::endgroup::' # Install dbt packages echo '::group:: Installing dbt packages' if [[ -f "${INPUT_WORKING_DIRECTORY}/packages.yml" ]]; then - default_dir="$(pwd)" - cd "$INPUT_WORKING_DIRECTORY" - dbt deps --profiles-dir "${SCRIPT_DIR}/resources/dummy_profiles" - cd "$default_dir" + default_dir="$(pwd)" + cd "$INPUT_WORKING_DIRECTORY" + dbt deps --profiles-dir "${SCRIPT_DIR}/resources/dummy_profiles" + cd "$default_dir" fi echo '::endgroup::' # Lint changed files if the mode is lint -if [[ "${SQLFLUFF_COMMAND:?}" == "lint" ]]; then - echo '::group:: Running sqlfluff 🐶 ...' - # Allow failures now, as reviewdog handles them - set +Eeuo pipefail - lint_results="sqlfluff-lint.json" - # shellcheck disable=SC2086,SC2046 - sqlfluff lint \ - --format json \ - $(if [[ "x${SQLFLUFF_CONFIG}" != "x" ]]; then echo "--config ${SQLFLUFF_CONFIG}"; fi) \ - $(if [[ "x${SQLFLUFF_DIALECT}" != "x" ]]; then echo "--dialect ${SQLFLUFF_DIALECT}"; fi) \ - $(if [[ "x${SQLFLUFF_PROCESSES}" != "x" ]]; then echo "--processes ${SQLFLUFF_PROCESSES}"; fi) \ - $(if [[ "x${SQLFLUFF_RULES}" != "x" ]]; then echo "--rules ${SQLFLUFF_RULES}"; fi) \ - $(if [[ "x${SQLFLUFF_EXCLUDE_RULES}" != "x" ]]; then echo "--exclude-rules ${SQLFLUFF_EXCLUDE_RULES}"; fi) \ - $(if [[ "x${SQLFLUFF_TEMPLATER}" != "x" ]]; then echo "--templater ${SQLFLUFF_TEMPLATER}"; fi) \ - $(if [[ "x${SQLFLUFF_DISABLE_NOQA}" != "x" ]]; then echo "--disable-noqa ${SQLFLUFF_DISABLE_NOQA}"; fi) \ - $(if [[ "x${SQLFLUFF_DIALECT}" != "x" ]]; then echo "--dialect ${SQLFLUFF_DIALECT}"; fi) \ - $changed_files | - tee "$lint_results" - sqlfluff_exit_code=$? - - echo "name=sqlfluff-results::$(cat <"$lint_results" | jq -r -c '.')" >> $GITHUB_OUTPUT # Convert to a single line - echo "name=sqlfluff-exit-code::${sqlfluff_exit_code}" >> $GITHUB_OUTPUT - - set -Eeuo pipefail - echo '::endgroup::' - - echo '::group:: Running reviewdog 🐶 ...' - # Allow failures now, as reviewdog handles them - set +Eeuo pipefail - - lint_results_rdjson="sqlfluff-lint.rdjson" - cat <"$lint_results" | - jq -r -f "${SCRIPT_DIR}/to-rdjson.jq" | - tee >"$lint_results_rdjson" - - cat <"$lint_results_rdjson" | - reviewdog -f=rdjson \ - -name="sqlfluff-lint" \ - -reporter="${REVIEWDOG_REPORTER}" \ - -filter-mode="${REVIEWDOG_FILTER_MODE}" \ - -fail-on-error="${REVIEWDOG_FAIL_ON_ERROR}" \ - -level="${REVIEWDOG_LEVEL}" - reviewdog_return_code="${PIPESTATUS[1]}" - - echo "name=sqlfluff-results-rdjson::$(cat <"$lint_results_rdjson" | jq -r -c '.')" >> $GITHUB_OUTPUT # Convert to a single line - echo "name=reviewdog-return-code::${reviewdog_return_code}" >> $GITHUB_OUTPUT - - set -Eeuo pipefail - echo '::endgroup::' - - exit $sqlfluff_exit_code +if [[ ${SQLFLUFF_COMMAND:?} == "lint" ]]; then + echo '::group:: Running sqlfluff 🐶 ...' + # Allow failures now, as reviewdog handles them + set +Eeuo pipefail + lint_results="sqlfluff-lint.json" + # shellcheck disable=SC2086,SC2046 + sqlfluff lint \ + --format json \ + $(if [[ "x${SQLFLUFF_CONFIG}" != "x" ]]; then echo "--config ${SQLFLUFF_CONFIG}"; fi) \ + $(if [[ "x${SQLFLUFF_DIALECT}" != "x" ]]; then echo "--dialect ${SQLFLUFF_DIALECT}"; fi) \ + $(if [[ "x${SQLFLUFF_PROCESSES}" != "x" ]]; then echo "--processes ${SQLFLUFF_PROCESSES}"; fi) \ + $(if [[ "x${SQLFLUFF_RULES}" != "x" ]]; then echo "--rules ${SQLFLUFF_RULES}"; fi) \ + $(if [[ "x${SQLFLUFF_EXCLUDE_RULES}" != "x" ]]; then echo "--exclude-rules ${SQLFLUFF_EXCLUDE_RULES}"; fi) \ + $(if [[ "x${SQLFLUFF_TEMPLATER}" != "x" ]]; then echo "--templater ${SQLFLUFF_TEMPLATER}"; fi) \ + $(if [[ "x${SQLFLUFF_DISABLE_NOQA}" != "x" ]]; then echo "--disable-noqa ${SQLFLUFF_DISABLE_NOQA}"; fi) \ + $(if [[ "x${SQLFLUFF_DIALECT}" != "x" ]]; then echo "--dialect ${SQLFLUFF_DIALECT}"; fi) \ + $changed_files | + tee "$lint_results" + sqlfluff_exit_code=$? + + echo "name=sqlfluff-results::$(cat <"$lint_results" | jq -r -c '.')" >>$GITHUB_OUTPUT # Convert to a single line + echo "name=sqlfluff-exit-code::${sqlfluff_exit_code}" >>$GITHUB_OUTPUT + + set -Eeuo pipefail + echo '::endgroup::' + + echo '::group:: Running reviewdog 🐶 ...' + # Allow failures now, as reviewdog handles them + set +Eeuo pipefail + + lint_results_rdjson="sqlfluff-lint.rdjson" + cat <"$lint_results" | + jq -r -f "${SCRIPT_DIR}/to-rdjson.jq" | + tee >"$lint_results_rdjson" + + cat <"$lint_results_rdjson" | + reviewdog -f=rdjson \ + -name="sqlfluff-lint" \ + -reporter="${REVIEWDOG_REPORTER}" \ + -filter-mode="${REVIEWDOG_FILTER_MODE}" \ + -fail-on-error="${REVIEWDOG_FAIL_ON_ERROR}" \ + -level="${REVIEWDOG_LEVEL}" + reviewdog_return_code="${PIPESTATUS[1]}" + + echo "name=sqlfluff-results-rdjson::$(cat <"$lint_results_rdjson" | jq -r -c '.')" >>$GITHUB_OUTPUT # Convert to a single line + echo "name=reviewdog-return-code::${reviewdog_return_code}" >>$GITHUB_OUTPUT + + set -Eeuo pipefail + echo '::endgroup::' + + exit $sqlfluff_exit_code # END OF lint # Format changed files if the mode is fix -elif [[ "${SQLFLUFF_COMMAND}" == "fix" ]]; then - echo '::group:: Running sqlfluff 🐶 ...' - # Allow failures now, as reviewdog handles them - set +Eeuo pipefail - # shellcheck disable=SC2086,SC2046 - sqlfluff fix --force \ - $(if [[ "x${SQLFLUFF_CONFIG}" != "x" ]]; then echo "--config ${SQLFLUFF_CONFIG}"; fi) \ - $(if [[ "x${SQLFLUFF_DIALECT}" != "x" ]]; then echo "--dialect ${SQLFLUFF_DIALECT}"; fi) \ - $(if [[ "x${SQLFLUFF_PROCESSES}" != "x" ]]; then echo "--processes ${SQLFLUFF_PROCESSES}"; fi) \ - $(if [[ "x${SQLFLUFF_RULES}" != "x" ]]; then echo "--rules ${SQLFLUFF_RULES}"; fi) \ - $(if [[ "x${SQLFLUFF_EXCLUDE_RULES}" != "x" ]]; then echo "--exclude-rules ${SQLFLUFF_EXCLUDE_RULES}"; fi) \ - $(if [[ "x${SQLFLUFF_TEMPLATER}" != "x" ]]; then echo "--templater ${SQLFLUFF_TEMPLATER}"; fi) \ - $(if [[ "x${SQLFLUFF_DISABLE_NOQA}" != "x" ]]; then echo "--disable-noqa ${SQLFLUFF_DISABLE_NOQA}"; fi) \ - $(if [[ "x${SQLFLUFF_DIALECT}" != "x" ]]; then echo "--dialect ${SQLFLUFF_DIALECT}"; fi) \ - $changed_files - sqlfluff_exit_code=$? - echo "name=sqlfluff-exit-code::${sqlfluff_exit_code}" >> $GITHUB_OUTPUT - - set -Eeuo pipefail - echo '::endgroup::' - - # SEE https://github.com/reviewdog/action-suggester/blob/master/script.sh - echo '::group:: Running reviewdog 🐶 ...' - # Allow failures now, as reviewdog handles them - set +Eeuo pipefail - - # Suggest the differences - temp_file=$(mktemp) - git diff | tee "${temp_file}" - git stash -u - - # shellcheck disable=SC2034 - reviewdog \ - -name="sqlfluff-fix" \ - -f=diff \ - -f.diff.strip=1 \ - -reporter="${REVIEWDOG_REPORTER}" \ - -filter-mode="${REVIEWDOG_FILTER_MODE}" \ - -fail-on-error="${REVIEWDOG_FAIL_ON_ERROR}" \ - -level="${REVIEWDOG_LEVEL}" <"${temp_file}" || exit_code=$? - - # Clean up - git stash drop || true - set -Eeuo pipefail - echo '::endgroup::' - - exit $sqlfluff_exit_code - # exit $exit_code +elif [[ ${SQLFLUFF_COMMAND} == "fix" ]]; then + echo '::group:: Running sqlfluff 🐶 ...' + # Allow failures now, as reviewdog handles them + set +Eeuo pipefail + # shellcheck disable=SC2086,SC2046 + sqlfluff fix \ + $(if [[ "x${SQLFLUFF_CONFIG}" != "x" ]]; then echo "--config ${SQLFLUFF_CONFIG}"; fi) \ + $(if [[ "x${SQLFLUFF_DIALECT}" != "x" ]]; then echo "--dialect ${SQLFLUFF_DIALECT}"; fi) \ + $(if [[ "x${SQLFLUFF_PROCESSES}" != "x" ]]; then echo "--processes ${SQLFLUFF_PROCESSES}"; fi) \ + $(if [[ "x${SQLFLUFF_RULES}" != "x" ]]; then echo "--rules ${SQLFLUFF_RULES}"; fi) \ + $(if [[ "x${SQLFLUFF_EXCLUDE_RULES}" != "x" ]]; then echo "--exclude-rules ${SQLFLUFF_EXCLUDE_RULES}"; fi) \ + $(if [[ "x${SQLFLUFF_TEMPLATER}" != "x" ]]; then echo "--templater ${SQLFLUFF_TEMPLATER}"; fi) \ + $(if [[ "x${SQLFLUFF_DISABLE_NOQA}" != "x" ]]; then echo "--disable-noqa ${SQLFLUFF_DISABLE_NOQA}"; fi) \ + $(if [[ "x${SQLFLUFF_DIALECT}" != "x" ]]; then echo "--dialect ${SQLFLUFF_DIALECT}"; fi) \ + $changed_files + sqlfluff_exit_code=$? + echo "name=sqlfluff-exit-code::${sqlfluff_exit_code}" >>$GITHUB_OUTPUT + + set -Eeuo pipefail + echo '::endgroup::' + + # SEE https://github.com/reviewdog/action-suggester/blob/master/script.sh + echo '::group:: Running reviewdog 🐶 ...' + # Allow failures now, as reviewdog handles them + set +Eeuo pipefail + + # Suggest the differences + temp_file=$(mktemp) + git diff | tee "${temp_file}" + git stash -u + + # shellcheck disable=SC2034 + reviewdog \ + -name="sqlfluff-fix" \ + -f=diff \ + -f.diff.strip=1 \ + -reporter="${REVIEWDOG_REPORTER}" \ + -filter-mode="${REVIEWDOG_FILTER_MODE}" \ + -fail-on-error="${REVIEWDOG_FAIL_ON_ERROR}" \ + -level="${REVIEWDOG_LEVEL}" <"${temp_file}" || exit_code=$? + + # Clean up + git stash drop || true + set -Eeuo pipefail + echo '::endgroup::' + + exit $sqlfluff_exit_code + # exit $exit_code # END OF fix else - echo 'ERROR: SQLFLUFF_COMMAND must be one of lint and fix' - exit 1 + echo 'ERROR: SQLFLUFF_COMMAND must be one of lint and fix' + exit 1 fi diff --git a/testdata/test_failed_dbt/models/staging/staging_02.sql b/testdata/test_failed_dbt/models/staging/staging_02.sql index b89cb15..d71d17d 100644 --- a/testdata/test_failed_dbt/models/staging/staging_02.sql +++ b/testdata/test_failed_dbt/models/staging/staging_02.sql @@ -4,5 +4,6 @@ ) }} + SELECT 1 2 3