diff --git a/.github/workflows/metadata-extract-airbyte.yml b/.github/workflows/metadata-extract-airbyte.yml index c38a3a240..50dd8e4e7 100644 --- a/.github/workflows/metadata-extract-airbyte.yml +++ b/.github/workflows/metadata-extract-airbyte.yml @@ -24,7 +24,7 @@ jobs: - uses: actions/checkout@v4.2.2 - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git + run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable - name: Get Variants List - Airbyte id: get-variants-list-airbyte @@ -34,7 +34,7 @@ jobs: id: setmatrix_airbyte run: | matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-airbyte.outputs.VARIANTS_AIRBYTE }}}" - echo "::set-output name=airbyte_matrix::$matrixStringifiedObject" + echo "airbyte_matrix=$matrixStringifiedObject" >> $GITHUB_ENV metadata_extract_airbyte_p1: name: Airbyte - Part 1 Metadata Extract @@ -83,7 +83,7 @@ jobs: name: extractor--${{ matrix.source-name }}--airbyte.json - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git + run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable - name: Upload Metadata run: hub-utils upload-airbyte "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" extractor--${{ matrix.source-name }}--airbyte.json diff --git a/.github/workflows/metadata-extract-extractors.yml b/.github/workflows/metadata-extract-extractors.yml index 0834d63df..3d23e7a09 100644 --- a/.github/workflows/metadata-extract-extractors.yml +++ b/.github/workflows/metadata-extract-extractors.yml @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v4.2.2 - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git + run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable - name: Get Variants List - SDK Extractors (p1) id: get-variants-list-sdk-ex-p1 @@ -35,7 +35,7 @@ jobs: id: setmatrix_sdk_ex_p1 run: | matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-ex-p1.outputs.VARIANTS_SDK_EX_P1 }}}" - echo "::set-output name=sdk_ex_p1_matrix::$matrixStringifiedObject" + echo "sdk_ex_p1_matrix=$matrixStringifiedObject" >> $GITHUB_ENV - name: Get Variants List - SDK Extractors (p2) id: get-variants-list-sdk-ex-p2 @@ -45,7 +45,7 @@ jobs: id: setmatrix_sdk_ex_p2 run: | matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-ex-p2.outputs.VARIANTS_SDK_EX_P2 }}}" - echo "::set-output name=sdk_ex_p2_matrix::$matrixStringifiedObject" + echo "sdk_ex_p2_matrix=$matrixStringifiedObject" >> $GITHUB_ENV metadata_extract_sdk_ex_p1: name: SDK Extractors - Metadata Extract (p1) @@ -70,67 +70,55 @@ jobs: role-session-name: "GitHubActions" - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git + run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable # This starts the attempts to install and extract metadata. Since we don't # know what python versions are accepted we start with the latest and continue # decreasing until we have a success or run out of python versions to attempt. - - name: Install Python 3.12 + - name: Install Python uses: actions/setup-python@v5 with: - python-version: '3.12' - + python-version: | + 3.13 + 3.12 + 3.11 + 3.10 + 3.9 + 3.8 + + - name: Extract Metadata (3.13) + id: extract_313 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.13 + continue-on-error: true + - name: Extract Metadata (3.12) + if: always() && (steps.extract_313.outcome == 'failure') id: extract_312 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.12 continue-on-error: true - - name: Install Python 3.11 (Only if 3.12 failed) - if: always() && (steps.extract_312.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Extract Metadata (3.11) if: always() && (steps.extract_312.outcome == 'failure') id: extract_311 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.11 continue-on-error: true - - name: Install Python 3.10 (Only if 3.11 failed) - if: always() && (steps.extract_311.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - name: Extract Metadata (3.10) if: always() && (steps.extract_311.outcome == 'failure') id: extract_310 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.10 continue-on-error: true - - name: Install Python 3.9 (Only if 3.10 failed) - if: always() && (steps.extract_310.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - name: Extract Metadata (3.9) if: always() && (steps.extract_310.outcome == 'failure') id: extract_39 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.9 continue-on-error: true - - name: Install Python 3.8 (Only if 3.9 failed) - if: always() && (steps.extract_39.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.8' - - name: Extract Metadata (3.8) if: always() && (steps.extract_39.outcome == 'failure') id: extract_38 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.8 metadata_extract_sdk_ex_p2: name: SDK Extractors - Metadata Extract (p2) @@ -155,37 +143,28 @@ jobs: role-session-name: "GitHubActions" - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git + run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable - - name: Install Python 3.10 + - name: Install Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: | + 3.10 + 3.9 + 3.8 - name: Extract Metadata (3.10) id: extract_310 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.10 continue-on-error: true - - name: Install Python 3.9 (Only if 3.10 failed) - if: always() && (steps.extract_310.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - name: Extract Metadata (3.9) if: always() && (steps.extract_310.outcome == 'failure') id: extract_39 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.9 continue-on-error: true - - name: Install Python 3.8 (Only if 3.9 failed) - if: always() && (steps.extract_39.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.8' - - name: Extract Metadata (3.8) if: always() && (steps.extract_39.outcome == 'failure') id: extract_38 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.8 diff --git a/.github/workflows/metadata-extract-loaders.yml b/.github/workflows/metadata-extract-loaders.yml index 1c482c750..07f5d4b96 100644 --- a/.github/workflows/metadata-extract-loaders.yml +++ b/.github/workflows/metadata-extract-loaders.yml @@ -24,7 +24,7 @@ jobs: - uses: actions/checkout@v4.2.2 - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git + run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable - name: Get Variants List - SDK Loaders id: get-variants-list-sdk-loaders @@ -34,7 +34,7 @@ jobs: id: setmatrix_sdk_loaders run: | matrixStringifiedObject="{\"include\": ${{ steps.get-variants-list-sdk-loaders.outputs.VARIANTS_SDK_LOADERS }}}" - echo "::set-output name=sdk_loaders_matrix::$matrixStringifiedObject" + echo "sdk_loaders_matrix=$matrixStringifiedObject" >> $GITHUB_ENV metadata_extract_sdk_loaders: name: SDK Loaders - Metadata Extract @@ -59,7 +59,7 @@ jobs: role-session-name: "GitHubActions" - name: Install hub-utils - run: pipx install git+https://github.com/meltano/hub-utils.git + run: pipx install git+https://github.com/meltano/hub-utils.git@edgarrmondragon/fix/custom-python-executable # This starts the attempts to install and extract metadata. Since we don't # know what python versions are accepted we start with the latest and continue @@ -67,56 +67,44 @@ jobs: - name: Install Python 3.12 uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: | + 3.13 + 3.12 + 3.11 + 3.10 + 3.9 + 3.8 + + - name: Extract Metadata (3.13) + id: extract_313 + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.13 + continue-on-error: true - name: Extract Metadata (3.12) + if: always() && (steps.extract_313.outcome == 'failure') id: extract_312 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.12 continue-on-error: true - - name: Install Python 3.11 (Only if 3.12 failed) - if: always() && (steps.extract_312.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Extract Metadata (3.11) if: always() && (steps.extract_312.outcome == 'failure') id: extract_311 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.11 continue-on-error: true - - name: Install Python 3.10 (Only if 3.11 failed) - if: always() && (steps.extract_311.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - name: Extract Metadata (3.10) if: always() && (steps.extract_311.outcome == 'failure') id: extract_310 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.10 continue-on-error: true - - name: Install Python 3.9 (Only if 3.10 failed) - if: always() && (steps.extract_310.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.9' - - name: Extract Metadata (3.9) if: always() && (steps.extract_310.outcome == 'failure') id: extract_39 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.9 continue-on-error: true - - name: Install Python 3.8 (Only if 3.9 failed) - if: always() && (steps.extract_39.outcome == 'failure') - uses: actions/setup-python@v5 - with: - python-version: '3.8' - - name: Extract Metadata (3.8) if: always() && (steps.extract_39.outcome == 'failure') id: extract_38 - run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data + run: hub-utils extract-sdk-metadata-to-s3 "/home/runner/work/hub/hub/_data/meltano/${{ matrix.plugin-name }}" $(pwd)/extract_data --python=python3.8 diff --git a/_data/meltano/extractors/tap-aircall/ticketswap.yml b/_data/meltano/extractors/tap-aircall/ticketswap.yml index 6c7ed6eb7..c5cbf765d 100644 --- a/_data/meltano/extractors/tap-aircall/ticketswap.yml +++ b/_data/meltano/extractors/tap-aircall/ticketswap.yml @@ -57,6 +57,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: array + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-apple-search-ads/ticketswap.yml b/_data/meltano/extractors/tap-apple-search-ads/ticketswap.yml index 30743451a..92a432f74 100644 --- a/_data/meltano/extractors/tap-apple-search-ads/ticketswap.yml +++ b/_data/meltano/extractors/tap-apple-search-ads/ticketswap.yml @@ -81,6 +81,20 @@ settings: label: Org ID name: org_id sensitive: true +- description: The granularity of reporting streams. One of HOURLY, DAILY, WEEKLY, + MONTHLY. + kind: options + label: Report Granularity + name: report_granularity + options: + - label: Hourly + value: HOURLY + - label: Daily + value: DAILY + - label: Weekly + value: WEEKLY + - label: Monthly + value: MONTHLY - description: Start date for reporting streams, format in YYYY-MM-DD. kind: date_iso8601 label: Start Date diff --git a/_data/meltano/extractors/tap-circle-ci/meltanolabs.yml b/_data/meltano/extractors/tap-circle-ci/meltanolabs.yml index ecf08e1e0..c52a5b59e 100644 --- a/_data/meltano/extractors/tap-circle-ci/meltanolabs.yml +++ b/_data/meltano/extractors/tap-circle-ci/meltanolabs.yml @@ -92,7 +92,7 @@ settings: kind: string label: User Agent name: user_agent - value: tap-circle-ci/0.3.2 Singer tap for the CircleCI API. + value: tap-circle-ci/0.3.3 Singer tap for the CircleCI API. settings_group_validation: - - org_slug - token diff --git a/_data/meltano/extractors/tap-dbt/meltanolabs.yml b/_data/meltano/extractors/tap-dbt/meltanolabs.yml index 025269b24..420ae1d26 100644 --- a/_data/meltano/extractors/tap-dbt/meltanolabs.yml +++ b/_data/meltano/extractors/tap-dbt/meltanolabs.yml @@ -95,7 +95,7 @@ settings: kind: string label: User Agent name: user_agent - value: tap-dbt/0.12.3.post1.dev8+a4e0b7b Singer tap for the dbt Cloud API. + value: tap-dbt/0.12.3.post1 Singer tap for the dbt Cloud API. settings_group_validation: - - account_ids - api_key diff --git a/_data/meltano/extractors/tap-mysql/meltanolabs.yml b/_data/meltano/extractors/tap-mysql/meltanolabs.yml index 06ff84b1b..9c5e59229 100644 --- a/_data/meltano/extractors/tap-mysql/meltanolabs.yml +++ b/_data/meltano/extractors/tap-mysql/meltanolabs.yml @@ -80,6 +80,12 @@ settings: kind: string label: Host name: host +- description: By default we'll check if the database is a Vitess instance. If you + would rather not automatically check, set this to `False`. See Vitess/PlanetScale + documentation below for more information. + kind: boolean + label: Is Vitess + name: is_vitess - description: Password used to authenticate. Note if sqlalchemy_url is set this will be ignored. kind: password @@ -92,7 +98,16 @@ settings: label: Port name: port value: 3306 -- description: Example mysql://[username]:[password]@localhost:3306/[db_name] +- description: Sqlalchemy_url options (also called the query), to connect to PlanetScale + you must turn on SSL see PlanetScale information below. Note if sqlalchemy_url + is set this will be ignored. + kind: object + label: Sqlalchemy Options + name: sqlalchemy_options +- description: Example pymysql://[username]:[password]@localhost:3306/[db_name][?options] + see + https://docs.sqlalchemy.org/en/20/dialects/mysql.html#module-sqlalchemy.dialects.mysql.pymysql + for more information kind: string label: Sqlalchemy URL name: sqlalchemy_url