diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ffd957971..919e73bd2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,2 @@ -* @rwfeather @cjohnhanson -/_data/ @sbalnojan @afolson @tayloramurphy @pnadolny13 +* @cjohnhanson +/_data/ @tayloramurphy @pnadolny13 @edgarrmondragon diff --git a/.github/ISSUE_TEMPLATE/new_plugin.yml b/.github/ISSUE_TEMPLATE/new_plugin.yml index 3d711620b..b3328da1c 100644 --- a/.github/ISSUE_TEMPLATE/new_plugin.yml +++ b/.github/ISSUE_TEMPLATE/new_plugin.yml @@ -2,7 +2,7 @@ name: 🚰 New Plugin description: Request a new Plugin to Meltano Hub title: "Add Plugin: " labels: [valuestream/Hub] -assignees: [tayloramurphy, pnadolny13] +assignees: [edgarrmondragon, pnadolny13] body: - type: markdown diff --git a/.github/workflows/build-api.yml b/.github/workflows/build-api.yml index a9e7be8cb..90db0621d 100644 --- a/.github/workflows/build-api.yml +++ b/.github/workflows/build-api.yml @@ -23,7 +23,7 @@ jobs: environment: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && 'production' || 'preview' }} if: ${{ !github.event.pull_request.head.repo.fork }} steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.6 - name: Set up Python 3.10 uses: actions/setup-python@v5 @@ -82,7 +82,7 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.6 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4.0.2 with: diff --git a/.github/workflows/check-yaml-format.yml b/.github/workflows/check-yaml-format.yml index 0b3fcb9bf..b88027e61 100644 --- a/.github/workflows/check-yaml-format.yml +++ b/.github/workflows/check-yaml-format.yml @@ -11,11 +11,11 @@ jobs: name: Lint YAML files runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.6 - name: Get changed files in the docs folder id: changed-files - uses: tj-actions/changed-files@v44.3.0 + uses: tj-actions/changed-files@v44.4.0 with: files: | _data/**/*.yml diff --git a/.github/workflows/lint_and_format.yml b/.github/workflows/lint_and_format.yml index 05bd04e03..ec0ca3693 100644 --- a/.github/workflows/lint_and_format.yml +++ b/.github/workflows/lint_and_format.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.6 - name: Use Node.js v16 uses: actions/setup-node@v4 with: @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.6 - name: Use Node.js v16 uses: actions/setup-node@v4 with: diff --git a/.github/workflows/metadata-extract.yml b/.github/workflows/metadata-extract.yml index dc3055cdc..e5272f250 100644 --- a/.github/workflows/metadata-extract.yml +++ b/.github/workflows/metadata-extract.yml @@ -23,7 +23,7 @@ jobs: airbyte_matrix: ${{ steps.setmatrix_airbyte.outputs.airbyte_matrix }} steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.6 - name: Install hub-utils run: pipx install git+https://github.com/meltano/hub-utils.git @@ -71,7 +71,7 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.6 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4.0.2 @@ -129,7 +129,7 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.6 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4.0.2 @@ -208,7 +208,7 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.6 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4.0.2 diff --git a/.github/workflows/metadata-refresh.yml b/.github/workflows/metadata-refresh.yml index 306e1eaa1..39a2d3857 100644 --- a/.github/workflows/metadata-refresh.yml +++ b/.github/workflows/metadata-refresh.yml @@ -25,7 +25,7 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.6 - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4.0.2 diff --git a/.github/workflows/plugin_definition_validation.yml b/.github/workflows/plugin_definition_validation.yml index 0600debd2..e9d4c7274 100644 --- a/.github/workflows/plugin_definition_validation.yml +++ b/.github/workflows/plugin_definition_validation.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Check out the repository - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.6 - name: Setup Python uses: actions/setup-python@v5 diff --git a/.github/workflows/test_dispatcher.yml b/.github/workflows/test_dispatcher.yml index 7414d24c6..51e53c9ba 100644 --- a/.github/workflows/test_dispatcher.yml +++ b/.github/workflows/test_dispatcher.yml @@ -17,7 +17,7 @@ jobs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - name: Checkout the repository - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.6 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} @@ -25,7 +25,7 @@ jobs: - name: Get changed files id: changed_files - uses: tj-actions/changed-files@v44.3.0 + uses: tj-actions/changed-files@v44.4.0 with: sha: ${{ github.event.pull_request.head.sha }} json: "true" @@ -47,7 +47,7 @@ jobs: fail-fast: false steps: - name: Checkout the repository - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.6 with: ref: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/test_meltano_add_install.yml b/.github/workflows/test_meltano_add_install.yml index bd2c19661..f3016d3ed 100644 --- a/.github/workflows/test_meltano_add_install.yml +++ b/.github/workflows/test_meltano_add_install.yml @@ -15,13 +15,13 @@ jobs: matrix: ${{ steps.changed_plugins.outputs.all_changed_files }} steps: - name: Checkout the repository - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.6 with: fetch-depth: 0 - name: Get changed plugins id: changed_plugins - uses: tj-actions/changed-files@v44.3.0 + uses: tj-actions/changed-files@v44.4.0 with: files: _data/meltano/*/*/*.yml matrix: true @@ -35,7 +35,7 @@ jobs: fail-fast: false steps: - name: Checkout the repository - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.6 - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/validate-urls.yml b/.github/workflows/validate-urls.yml index 4fcb7467a..80f015952 100644 --- a/.github/workflows/validate-urls.yml +++ b/.github/workflows/validate-urls.yml @@ -15,7 +15,7 @@ jobs: contents: read issues: write steps: - - uses: actions/checkout@v4.1.4 + - uses: actions/checkout@v4.1.6 - name: Setup Python uses: actions/setup-python@v5 diff --git a/_data/default_variants.yml b/_data/default_variants.yml index b594c0733..21115c42a 100644 --- a/_data/default_variants.yml +++ b/_data/default_variants.yml @@ -17,7 +17,7 @@ extractors: tap-aha: airbyte tap-airbyte-wrapper: meltanolabs tap-aircall: ticketswap - tap-airtable: goes-funky + tap-airtable: tomasvotava tap-alpha-vantage: airbyte tap-amazon-ads-dsp: singer-io tap-amazon-advertising: dbt-labs @@ -90,7 +90,6 @@ extractors: tap-cin7: hotgluexyz tap-circle-ci: sisudata tap-clarabridge: pathlight - tap-clickcast: datateer tap-clickhouse: akurdyukov tap-clickup: autoidm tap-clientsuccess: antiny @@ -212,7 +211,7 @@ extractors: tap-gmail-csv: food-spotter tap-gnews: airbyte tap-gocardless: airbyte - tap-gohighlevel: hotgluexyz + tap-gohighlevel: meltanolabs tap-gong: symon-ai tap-google-analytics: meltanolabs tap-google-directory: airbyte @@ -265,7 +264,6 @@ extractors: tap-kiotviet: chienazazaz tap-klarna: airbyte tap-klaviyo: meltanolabs - tap-krow: datateer tap-kustomer: meltanolabs tap-kwanko: uptilab2 tap-kyriba: airbyte diff --git a/_data/maintainers.yml b/_data/maintainers.yml index e901a3eae..aa1f51800 100644 --- a/_data/maintainers.yml +++ b/_data/maintainers.yml @@ -1173,6 +1173,10 @@ tmonks: label: Tom Monks name: tmonks url: https://github.com/tmonks +tomasvotava: + label: Tomas Votava + name: tomasvotava + url: https://github.com/tomasvotava tomekzbrozek: label: Tomek Zbrozek name: tomekzbrozek diff --git a/_data/meltano/extractors/tap-airtable/tomasvotava.yml b/_data/meltano/extractors/tap-airtable/tomasvotava.yml new file mode 100644 index 000000000..990f04a54 --- /dev/null +++ b/_data/meltano/extractors/tap-airtable/tomasvotava.yml @@ -0,0 +1,93 @@ +capabilities: +- about +- batch +- catalog +- discover +- schema-flattening +- state +- stream-maps +description: Cloud collaboration service +domain_url: https://airtable.com/api +executable: tap-airtable +keywords: +- meltano_sdk +label: Airtable +logo_url: /assets/logos/extractors/airtable.png +maintenance_status: active +name: tap-airtable +namespace: tap_airtable +next_steps: '' +pip_url: tap-airtable +quality: silver +repo: https://github.com/tomasvotava/tap-airtable +settings: +- description: Selected base ids (all if not specified) + kind: array + label: Base IDs + name: base_ids +- description: Compression format to use for batch files. + kind: options + label: Batch Config Encoding Compression + name: batch_config.encoding.compression + options: + - label: Gzip + value: gzip + - label: None + value: none +- description: Format to use for batch files. + kind: options + label: Batch Config Encoding Format + name: batch_config.encoding.format + options: + - label: Jsonl + value: jsonl + - label: Parquet + value: parquet +- description: Prefix to use when writing batch files. + kind: string + label: Batch Config Storage Prefix + name: batch_config.storage.prefix +- description: Root path to use when writing batch files. + kind: string + label: Batch Config Storage Root + name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: array + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed +- description: "'True' to enable schema flattening and automatically expand nested + properties." + kind: boolean + label: Flattening Enabled + name: flattening_enabled +- description: The max depth to flatten schemas. + kind: integer + label: Flattening Max Depth + name: flattening_max_depth +- description: User-defined config values to be used within map expressions. + kind: object + label: Stream Map Config + name: stream_map_config +- description: Config object for stream maps capability. For more information check + out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). + kind: object + label: Stream Maps + name: stream_maps +- description: Mapping of table_id => custom_table_name + kind: object + label: Table Mapping + name: table_mapping +- description: The token to authenticate against Airtable API + kind: password + label: Token + name: token + sensitive: true +settings_group_validation: +- - token +settings_preamble: '' +usage: '' +variant: tomasvotava diff --git a/_data/meltano/extractors/tap-circle-ci/meltanolabs.yml b/_data/meltano/extractors/tap-circle-ci/meltanolabs.yml index 99ddc80ad..f9d72bdaa 100644 --- a/_data/meltano/extractors/tap-circle-ci/meltanolabs.yml +++ b/_data/meltano/extractors/tap-circle-ci/meltanolabs.yml @@ -92,7 +92,7 @@ settings: kind: string label: User Agent name: user_agent - value: tap-circle-ci/0.2.1 Singer tap for the CircleCI API. + value: tap-circle-ci/0.2.2 Singer tap for the CircleCI API. settings_group_validation: - - org_slug - token diff --git a/_data/meltano/extractors/tap-clickcast/datateer.yml b/_data/meltano/extractors/tap-clickcast/datateer.yml deleted file mode 100644 index 2458387e2..000000000 --- a/_data/meltano/extractors/tap-clickcast/datateer.yml +++ /dev/null @@ -1,52 +0,0 @@ -capabilities: -- about -- catalog -- discover -- schema-flattening -- state -- stream-maps -description: Clickcast Job ad platform by Appcast -domain_url: https://www.appcast.io -keywords: -- api -- meltano_sdk -label: Clickcast -logo_url: /assets/logos/extractors/clickcast.png -maintenance_status: active -name: tap-clickcast -namespace: tap_clickcast -pip_url: git+https://github.com/datateer/tap-clickcast.git -quality: silver -repo: https://github.com/datateer/tap-clickcast -settings: -- description: The URL base for the Clickcast API. Defaults to https://api.clickcast.cloud/clickcast/api - kind: string - label: API URL Base - name: api_url_base - value: https://api.clickcast.cloud/clickcast/api -- description: "'True' to enable schema flattening and automatically expand nested - properties." - kind: boolean - label: Flattening Enabled - name: flattening_enabled -- description: The max depth to flatten schemas. - kind: integer - label: Flattening Max Depth - name: flattening_max_depth -- description: A partner token generated by the Clickcast portal - kind: password - label: Partner Token - name: partner_token - sensitive: true -- description: User-defined config values to be used within map expressions. - kind: object - label: Stream Map Config - name: stream_map_config -- description: Config object for stream maps capability. For more information check - out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). - kind: object - label: Stream Maps - name: stream_maps -settings_group_validation: -- - partner_token -variant: datateer diff --git a/_data/meltano/extractors/tap-gohighlevel/meltanolabs.yml b/_data/meltano/extractors/tap-gohighlevel/meltanolabs.yml new file mode 100644 index 000000000..f68ccd6ae --- /dev/null +++ b/_data/meltano/extractors/tap-gohighlevel/meltanolabs.yml @@ -0,0 +1,106 @@ +capabilities: +- about +- batch +- catalog +- discover +- schema-flattening +- state +- stream-maps +description: All-In-One Sales & Marketing Platform +domain_url: https://www.gohighlevel.com/ +executable: tap-gohighlevel +keywords: +- meltano_sdk +label: HighLevel +logo_url: /assets/logos/extractors/gohighlevel.png +maintenance_status: active +name: tap-gohighlevel +namespace: tap_gohighlevel +next_steps: '' +pip_url: git+https://github.com/MeltanoLabs/tap-gohighlevel.git +quality: gold +repo: https://github.com/MeltanoLabs/tap-gohighlevel +settings: +- description: Compression format to use for batch files. + kind: options + label: Batch Config Encoding Compression + name: batch_config.encoding.compression + options: + - label: Gzip + value: gzip + - label: None + value: none +- description: Format to use for batch files. + kind: options + label: Batch Config Encoding Format + name: batch_config.encoding.format + options: + - label: Jsonl + value: jsonl + - label: Parquet + value: parquet +- description: Prefix to use when writing batch files. + kind: string + label: Batch Config Storage Prefix + name: batch_config.storage.prefix +- description: Root path to use when writing batch files. + kind: string + label: Batch Config Storage Root + name: batch_config.storage.root +- description: Client ID for the API service + kind: password + label: Client ID + name: client_id + sensitive: true +- description: Client Secret for the API service + kind: password + label: Client Secret + name: client_secret + sensitive: true +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: array + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed +- description: "'True' to enable schema flattening and automatically expand nested + properties." + kind: boolean + label: Flattening Enabled + name: flattening_enabled +- description: The max depth to flatten schemas. + kind: integer + label: Flattening Max Depth + name: flattening_max_depth +- description: The Location Id to request data + kind: password + label: Location ID + name: location_id + sensitive: true +- description: Refresh token for the API service + kind: password + label: Refresh Token + name: refresh_token + sensitive: true +- description: User-defined config values to be used within map expressions. + kind: object + label: Stream Map Config + name: stream_map_config +- description: Config object for stream maps capability. For more information check + out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). + kind: object + label: Stream Maps + name: stream_maps +settings_group_validation: +- - client_id + - client_secret + - location_id + - refresh_token +settings_preamble: The HighLevel API uses a refresh token that expires after every + use which means this extractor has to write the updated token back to the config + file. There's currently no feature in Meltano (see https://github.com/meltano/meltano/issues/2660) + to persist these types of changes between runs, so you'll have to implement it yourself. +usage: '' +variant: meltanolabs diff --git a/_data/meltano/extractors/tap-krow/datateer.yml b/_data/meltano/extractors/tap-krow/datateer.yml deleted file mode 100644 index 865ad264a..000000000 --- a/_data/meltano/extractors/tap-krow/datateer.yml +++ /dev/null @@ -1,53 +0,0 @@ -capabilities: -- about -- catalog -- discover -- schema-flattening -- state -- stream-maps -domain_url: https://krow.ai/ -keywords: -- meltano_sdk -label: Krow -logo_url: /assets/logos/extractors/krow.png -maintenance_status: active -name: tap-krow -namespace: tap_krow -next_steps: '' -pip_url: git+https://github.com/Datateer/tap-krow.git -quality: silver -repo: https://github.com/Datateer/tap-krow -settings: -- description: Your KROW API key. - kind: password - label: API Key - name: api_key - sensitive: true -- description: Your KROW url. - kind: string - label: API URL Base - name: api_url_base - value: https://industry-staging.herokuapp.com/v1 -- description: "'True' to enable schema flattening and automatically expand nested - properties." - kind: boolean - label: Flattening Enabled - name: flattening_enabled -- description: The max depth to flatten schemas. - kind: integer - label: Flattening Max Depth - name: flattening_max_depth -- description: User-defined config values to be used within map expressions. - kind: object - label: Stream Map Config - name: stream_map_config -- description: Config object for stream maps capability. For more information check - out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). - kind: object - label: Stream Maps - name: stream_maps -settings_group_validation: -- - api_key -settings_preamble: '' -usage: '' -variant: datateer diff --git a/_data/meltano/extractors/tap-totango/edsoncezar16.yml b/_data/meltano/extractors/tap-totango/edsoncezar16.yml index e3a113e4a..3b611e1b4 100644 --- a/_data/meltano/extractors/tap-totango/edsoncezar16.yml +++ b/_data/meltano/extractors/tap-totango/edsoncezar16.yml @@ -76,9 +76,9 @@ settings: label: API URL name: api_url options: - - label: https://api.totango.com + - label: US Endpoint value: https://api.totango.com - - label: 'https://api-eu1.totango.com' + - label: EU Endpoint value: 'https://api-eu1.totango.com ' value: https://api.totango.com - description: The token to authenticate against the API service diff --git a/_data/meltano/loaders/target-postgres/meltanolabs.yml b/_data/meltano/loaders/target-postgres/meltanolabs.yml index e8053c198..8356415b0 100644 --- a/_data/meltano/loaders/target-postgres/meltanolabs.yml +++ b/_data/meltano/loaders/target-postgres/meltanolabs.yml @@ -33,6 +33,10 @@ settings: label: Add Record Metadata name: add_record_metadata value: true +- description: Maximum number of rows in each batch. + kind: integer + label: Batch Size Rows + name: batch_size_rows - description: Database name. Note if sqlalchemy_url is set this will be ignored. kind: string label: Database diff --git a/_data/meltano/loaders/target-redshift/ticketswap.yml b/_data/meltano/loaders/target-redshift/ticketswap.yml index 76c6e6e1b..971f3f9c1 100644 --- a/_data/meltano/loaders/target-redshift/ticketswap.yml +++ b/_data/meltano/loaders/target-redshift/ticketswap.yml @@ -1,9 +1,7 @@ capabilities: - about -- stream-maps - schema-flattening -- target-schema -- hard-delete +- stream-maps description: Amazon Redshift loader domain_url: https://aws.amazon.com/redshift/ executable: target-redshift @@ -25,14 +23,24 @@ settings: kind: boolean label: Activate Version name: activate_version - value: true + value: false - description: Note that this must be enabled for activate_version to work!This adds _sdc_extracted_at, _sdc_batched_at, and more to every table. See https://sdk.meltano.com/en/latest/implementation/record_metadata.html for more information. kind: boolean label: Add Record Metadata name: add_record_metadata - value: true + value: false +- description: Redshift copy role arn to use for the COPY command from s3 + kind: string + label: AWS Redshift Copy Role Arn + name: aws_redshift_copy_role_arn +- description: Redshift cluster identifier. Note if sqlalchemy_url is set or enable_iam_authentication + is false this will be ignored. + kind: password + label: Cluster IDentifier + name: cluster_identifier + sensitive: true - description: Database name. Note if sqlalchemy_url is set this will be ignored. kind: string label: Database name @@ -42,12 +50,19 @@ settings: label: Default Target Schema name: default_target_schema value: $MELTANO_EXTRACT__LOAD_SCHEMA -- description: Dialect+driver see https://aws.amazon.com/blogs/big-data/use-the-amazon-redshift-sqlalchemy-dialect-to-interact-with-amazon-redshift. +- description: Dialect+driver see + https://aws.amazon.com/blogs/big-data/use-the-amazon-redshift-sqlalchemy-dialect-to-interact-with-amazon-redshift. Generally just leave this alone. Note if sqlalchemy_url is set this will be ignored. kind: string label: Dialect+Driver name: dialect+driver value: redshift+redshift_connector +- description: If true, use temporary credentials + (https://docs.aws.amazon.com/redshift/latest/mgmt/generating-iam-credentials-cli-api.html). + Note if sqlalchemy_url is set this will be ignored. + kind: boolean + label: Enable Iam Authentication + name: enable_iam_authentication - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean @@ -69,14 +84,21 @@ settings: kind: string label: Host name: host -- description: If set to true, the target will interpret the content encoding of the - schema to determine how to store the data. Using this option may result in a more - efficient storage of the data but may also result in an error if the data is not - encoded as expected. - kind: boolean - label: Interpret Content Encoding - name: interpret_content_encoding - value: false +- description: The method to use when loading data into the destination. `append-only` + will always write all input records whether that records already exists or not. + `upsert` will update existing records and insert new records. `overwrite` will + delete all existing records and insert all input records. + kind: options + label: Load Method + name: load_method + options: + - label: Append Only + value: append-only + - label: Upsert + value: upsert + - label: Overwrite + value: overwrite + value: append-only - description: Password used to authenticate. Note if sqlalchemy_url is set this will be ignored. kind: password @@ -88,7 +110,22 @@ settings: kind: string label: Port name: port - value: "5432" + value: '5432' +- description: If you want to remove staging files in S3 + kind: boolean + label: Remove S3 Files + name: remove_s3_files + value: false +- description: S3 bucket to save staging files before using COPY command + kind: string + label: S3 Bucket + name: s3_bucket +- description: S3 key prefix to save staging files before using COPY command + kind: password + label: S3 Key Prefix + name: s3_key_prefix + sensitive: true + value: '' - description: SQLAlchemy connection string. This will override using host, user, password, port, dialect, and all ssl settings. Note that you must escape password special characters properly. See @@ -96,10 +133,6 @@ settings: kind: string label: Sqlalchemy URL name: sqlalchemy_url -- description: Username to connect to bastion host - kind: string - label: SSH Tunnel Username - name: ssh_tunnel.username - description: Whether or not to use ssl to verify the server's identity. Use ssl_certificate_authority and ssl_mode for further customization. To use a client certificate to authenticate yourself to the server, use ssl_client_certificate_enable instead. Note if sqlalchemy_url @@ -108,7 +141,9 @@ settings: label: SSL Enable name: ssl_enable value: false -- description: SSL Protection method. Must be one of disable, allow, prefer, require, verify-ca, +- description: SSL Protection method, see [postgres + documentation](https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION) + for more information. Must be one of disable, allow, prefer, require, verify-ca, or verify-full. Note if sqlalchemy_url is set this will be ignored. kind: string label: SSL Mode @@ -123,47 +158,22 @@ settings: kind: object label: Stream Maps name: stream_maps +- description: Where you want to store your temp data files. + kind: string + label: Temp Dir + name: temp_dir + value: temp - description: User name used to authenticate. Note if sqlalchemy_url is set this will be ignored. kind: string label: User name: user -- description: If true, use temporary credentials - (https://docs.aws.amazon.com/redshift/latest/mgmt/generating-iam-credentials-cli-api.html). - Note if sqlalchemy_url is set this will be ignored. - kind: boolean - label: Enable iam authentication - name: enable_iam_authentication - value: false -- description: Redshift cluster identifier. Note if sqlalchemy_url is set or enable_iam_authentication is false this will be ignored. - kind: string - label: Cluster identifier - name: cluster_identifier -- description: Redshift copy role arn to use for the COPY command from s3. - kind: string - label: AWS redshift copy role arn - name: aws_redshift_copy_role_arn -- description: S3 bucket to save staging files before using COPY command. - kind: string - label: S3 bucket - name: s3_bucket -- description: S3 key prefix to save staging files before using COPY command. - kind: string - label: S3 key prefix - name: s3_key_prefix -- description: If you want to remove staging files in S3. - kind: boolean - label: Remove S3 files - name: remove_s3_files - value: true -- description: Where you want to store your staging files locally. - kind: string - label: Temp dir - name: temp_dir - value: .meltano/temp settings_group_validation: -- [] -settings_preamble: 'This target uses boto3 to connect to AWS and perform copy and possibly credential operations. The agent running this target should be connected to AWS to be able to open a boto3 client.' +- - aws_redshift_copy_role_arn + - s3_bucket +settings_preamble: This target uses boto3 to connect to AWS and perform copy and possibly + credential operations. The agent running this target should be connected to AWS + to be able to open a boto3 client. target_schema: $TARGET_REDSHIFT_SCHEMA usage: '' variant: ticketswap diff --git a/_data/meltano/utilities/matatika/matatika.yml b/_data/meltano/utilities/matatika/matatika.yml index fdeffc3fc..4f132779b 100644 --- a/_data/meltano/utilities/matatika/matatika.yml +++ b/_data/meltano/utilities/matatika/matatika.yml @@ -50,7 +50,7 @@ next_steps: | If you're running into problems with this extension, or just want to chat about all things data, [join the Matatika Slack community](https://join.slack.com/t/matatika/shared_invite/zt-19n1bfokx-F31DNitTpSxWCFO2aFlgxg).

-pip_url: git+https://github.com/Matatika/matatika-ext.git@v0.31.0 +pip_url: git+https://github.com/Matatika/matatika-ext.git@v0.32.0 repo: https://github.com/Matatika/matatika-ce usage: For help, try `meltano invoke matatika --help` (or `--help` on any subcommand). variant: matatika diff --git a/_data/variant_metrics.yml b/_data/variant_metrics.yml index 6ec400daa..825c2da9d 100644 --- a/_data/variant_metrics.yml +++ b/_data/variant_metrics.yml @@ -227,25 +227,6 @@ metrics: success_projects_unstruct_by_name: 0 success_projects_unstruct_by_variant: 0 variant: clickhouse - https://github.com/Datateer/tap-krow: - all_execs: 0 - all_execs_by_name: 23 - all_execs_unstruct_by_name: 23 - all_execs_unstruct_by_variant: 0 - all_projects: 0 - all_projects_by_name: 1 - all_projects_unstruct_by_name: 1 - all_projects_unstruct_by_variant: 0 - name: tap-krow - pip_url: git+https://github.com/Datateer/tap-krow.git - plugin_type: extractors - success_execs_by_name: 0 - success_execs_unstruct_by_name: 0 - success_execs_unstruct_by_variant: 0 - success_projects_by_name: 0 - success_projects_unstruct_by_name: 0 - success_projects_unstruct_by_variant: 0 - variant: datateer https://github.com/Datateer/target-gcs: all_execs: 1096 all_execs_by_name: 1261 @@ -9540,25 +9521,6 @@ metrics: success_projects_unstruct_by_name: 0 success_projects_unstruct_by_variant: 0 variant: dataops-tk - https://github.com/datateer/tap-clickcast: - all_execs: 0 - all_execs_by_name: 0 - all_execs_unstruct_by_name: 0 - all_execs_unstruct_by_variant: 0 - all_projects: 0 - all_projects_by_name: 0 - all_projects_unstruct_by_name: 0 - all_projects_unstruct_by_variant: 0 - name: tap-clickcast - pip_url: git+https://github.com/datateer/tap-clickcast.git - plugin_type: extractors - success_execs_by_name: 0 - success_execs_unstruct_by_name: 0 - success_execs_unstruct_by_variant: 0 - success_projects_by_name: 0 - success_projects_unstruct_by_name: 0 - success_projects_unstruct_by_variant: 0 - variant: datateer https://github.com/dbt-labs/dbt-bigquery: all_execs: 1177 all_execs_by_name: 0 diff --git a/poetry.lock b/poetry.lock index acee7cbba..6c0b23054 100644 --- a/poetry.lock +++ b/poetry.lock @@ -356,13 +356,13 @@ files = [ [[package]] name = "requests" -version = "2.31.0" +version = "2.32.2" description = "Python HTTP for Humans." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, - {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, + {file = "requests-2.32.2-py3-none-any.whl", hash = "sha256:fc06670dd0ed212426dfeb94fc1b983d917c4f9847c863f313c9dfaaffb7c23c"}, + {file = "requests-2.32.2.tar.gz", hash = "sha256:dd951ff5ecf3e3b3aa26b40703ba77495dab41da839ae72ef3c8e5d8e2433289"}, ] [package.dependencies] @@ -499,4 +499,4 @@ dev = ["doc8", "flake8", "flake8-import-order", "rstcheck[sphinx]", "sphinx"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "8b82515678c581ccdab96590b429515625e3d3bcd314f0d3b374dd30256050df" +content-hash = "1af752ae7e7dcccefd1426f7da170272bea96ba758eaf8ec894a345931c4e97b" diff --git a/pyproject.toml b/pyproject.toml index 1ceeae669..6e07a4584 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ python = "^3.9" jsonschema = "^4.17.3" "ruamel.yaml" = "^0.18.6" yamllint = "^1.35.1" -requests = "^2.31.0" +requests = "^2.32.2" [tool.poetry.dev-dependencies] black = "^24.4.2" diff --git a/singer/taps/tap-clickcast.json b/singer/taps/tap-clickcast.json deleted file mode 100644 index d60bd3c34..000000000 --- a/singer/taps/tap-clickcast.json +++ /dev/null @@ -1 +0,0 @@ -{"name":"tap-clickcast","description":"Clickcast Job ad platform by Appcast","label":"Clickcast","entity_type":"api","entity_url":"https://www.appcast.io","singer_name":"tap-clickcast","type":"tap","variants":[{"default":true,"maintenance_status":"active","meltano_sdk":true,"name":"datateer","pip_url":"git+https://github.com/datateer/tap-clickcast.git","repo":"https://github.com/datateer/tap-clickcast","capabilities":["about","catalog","discover","state","stream-maps"],"settings_group_validation":[["partner_token"]],"settings":[{"name":"partner_token","kind":"password","label":"Partner Token","description":"A partner token generated by the Clickcast portal"},{"name":"api_url_base","label":"Clickcast URL base","description":"The URL base for the Clickcast API. Defaults to https://api.clickcast.cloud/clickcast/api"}],"maintainer":{"label":"Datateer","url":"https://github.com/datateer","plugins":[{"url":"/taps/clickcast--datateer","logo_url":"/assets/logos/taps/clickcast.png","label":"Clickcast"}],"page_url":"/singer/maintainers/datateer"},"metrics":null,"url":"/taps/clickcast--datateer"}],"logo_url":"/assets/logos/taps/clickcast.png","url":"/taps/clickcast"} \ No newline at end of file diff --git a/static/assets/logos/extractors/clickcast.png b/static/assets/logos/extractors/clickcast.png deleted file mode 100644 index 30ccf6641..000000000 Binary files a/static/assets/logos/extractors/clickcast.png and /dev/null differ diff --git a/static/assets/logos/extractors/krow.png b/static/assets/logos/extractors/krow.png deleted file mode 100644 index d93f71c82..000000000 Binary files a/static/assets/logos/extractors/krow.png and /dev/null differ