diff --git a/.github/workflows/build-api.yml b/.github/workflows/build-api.yml index 99032fa0e..941131ddb 100644 --- a/.github/workflows/build-api.yml +++ b/.github/workflows/build-api.yml @@ -8,13 +8,14 @@ on: branches: - main workflow_dispatch: - hub-url: - description: The base URL of the Meltano Hub site - required: true - default: 'https://hub.meltano.com' - api-url: - description: The base URL of the Hub API - required: true + inputs: + hub-url: + description: The base URL of the Meltano Hub site + required: true + default: 'https://hub.meltano.com' + api-url: + description: The base URL of the Hub API + required: true jobs: build: @@ -83,7 +84,7 @@ jobs: steps: - uses: actions/checkout@v4.1.1 - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4.0.1 + uses: aws-actions/configure-aws-credentials@v4.0.2 with: role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} aws-region: us-west-2 diff --git a/.github/workflows/check-yaml-format.yml b/.github/workflows/check-yaml-format.yml index ea7dd6c83..7ab308399 100644 --- a/.github/workflows/check-yaml-format.yml +++ b/.github/workflows/check-yaml-format.yml @@ -15,7 +15,7 @@ jobs: - name: Get changed files in the docs folder id: changed-files - uses: tj-actions/changed-files@v42.0.2 + uses: tj-actions/changed-files@v42.0.4 with: files: | _data/**/*.yml diff --git a/.github/workflows/metadata-extract.yml b/.github/workflows/metadata-extract.yml index 24526e123..1bbc33959 100644 --- a/.github/workflows/metadata-extract.yml +++ b/.github/workflows/metadata-extract.yml @@ -74,7 +74,7 @@ jobs: - uses: actions/checkout@v4.1.1 - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4.0.1 + uses: aws-actions/configure-aws-credentials@v4.0.2 with: role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} aws-region: us-west-2 @@ -132,7 +132,7 @@ jobs: - uses: actions/checkout@v4.1.1 - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4.0.1 + uses: aws-actions/configure-aws-credentials@v4.0.2 with: role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} aws-region: us-west-2 @@ -211,7 +211,7 @@ jobs: - uses: actions/checkout@v4.1.1 - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4.0.1 + uses: aws-actions/configure-aws-credentials@v4.0.2 with: role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} aws-region: us-west-2 diff --git a/.github/workflows/metadata-refresh.yml b/.github/workflows/metadata-refresh.yml index 21866ee11..853ee20a3 100644 --- a/.github/workflows/metadata-refresh.yml +++ b/.github/workflows/metadata-refresh.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/checkout@v4.1.1 - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4.0.1 + uses: aws-actions/configure-aws-credentials@v4.0.2 with: role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} aws-region: us-west-2 diff --git a/.github/workflows/test_dispatcher.yml b/.github/workflows/test_dispatcher.yml index 05af7f86f..ca65a98bc 100644 --- a/.github/workflows/test_dispatcher.yml +++ b/.github/workflows/test_dispatcher.yml @@ -25,7 +25,7 @@ jobs: - name: Get changed files id: changed_files - uses: tj-actions/changed-files@v42.0.2 + uses: tj-actions/changed-files@v42.0.4 with: sha: ${{ github.event.pull_request.head.sha }} json: "true" diff --git a/_data/default_variants.yml b/_data/default_variants.yml index 156fe56bf..5e7019369 100644 --- a/_data/default_variants.yml +++ b/_data/default_variants.yml @@ -127,6 +127,7 @@ extractors: tap-dash-hudson: gthesheep tap-datadog: degreed-data-engineering tap-datascope: airbyte + tap-dataverse: mjsqu tap-dayforce: goodeggs tap-db2: mjsqu tap-dbf: edgarrmondragon @@ -243,6 +244,7 @@ extractors: tap-insided: karbonhq tap-insightly: fostersdata tap-instagram: voxmedia + tap-instantly-ai: strvcom tap-intacct: hotgluexyz tap-intercom: singer-io tap-intruder: airbyte @@ -307,7 +309,6 @@ extractors: tap-messagebird: meltanolabs tap-metabase: airbyte tap-metronome: hotgluexyz - tap-microsoft-dataverse: airbyte tap-mindstamp: manuelapilongo tap-miro: slalom-consulting tap-mixpanel: hotgluexyz @@ -626,7 +627,7 @@ loaders: target-mysql: thkwag target-oracle: radbrt target-pardot: anelendata - target-parquet: estrategiahq + target-parquet: automattic target-pinecone: meltanolabs target-postgres: meltanolabs target-redshift: transferwise diff --git a/_data/maintainers.yml b/_data/maintainers.yml index fefe57206..94e7fd55a 100644 --- a/_data/maintainers.yml +++ b/_data/maintainers.yml @@ -1097,6 +1097,10 @@ streetteam: label: Pollen name: streetteam url: https://pollen.co/ +strvcom: + label: strvcom + name: strvcom + url: https://github.com/strvcom stvhanna: label: Steve Hanna name: stvhanna diff --git a/_data/meltano/extractors/tap-athena/meltanolabs.yml b/_data/meltano/extractors/tap-athena/meltanolabs.yml index 4f037f39f..c48041a79 100644 --- a/_data/meltano/extractors/tap-athena/meltanolabs.yml +++ b/_data/meltano/extractors/tap-athena/meltanolabs.yml @@ -60,6 +60,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-bitso/edgarrmondragon.yml b/_data/meltano/extractors/tap-bitso/edgarrmondragon.yml index 41660144a..e95c0b76a 100644 --- a/_data/meltano/extractors/tap-bitso/edgarrmondragon.yml +++ b/_data/meltano/extractors/tap-bitso/edgarrmondragon.yml @@ -59,6 +59,14 @@ settings: name: books value: - btc_mxn +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-circle-ci/meltanolabs.yml b/_data/meltano/extractors/tap-circle-ci/meltanolabs.yml index 3a1af36d3..973da98e8 100644 --- a/_data/meltano/extractors/tap-circle-ci/meltanolabs.yml +++ b/_data/meltano/extractors/tap-circle-ci/meltanolabs.yml @@ -52,6 +52,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-csv/meltanolabs.yml b/_data/meltano/extractors/tap-csv/meltanolabs.yml index 988ea1382..cec219475 100644 --- a/_data/meltano/extractors/tap-csv/meltanolabs.yml +++ b/_data/meltano/extractors/tap-csv/meltanolabs.yml @@ -39,6 +39,14 @@ settings: label: Csv Files Definition name: csv_files_definition placeholder: Ex. files-def.json +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: | Array of objects with `entity`, `path`, `keys`, and `encoding` [Optional] keys: diff --git a/_data/meltano/extractors/tap-microsoft-dataverse/airbyte.yml b/_data/meltano/extractors/tap-dataverse/airbyte.yml similarity index 96% rename from _data/meltano/extractors/tap-microsoft-dataverse/airbyte.yml rename to _data/meltano/extractors/tap-dataverse/airbyte.yml index 6196f0b20..b00630763 100644 --- a/_data/meltano/extractors/tap-microsoft-dataverse/airbyte.yml +++ b/_data/meltano/extractors/tap-dataverse/airbyte.yml @@ -11,9 +11,9 @@ executable: tap-airbyte keywords: - airbyte_protocol label: Microsoft Dataverse -logo_url: /assets/logos/extractors/microsoft-dataverse.png +logo_url: /assets/logos/extractors/dataverse.png maintenance_status: beta -name: tap-microsoft-dataverse +name: tap-dataverse namespace: tap_airbyte next_steps: '' pip_url: git+https://github.com/MeltanoLabs/tap-airbyte-wrapper.git diff --git a/_data/meltano/extractors/tap-dataverse/mjsqu.yml b/_data/meltano/extractors/tap-dataverse/mjsqu.yml new file mode 100644 index 000000000..2d4836aaf --- /dev/null +++ b/_data/meltano/extractors/tap-dataverse/mjsqu.yml @@ -0,0 +1,126 @@ +capabilities: +- about +- batch +- catalog +- discover +- schema-flattening +- state +- stream-maps +description: Microsoft Dataverse is a data storage and management engine +domain_url: + https://learn.microsoft.com/en-us/power-apps/developer/data-platform/webapi/query-data-web-api +executable: tap-dataverse +keywords: +- meltano_sdk +label: Microsoft Dataverse +logo_url: /assets/logos/extractors/dataverse.png +maintenance_status: active +name: tap-dataverse +namespace: tap_dataverse +next_steps: '' +pip_url: git+https://github.com/mjsqu/tap-dataverse.git +quality: silver +repo: https://github.com/mjsqu/tap-dataverse +settings: +- description: Turns on annotations + kind: boolean + label: Annotations + name: annotations + value: false +- description: The url for the API service + kind: string + label: API URL + name: api_url +- description: The API version found in the /api/data/v{x. Y} of URLs + kind: string + label: API Version + name: api_version + value: '9.2' +- description: Compression format to use for batch files. + kind: options + label: Batch Config Encoding Compression + name: batch_config.encoding.compression + options: + - label: Gzip + value: gzip + - label: None + value: none +- description: Format to use for batch files. + kind: options + label: Batch Config Encoding Format + name: batch_config.encoding.format + options: + - label: Jsonl + value: jsonl + - label: Parquet + value: parquet +- description: Prefix to use when writing batch files. + kind: string + label: Batch Config Storage Prefix + name: batch_config.storage.prefix +- description: Root path to use when writing batch files. + kind: string + label: Batch Config Storage Root + name: batch_config.storage.root +- description: Client (application) ID + kind: password + label: Client ID + name: client_id +- description: The client secret to authenticate against the API service + kind: password + label: Client Secret + name: client_secret +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: array + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed +- description: "'True' to enable schema flattening and automatically expand nested + properties." + kind: boolean + label: Flattening Enabled + name: flattening_enabled +- description: The max depth to flatten schemas. + kind: integer + label: Flattening Max Depth + name: flattening_max_depth +- description: Uses the Snowflake column name rules to translate anycharacters outside + the standard to an underscore. Particularly helpfulwhen annotations are turned + on + kind: boolean + label: Sql Attribute Names + name: sql_attribute_names + value: false +- description: The earliest record date to sync + kind: date_iso8601 + label: Start Date + name: start_date +- description: User-defined config values to be used within map expressions. + kind: object + label: Stream Map Config + name: stream_map_config +- description: Config object for stream maps capability. For more information check + out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). + kind: object + label: Stream Maps + name: stream_maps +- description: An array of streams, designed for separate paths using thesame base + url. + kind: array + label: Streams + name: streams +- description: Tenant ID + kind: password + label: Tenant ID + name: tenant_id +settings_group_validation: +- - api_url + - client_id + - client_secret + - tenant_id +settings_preamble: '' +usage: '' +variant: mjsqu diff --git a/_data/meltano/extractors/tap-db2/danielptv.yml b/_data/meltano/extractors/tap-db2/danielptv.yml index 3cd489024..deab0e9ff 100644 --- a/_data/meltano/extractors/tap-db2/danielptv.yml +++ b/_data/meltano/extractors/tap-db2/danielptv.yml @@ -35,18 +35,26 @@ settings: kind: password label: Encryption SSL Client Key Store Database Database name: encryption.ssl_client_key_store_db.database + sensitive: true - description: The full path to the client key stash. kind: password label: Encryption SSL Client Key Store Database Key Stash name: encryption.ssl_client_key_store_db.key_stash + sensitive: true - description: The keystore password. kind: password label: Encryption SSL Client Key Store Database Password name: encryption.ssl_client_key_store_db.password + sensitive: true - description: The path to the SSL server certificate. kind: string label: Encryption SSL Server Certificate name: encryption.ssl_server_certificate +- description: Apply a custom WHERE condition per stream. Unlike the filter available + in stream_maps, this will be evaluated BEFORE extracting the data. + kind: object + label: Filter + name: filter - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean @@ -67,12 +75,18 @@ settings: label: Ignore Supplied Tables name: ignore_supplied_tables value: true +- description: Ignore views. + kind: boolean + label: Ignore Views + name: ignore_views + value: false - description: The DB2 password. kind: password label: Password name: password + sensitive: true - description: The DB2 port. - kind: string + kind: integer label: Port name: port - description: Partition query into smaller subsets. Useful when working with DB2 diff --git a/_data/meltano/extractors/tap-dbf/edgarrmondragon.yml b/_data/meltano/extractors/tap-dbf/edgarrmondragon.yml index f90cf7be7..13767fe44 100644 --- a/_data/meltano/extractors/tap-dbf/edgarrmondragon.yml +++ b/_data/meltano/extractors/tap-dbf/edgarrmondragon.yml @@ -46,6 +46,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean @@ -55,33 +63,34 @@ settings: kind: integer label: Flattening Max Depth name: flattening_max_depth -- description: Fs Root +- description: The root of the filesystem to read from. kind: string label: Fs Root name: fs_root value: file:// -- description: '' +- description: OAuth 2.0 token for GCS. kind: password label: Gcs Token name: gcs.token -- description: A boolean whether to ignore missing memofile. +- description: Whether to proceed reading the file even if the [memofile] is not present. kind: boolean label: Ignore Missing Memofile name: ignore_missing_memofile value: false -- description: The file path. +- description: Glob expression where the files are located. Stream names will be extracted + from the file name. kind: string label: Path name: path -- description: '' +- description: The S3 endpoint URL. kind: string label: S3 Endpoint URL name: s3.endpoint_url -- description: '' +- description: The AWS key ID. kind: password label: S3 Key name: s3.key -- description: '' +- description: The AWS secret key. kind: password label: S3 Secret name: s3.secret diff --git a/_data/meltano/extractors/tap-dbt/meltanolabs.yml b/_data/meltano/extractors/tap-dbt/meltanolabs.yml index 9516faad4..3a305c6f1 100644 --- a/_data/meltano/extractors/tap-dbt/meltanolabs.yml +++ b/_data/meltano/extractors/tap-dbt/meltanolabs.yml @@ -59,6 +59,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean @@ -86,7 +94,7 @@ settings: kind: string label: User Agent name: user_agent - value: tap-dbt/0.9.1.dev30+9f90f0c Singer tap for the dbt Cloud API. + value: tap-dbt/0.10.1.dev8+c41fe9f Singer tap for the dbt Cloud API. settings_group_validation: - - account_ids - api_key diff --git a/_data/meltano/extractors/tap-delighted/ticketswap.yml b/_data/meltano/extractors/tap-delighted/ticketswap.yml index c8d7b8002..c92dc0290 100644 --- a/_data/meltano/extractors/tap-delighted/ticketswap.yml +++ b/_data/meltano/extractors/tap-delighted/ticketswap.yml @@ -52,6 +52,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-f1/reubenf.yml b/_data/meltano/extractors/tap-f1/reubenf.yml index 47e84724d..56ab18693 100644 --- a/_data/meltano/extractors/tap-f1/reubenf.yml +++ b/_data/meltano/extractors/tap-f1/reubenf.yml @@ -51,6 +51,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-instantly-ai/strvcom.yml b/_data/meltano/extractors/tap-instantly-ai/strvcom.yml new file mode 100644 index 000000000..fe43e8627 --- /dev/null +++ b/_data/meltano/extractors/tap-instantly-ai/strvcom.yml @@ -0,0 +1,89 @@ +capabilities: +- about +- batch +- catalog +- discover +- schema-flattening +- state +- stream-maps +description: AI-powered cold email campaign management platform +domain_url: https://developer.instantly.ai +executable: tap-instantly-ai +keywords: +- meltano_sdk +label: Instantly AI +logo_url: /assets/logos/extractors/instantly-ai.png +maintenance_status: active +name: tap-instantly-ai +namespace: tap_instantly_ai +next_steps: '' +pip_url: git+https://github.com/strvcom/tap-instantly-ai.git +quality: silver +repo: https://github.com/strvcom/tap-instantly-ai +settings: +- description: The API key to authenticate against the API service + kind: password + label: API Key + name: api_key +- description: Compression format to use for batch files. + kind: options + label: Batch Config Encoding Compression + name: batch_config.encoding.compression + options: + - label: Gzip + value: gzip + - label: None + value: none +- description: Format to use for batch files. + kind: options + label: Batch Config Encoding Format + name: batch_config.encoding.format + options: + - label: Jsonl + value: jsonl + - label: Parquet + value: parquet +- description: Prefix to use when writing batch files. + kind: string + label: Batch Config Storage Prefix + name: batch_config.storage.prefix +- description: Root path to use when writing batch files. + kind: string + label: Batch Config Storage Root + name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: array + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed +- description: "'True' to enable schema flattening and automatically expand nested + properties." + kind: boolean + label: Flattening Enabled + name: flattening_enabled +- description: The max depth to flatten schemas. + kind: integer + label: Flattening Max Depth + name: flattening_max_depth +- description: The earliest record date to sync + kind: date_iso8601 + label: Start Date + name: start_date +- description: User-defined config values to be used within map expressions. + kind: object + label: Stream Map Config + name: stream_map_config +- description: Config object for stream maps capability. For more information check + out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). + kind: object + label: Stream Maps + name: stream_maps +settings_group_validation: +- - api_key + - start_date +settings_preamble: '' +usage: '' +variant: strvcom diff --git a/_data/meltano/extractors/tap-mongodb/meltanolabs.yml b/_data/meltano/extractors/tap-mongodb/meltanolabs.yml index 985ad6bfd..1647255b7 100644 --- a/_data/meltano/extractors/tap-mongodb/meltanolabs.yml +++ b/_data/meltano/extractors/tap-mongodb/meltanolabs.yml @@ -185,7 +185,7 @@ settings_preamble: | ```yaml metadata: '*': - replication-key: _id + replication-key: replication_key replication-method: LOG_BASED ``` diff --git a/_data/meltano/extractors/tap-mongodb/z3z1ma.yml b/_data/meltano/extractors/tap-mongodb/z3z1ma.yml index 8df28fd0e..cb41cfa88 100644 --- a/_data/meltano/extractors/tap-mongodb/z3z1ma.yml +++ b/_data/meltano/extractors/tap-mongodb/z3z1ma.yml @@ -36,6 +36,14 @@ settings: kind: array label: Database Includes name: database_includes +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-mssql/buzzcutnorman.yml b/_data/meltano/extractors/tap-mssql/buzzcutnorman.yml index 519100a7f..e50c8d3a6 100644 --- a/_data/meltano/extractors/tap-mssql/buzzcutnorman.yml +++ b/_data/meltano/extractors/tap-mssql/buzzcutnorman.yml @@ -81,6 +81,7 @@ settings: kind: password label: Password name: password + sensitive: true - description: The port on which SQL awaiting connection kind: integer label: Port diff --git a/_data/meltano/extractors/tap-mysql/meltanolabs.yml b/_data/meltano/extractors/tap-mysql/meltanolabs.yml index d36c03687..930b62942 100644 --- a/_data/meltano/extractors/tap-mysql/meltanolabs.yml +++ b/_data/meltano/extractors/tap-mysql/meltanolabs.yml @@ -52,6 +52,14 @@ settings: kind: string label: Database name: database +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: If an array of schema names is provided, the tap will only process the specified MySQL schemas and ignore others. If left blank, the tap automatically determines ALL available MySQL schemas. diff --git a/_data/meltano/extractors/tap-postgres/meltanolabs.yml b/_data/meltano/extractors/tap-postgres/meltanolabs.yml index f9a71c35d..ec908a5ae 100644 --- a/_data/meltano/extractors/tap-postgres/meltanolabs.yml +++ b/_data/meltano/extractors/tap-postgres/meltanolabs.yml @@ -70,6 +70,14 @@ settings: - label: Log Based value: LOG_BASED value: FULL_TABLE +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: If an array of schema names is provided, the tap will only process the specified Postgres schemas and ignore others. If left blank, the tap automatically determines ALL available Postgres schemas. diff --git a/_data/meltano/extractors/tap-pulumi-cloud/meltanolabs.yml b/_data/meltano/extractors/tap-pulumi-cloud/meltanolabs.yml index 6844acdf6..2ce4bb6be 100644 --- a/_data/meltano/extractors/tap-pulumi-cloud/meltanolabs.yml +++ b/_data/meltano/extractors/tap-pulumi-cloud/meltanolabs.yml @@ -47,6 +47,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-qualified/z3z1ma.yml b/_data/meltano/extractors/tap-qualified/z3z1ma.yml index 9de336b54..936cd2e4a 100644 --- a/_data/meltano/extractors/tap-qualified/z3z1ma.yml +++ b/_data/meltano/extractors/tap-qualified/z3z1ma.yml @@ -51,6 +51,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-shortcut/edgarrmondragon.yml b/_data/meltano/extractors/tap-shortcut/edgarrmondragon.yml index f87e2ba96..4237baea4 100644 --- a/_data/meltano/extractors/tap-shortcut/edgarrmondragon.yml +++ b/_data/meltano/extractors/tap-shortcut/edgarrmondragon.yml @@ -49,6 +49,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-showpad/z3z1ma.yml b/_data/meltano/extractors/tap-showpad/z3z1ma.yml index 0759656c2..b1a9af991 100644 --- a/_data/meltano/extractors/tap-showpad/z3z1ma.yml +++ b/_data/meltano/extractors/tap-showpad/z3z1ma.yml @@ -51,6 +51,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-smoke-test/meltano.yml b/_data/meltano/extractors/tap-smoke-test/meltano.yml index e088de18d..f5b2a5713 100644 --- a/_data/meltano/extractors/tap-smoke-test/meltano.yml +++ b/_data/meltano/extractors/tap-smoke-test/meltano.yml @@ -47,6 +47,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-snowflake/meltanolabs.yml b/_data/meltano/extractors/tap-snowflake/meltanolabs.yml index 22690744b..971194bee 100644 --- a/_data/meltano/extractors/tap-snowflake/meltanolabs.yml +++ b/_data/meltano/extractors/tap-snowflake/meltanolabs.yml @@ -56,6 +56,14 @@ settings: kind: string label: Database name: database +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-stackexchange/meltanolabs.yml b/_data/meltano/extractors/tap-stackexchange/meltanolabs.yml index 7fa7d6c33..154bee980 100644 --- a/_data/meltano/extractors/tap-stackexchange/meltanolabs.yml +++ b/_data/meltano/extractors/tap-stackexchange/meltanolabs.yml @@ -47,6 +47,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: Custom API filter to apply to all requests documentation: https://api.stackexchange.com/docs/filters kind: string diff --git a/_data/meltano/extractors/tap-udemy-for-business/immuta.yml b/_data/meltano/extractors/tap-udemy-for-business/immuta.yml index 6174c056c..5f7a7e68d 100644 --- a/_data/meltano/extractors/tap-udemy-for-business/immuta.yml +++ b/_data/meltano/extractors/tap-udemy-for-business/immuta.yml @@ -55,6 +55,14 @@ settings: kind: password label: Client Secret name: client_secret +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/loaders/target-clickhouse/shaped-ai.yml b/_data/meltano/loaders/target-clickhouse/shaped-ai.yml index 3a0d066d9..c2cf949bb 100644 --- a/_data/meltano/loaders/target-clickhouse/shaped-ai.yml +++ b/_data/meltano/loaders/target-clickhouse/shaped-ai.yml @@ -22,10 +22,54 @@ settings: kind: boolean label: Add Record Metadata name: add_record_metadata +- description: The cluster to create tables in. This is passed as the `clickhouse_cluster` + argument when creating a table. [documentation](https://clickhouse.com/docs/en/sql-reference/distributed-ddl) + can be found here. + kind: string + label: Cluster Name + name: cluster_name +- description: Database name + kind: string + label: Database + name: database + value: default - description: The default target database schema name to use for all streams. kind: string label: Default Target Schema name: default_target_schema +- description: Driver type + kind: options + label: Driver + name: driver + options: + - label: Http + value: http + - label: Native + value: native + - label: Asynch + value: asynch + value: http +- description: The engine type to use for the table. + kind: options + label: Engine Type + name: engine_type + options: + - label: Mergetree + value: MergeTree + - label: Replacingmergetree + value: ReplacingMergeTree + - label: Summingmergetree + value: SummingMergeTree + - label: Aggregatingmergetree + value: AggregatingMergeTree + - label: Replicatedmergetree + value: ReplicatedMergeTree + - label: Replicatedreplacingmergetree + value: ReplicatedReplacingMergeTree + - label: Replicatedsummingmergetree + value: ReplicatedSummingMergeTree + - label: Replicatedaggregatingmergetree + value: ReplicatedAggregatingMergeTree - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean @@ -35,6 +79,11 @@ settings: kind: integer label: Flattening Max Depth name: flattening_max_depth +- description: Database host + kind: string + label: Host + name: host + value: localhost - description: The method to use when loading data into the destination. `append-only` will always write all input records whether that records already exists or not. `upsert` will update existing records and insert new records. `overwrite` will @@ -50,7 +99,33 @@ settings: - label: Overwrite value: overwrite value: append-only -- description: The SQLAlchemy connection string for the ClickHouse database +- description: Run 'OPTIMIZE TABLE' after data insert. Useful whentable engine removes + duplicate rows. + kind: boolean + label: Optimize After + name: optimize_after + value: false +- description: Username password + kind: password + label: Password + name: password +- description: Database connection port + kind: integer + label: Port + name: port + value: 8123 +- description: The `replica_name` for replicated tables. This is required when using + any of the replication engines. + kind: string + label: Replica Name + name: replica_name +- description: Should the connection be secure + kind: boolean + label: Secure + name: secure + value: false +- description: The SQLAlchemy connection string for the ClickHouse database. Used + if set, otherwise separate settings are used kind: string label: Sqlalchemy URL name: sqlalchemy_url @@ -67,6 +142,23 @@ settings: kind: string label: Table Name name: table_name +- description: The table path for replicated tables. This is required when using any + of the replication engines. Check out the + [documentation](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication#replicatedmergetree-parameters) + for more information. Use `$table_name` to substitute the table name. + kind: string + label: Table Path + name: table_path +- description: Database user + kind: string + label: Username + name: username + value: default +- description: Should secure connection need to verify SSL/TLS + kind: boolean + label: Verify + name: verify + value: true settings_group_validation: - - sqlalchemy_url settings_preamble: '' diff --git a/_data/meltano/loaders/target-csv/meltanolabs.yml b/_data/meltano/loaders/target-csv/meltanolabs.yml index 84b01fed4..c713a253a 100644 --- a/_data/meltano/loaders/target-csv/meltanolabs.yml +++ b/_data/meltano/loaders/target-csv/meltanolabs.yml @@ -36,6 +36,14 @@ settings: kind: string label: Escape Character name: escape_character +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "The scheme with which output files will be named. Naming scheme may leverage any of the following substitutions: \n\n- `{stream_name}`- `{datestamp}`- `{timestamp}`" @@ -110,6 +118,11 @@ settings: label: Timestamp Timezone name: timestamp_timezone value: UTC +- description: Whether to validate the schema of the incoming streams. + kind: boolean + label: Validate Records + name: validate_records + value: true settings_group_validation: - [] settings_preamble: '' diff --git a/_data/meltano/loaders/target-parquet/automattic.yml b/_data/meltano/loaders/target-parquet/automattic.yml new file mode 100644 index 000000000..5cca73101 --- /dev/null +++ b/_data/meltano/loaders/target-parquet/automattic.yml @@ -0,0 +1,110 @@ +capabilities: +- about +- schema-flattening +- stream-maps +- validate-records +description: Columnar Storage Format +domain_url: https://parquet.apache.org/ +executable: target-parquet +keywords: +- file +- meltano_sdk +label: Parquet +logo_url: /assets/logos/loaders/parquet.png +maintenance_status: active +name: target-parquet +namespace: target_parquet +next_steps: '' +pip_url: git+https://github.com/Automattic/target-parquet.git +quality: silver +repo: https://github.com/Automattic/target-parquet +settings: +- description: Add metadata to records. + kind: boolean + label: Add Record Metadata + name: add_record_metadata +- description: (default - gzip) Compression methods have to be supported by Pyarrow, + and currently the compression modes available are - snappy, zstd, brotli and gzip. + kind: string + label: Compression Method + name: compression_method + value: gzip +- description: Destination Path + kind: string + label: Destination Path + name: destination_path +- description: Extra fields to add to the flattened record. (e.g. extra_col1=value1,extra_col2=value2) + kind: string + label: Extra Fields + name: extra_fields +- description: Extra fields types. (e.g. extra_col1=string,extra_col2=integer) + kind: string + label: Extra Fields Types + name: extra_fields_types +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: array + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed +- description: "'True' to enable schema flattening and automatically expand nested + properties." + kind: boolean + label: Flattening Enabled + name: flattening_enabled +- description: The max depth to flatten schemas. + kind: integer + label: Flattening Max Depth + name: flattening_max_depth +- description: The method to use when loading data into the destination. `append-only` + will always write all input records whether that records already exists or not. + `upsert` will update existing records and insert new records. `overwrite` will + delete all existing records and insert all input records. + kind: options + label: Load Method + name: load_method + options: + - label: Append Only + value: append-only + - label: Upsert + value: upsert + - label: Overwrite + value: overwrite + value: append-only +- description: Max records to write in one batch. It can control the memory usage + of the target. + kind: integer + label: Max Batch Size + name: max_batch_size + value: 10000 +- description: Max size of pyarrow table in MB (before writing to parquet file). It + can control the memory usage of the target. + kind: integer + label: Max Pyarrow Table Size + name: max_pyarrow_table_size + value: 800 +- description: Extra fields to add to the flattened record. (e.g. extra_col1,extra_col2) + kind: string + label: Partition Cols + name: partition_cols +- description: User-defined config values to be used within map expressions. + kind: object + label: Stream Map Config + name: stream_map_config +- description: Config object for stream maps capability. For more information check + out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). + kind: object + label: Stream Maps + name: stream_maps +- description: Whether to validate the schema of the incoming streams. + kind: boolean + label: Validate Records + name: validate_records + value: true +settings_group_validation: +- [] +settings_preamble: '' +usage: '' +variant: automattic diff --git a/poetry.lock b/poetry.lock index 92051316d..ee7a76d2f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -19,33 +19,33 @@ tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy" [[package]] name = "black" -version = "24.1.1" +version = "24.2.0" description = "The uncompromising code formatter." optional = false python-versions = ">=3.8" files = [ - {file = "black-24.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2588021038bd5ada078de606f2a804cadd0a3cc6a79cb3e9bb3a8bf581325a4c"}, - {file = "black-24.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1a95915c98d6e32ca43809d46d932e2abc5f1f7d582ffbe65a5b4d1588af7445"}, - {file = "black-24.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fa6a0e965779c8f2afb286f9ef798df770ba2b6cee063c650b96adec22c056a"}, - {file = "black-24.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:5242ecd9e990aeb995b6d03dc3b2d112d4a78f2083e5a8e86d566340ae80fec4"}, - {file = "black-24.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fc1ec9aa6f4d98d022101e015261c056ddebe3da6a8ccfc2c792cbe0349d48b7"}, - {file = "black-24.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0269dfdea12442022e88043d2910429bed717b2d04523867a85dacce535916b8"}, - {file = "black-24.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3d64db762eae4a5ce04b6e3dd745dcca0fb9560eb931a5be97472e38652a161"}, - {file = "black-24.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:5d7b06ea8816cbd4becfe5f70accae953c53c0e53aa98730ceccb0395520ee5d"}, - {file = "black-24.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e2c8dfa14677f90d976f68e0c923947ae68fa3961d61ee30976c388adc0b02c8"}, - {file = "black-24.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a21725862d0e855ae05da1dd25e3825ed712eaaccef6b03017fe0853a01aa45e"}, - {file = "black-24.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07204d078e25327aad9ed2c64790d681238686bce254c910de640c7cc4fc3aa6"}, - {file = "black-24.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:a83fe522d9698d8f9a101b860b1ee154c1d25f8a82ceb807d319f085b2627c5b"}, - {file = "black-24.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:08b34e85170d368c37ca7bf81cf67ac863c9d1963b2c1780c39102187ec8dd62"}, - {file = "black-24.1.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7258c27115c1e3b5de9ac6c4f9957e3ee2c02c0b39222a24dc7aa03ba0e986f5"}, - {file = "black-24.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40657e1b78212d582a0edecafef133cf1dd02e6677f539b669db4746150d38f6"}, - {file = "black-24.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e298d588744efda02379521a19639ebcd314fba7a49be22136204d7ed1782717"}, - {file = "black-24.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:34afe9da5056aa123b8bfda1664bfe6fb4e9c6f311d8e4a6eb089da9a9173bf9"}, - {file = "black-24.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:854c06fb86fd854140f37fb24dbf10621f5dab9e3b0c29a690ba595e3d543024"}, - {file = "black-24.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3897ae5a21ca132efa219c029cce5e6bfc9c3d34ed7e892113d199c0b1b444a2"}, - {file = "black-24.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:ecba2a15dfb2d97105be74bbfe5128bc5e9fa8477d8c46766505c1dda5883aac"}, - {file = "black-24.1.1-py3-none-any.whl", hash = "sha256:5cdc2e2195212208fbcae579b931407c1fa9997584f0a415421748aeafff1168"}, - {file = "black-24.1.1.tar.gz", hash = "sha256:48b5760dcbfe5cf97fd4fba23946681f3a81514c6ab8a45b50da67ac8fbc6c7b"}, + {file = "black-24.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6981eae48b3b33399c8757036c7f5d48a535b962a7c2310d19361edeef64ce29"}, + {file = "black-24.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d533d5e3259720fdbc1b37444491b024003e012c5173f7d06825a77508085430"}, + {file = "black-24.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61a0391772490ddfb8a693c067df1ef5227257e72b0e4108482b8d41b5aee13f"}, + {file = "black-24.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:992e451b04667116680cb88f63449267c13e1ad134f30087dec8527242e9862a"}, + {file = "black-24.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:163baf4ef40e6897a2a9b83890e59141cc8c2a98f2dda5080dc15c00ee1e62cd"}, + {file = "black-24.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e37c99f89929af50ffaf912454b3e3b47fd64109659026b678c091a4cd450fb2"}, + {file = "black-24.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9de21bafcba9683853f6c96c2d515e364aee631b178eaa5145fc1c61a3cc92"}, + {file = "black-24.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:9db528bccb9e8e20c08e716b3b09c6bdd64da0dd129b11e160bf082d4642ac23"}, + {file = "black-24.2.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d84f29eb3ee44859052073b7636533ec995bd0f64e2fb43aeceefc70090e752b"}, + {file = "black-24.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e08fb9a15c914b81dd734ddd7fb10513016e5ce7e6704bdd5e1251ceee51ac9"}, + {file = "black-24.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:810d445ae6069ce64030c78ff6127cd9cd178a9ac3361435708b907d8a04c693"}, + {file = "black-24.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:ba15742a13de85e9b8f3239c8f807723991fbfae24bad92d34a2b12e81904982"}, + {file = "black-24.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7e53a8c630f71db01b28cd9602a1ada68c937cbf2c333e6ed041390d6968faf4"}, + {file = "black-24.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:93601c2deb321b4bad8f95df408e3fb3943d85012dddb6121336b8e24a0d1218"}, + {file = "black-24.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0057f800de6acc4407fe75bb147b0c2b5cbb7c3ed110d3e5999cd01184d53b0"}, + {file = "black-24.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:faf2ee02e6612577ba0181f4347bcbcf591eb122f7841ae5ba233d12c39dcb4d"}, + {file = "black-24.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:057c3dc602eaa6fdc451069bd027a1b2635028b575a6c3acfd63193ced20d9c8"}, + {file = "black-24.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:08654d0797e65f2423f850fc8e16a0ce50925f9337fb4a4a176a7aa4026e63f8"}, + {file = "black-24.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca610d29415ee1a30a3f30fab7a8f4144e9d34c89a235d81292a1edb2b55f540"}, + {file = "black-24.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:4dd76e9468d5536abd40ffbc7a247f83b2324f0c050556d9c371c2b9a9a95e31"}, + {file = "black-24.2.0-py3-none-any.whl", hash = "sha256:e8a6ae970537e67830776488bca52000eaa37fa63b9988e8c487458d9cd5ace6"}, + {file = "black-24.2.0.tar.gz", hash = "sha256:bce4f25c27c3435e4dace4815bcb2008b87e167e3bf4ee47ccdc5ce906eb4894"}, ] [package.dependencies] @@ -480,13 +480,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "yamllint" -version = "1.34.0" +version = "1.35.1" description = "A linter for YAML files." optional = false python-versions = ">=3.8" files = [ - {file = "yamllint-1.34.0-py3-none-any.whl", hash = "sha256:33b813f6ff2ffad2e57a288281098392b85f7463ce1f3d5cd45aa848b916a806"}, - {file = "yamllint-1.34.0.tar.gz", hash = "sha256:7f0a6a41e8aab3904878da4ae34b6248b6bc74634e0d3a90f0fb2d7e723a3d4f"}, + {file = "yamllint-1.35.1-py3-none-any.whl", hash = "sha256:2e16e504bb129ff515b37823b472750b36b6de07963bd74b307341ef5ad8bdc3"}, + {file = "yamllint-1.35.1.tar.gz", hash = "sha256:7a003809f88324fd2c877734f2d575ee7881dd9043360657cc8049c809eba6cd"}, ] [package.dependencies] @@ -499,4 +499,4 @@ dev = ["doc8", "flake8", "flake8-import-order", "rstcheck[sphinx]", "sphinx"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "2aac13dee53d31977f5644e004f8a63dd7e9d2f043223df292ac6670f209e4ef" +content-hash = "10ee8a0539ff8c11f7e20b8e346daf8d1455783dd47a3e63a7de2b6fc323ddb5" diff --git a/pyproject.toml b/pyproject.toml index 88549ac5f..7a1ed8f8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,11 +8,11 @@ authors = ["Meltano Team"] python = "^3.9" jsonschema = "^4.17.3" "ruamel.yaml" = "^0.18.6" -yamllint = "^1.34.0" +yamllint = "^1.35.1" requests = "^2.31.0" [tool.poetry.dev-dependencies] -black = "^24.1.1" +black = "^24.2.0" isort = "^5.13.2" [build-system] diff --git a/schemas/common/base_setting.schema.json b/schemas/common/base_setting.schema.json index 1674f549f..a72b362b1 100644 --- a/schemas/common/base_setting.schema.json +++ b/schemas/common/base_setting.schema.json @@ -143,6 +143,16 @@ "docs": { "type": "string", "description": "A URL to the documentation for this plugin" + }, + "hidden": { + "type": "boolean", + "description": "A hidden setting should not be user-configurable", + "default": false + }, + "sensitive": { + "type": "boolean", + "description": "A sensitive setting is redacted in command output", + "default": false } } } diff --git a/schemas/common/loaders_specific.schema.json b/schemas/common/loaders_specific.schema.json index 11cacbe9e..bc742c5ab 100644 --- a/schemas/common/loaders_specific.schema.json +++ b/schemas/common/loaders_specific.schema.json @@ -20,7 +20,8 @@ "datatype-failsafe", "record-flattening", "schema-flattening", - "target-schema" + "target-schema", + "validate-records" ] } }, diff --git a/static/assets/logos/extractors/microsoft-dataverse.png b/static/assets/logos/extractors/dataverse.png similarity index 100% rename from static/assets/logos/extractors/microsoft-dataverse.png rename to static/assets/logos/extractors/dataverse.png diff --git a/static/assets/logos/extractors/instantly-ai.png b/static/assets/logos/extractors/instantly-ai.png new file mode 100644 index 000000000..20831f5ab Binary files /dev/null and b/static/assets/logos/extractors/instantly-ai.png differ