From 5baa08defe2114b43b7f622781f642814db29fe1 Mon Sep 17 00:00:00 2001 From: Julio Contreras <48543745+JulioWhile@users.noreply.github.com> Date: Sun, 11 Feb 2024 10:34:38 -0700 Subject: [PATCH 1/6] Update MongoDB Documentation for Replication Key (#1668) Signed-off-by: Julio Contreras --- _data/meltano/extractors/tap-mongodb/meltanolabs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/meltano/extractors/tap-mongodb/meltanolabs.yml b/_data/meltano/extractors/tap-mongodb/meltanolabs.yml index 985ad6bfd..1647255b7 100644 --- a/_data/meltano/extractors/tap-mongodb/meltanolabs.yml +++ b/_data/meltano/extractors/tap-mongodb/meltanolabs.yml @@ -185,7 +185,7 @@ settings_preamble: | ```yaml metadata: '*': - replication-key: _id + replication-key: replication_key replication-method: LOG_BASED ``` From e3332dc35638edd511b8bf7aeb31a65c31b09cf5 Mon Sep 17 00:00:00 2001 From: MeltyBot <105875157+MeltyBot@users.noreply.github.com> Date: Mon, 12 Feb 2024 16:21:34 -0500 Subject: [PATCH 2/6] chore: [Hub Bot] Refresh metadata 2024-02-12 (#1669) Co-authored-by: pnadolny13 Co-authored-by: Pat Nadolny --- .../extractors/tap-athena/meltanolabs.yml | 8 ++ .../extractors/tap-bitso/edgarrmondragon.yml | 8 ++ .../extractors/tap-circle-ci/meltanolabs.yml | 8 ++ .../extractors/tap-csv/meltanolabs.yml | 8 ++ .../extractors/tap-dbf/edgarrmondragon.yml | 23 +++-- .../extractors/tap-dbt/meltanolabs.yml | 10 +- .../extractors/tap-delighted/ticketswap.yml | 8 ++ _data/meltano/extractors/tap-f1/reubenf.yml | 8 ++ .../meltano/extractors/tap-mongodb/z3z1ma.yml | 8 ++ .../extractors/tap-mysql/meltanolabs.yml | 8 ++ .../extractors/tap-postgres/meltanolabs.yml | 8 ++ .../tap-pulumi-cloud/meltanolabs.yml | 8 ++ .../extractors/tap-qualified/z3z1ma.yml | 8 ++ .../tap-shortcut/edgarrmondragon.yml | 8 ++ .../meltano/extractors/tap-showpad/z3z1ma.yml | 8 ++ .../extractors/tap-smoke-test/meltano.yml | 8 ++ .../extractors/tap-snowflake/meltanolabs.yml | 8 ++ .../tap-stackexchange/meltanolabs.yml | 8 ++ .../tap-udemy-for-business/immuta.yml | 8 ++ .../loaders/target-clickhouse/shaped-ai.yml | 94 ++++++++++++++++++- .../loaders/target-csv/meltanolabs.yml | 13 +++ 21 files changed, 267 insertions(+), 9 deletions(-) diff --git a/_data/meltano/extractors/tap-athena/meltanolabs.yml b/_data/meltano/extractors/tap-athena/meltanolabs.yml index 4f037f39f..c48041a79 100644 --- a/_data/meltano/extractors/tap-athena/meltanolabs.yml +++ b/_data/meltano/extractors/tap-athena/meltanolabs.yml @@ -60,6 +60,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-bitso/edgarrmondragon.yml b/_data/meltano/extractors/tap-bitso/edgarrmondragon.yml index 41660144a..e95c0b76a 100644 --- a/_data/meltano/extractors/tap-bitso/edgarrmondragon.yml +++ b/_data/meltano/extractors/tap-bitso/edgarrmondragon.yml @@ -59,6 +59,14 @@ settings: name: books value: - btc_mxn +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-circle-ci/meltanolabs.yml b/_data/meltano/extractors/tap-circle-ci/meltanolabs.yml index 3a1af36d3..973da98e8 100644 --- a/_data/meltano/extractors/tap-circle-ci/meltanolabs.yml +++ b/_data/meltano/extractors/tap-circle-ci/meltanolabs.yml @@ -52,6 +52,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-csv/meltanolabs.yml b/_data/meltano/extractors/tap-csv/meltanolabs.yml index 988ea1382..cec219475 100644 --- a/_data/meltano/extractors/tap-csv/meltanolabs.yml +++ b/_data/meltano/extractors/tap-csv/meltanolabs.yml @@ -39,6 +39,14 @@ settings: label: Csv Files Definition name: csv_files_definition placeholder: Ex. files-def.json +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: | Array of objects with `entity`, `path`, `keys`, and `encoding` [Optional] keys: diff --git a/_data/meltano/extractors/tap-dbf/edgarrmondragon.yml b/_data/meltano/extractors/tap-dbf/edgarrmondragon.yml index f90cf7be7..13767fe44 100644 --- a/_data/meltano/extractors/tap-dbf/edgarrmondragon.yml +++ b/_data/meltano/extractors/tap-dbf/edgarrmondragon.yml @@ -46,6 +46,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean @@ -55,33 +63,34 @@ settings: kind: integer label: Flattening Max Depth name: flattening_max_depth -- description: Fs Root +- description: The root of the filesystem to read from. kind: string label: Fs Root name: fs_root value: file:// -- description: '' +- description: OAuth 2.0 token for GCS. kind: password label: Gcs Token name: gcs.token -- description: A boolean whether to ignore missing memofile. +- description: Whether to proceed reading the file even if the [memofile] is not present. kind: boolean label: Ignore Missing Memofile name: ignore_missing_memofile value: false -- description: The file path. +- description: Glob expression where the files are located. Stream names will be extracted + from the file name. kind: string label: Path name: path -- description: '' +- description: The S3 endpoint URL. kind: string label: S3 Endpoint URL name: s3.endpoint_url -- description: '' +- description: The AWS key ID. kind: password label: S3 Key name: s3.key -- description: '' +- description: The AWS secret key. kind: password label: S3 Secret name: s3.secret diff --git a/_data/meltano/extractors/tap-dbt/meltanolabs.yml b/_data/meltano/extractors/tap-dbt/meltanolabs.yml index 9516faad4..3a305c6f1 100644 --- a/_data/meltano/extractors/tap-dbt/meltanolabs.yml +++ b/_data/meltano/extractors/tap-dbt/meltanolabs.yml @@ -59,6 +59,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean @@ -86,7 +94,7 @@ settings: kind: string label: User Agent name: user_agent - value: tap-dbt/0.9.1.dev30+9f90f0c Singer tap for the dbt Cloud API. + value: tap-dbt/0.10.1.dev8+c41fe9f Singer tap for the dbt Cloud API. settings_group_validation: - - account_ids - api_key diff --git a/_data/meltano/extractors/tap-delighted/ticketswap.yml b/_data/meltano/extractors/tap-delighted/ticketswap.yml index c8d7b8002..c92dc0290 100644 --- a/_data/meltano/extractors/tap-delighted/ticketswap.yml +++ b/_data/meltano/extractors/tap-delighted/ticketswap.yml @@ -52,6 +52,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-f1/reubenf.yml b/_data/meltano/extractors/tap-f1/reubenf.yml index 47e84724d..56ab18693 100644 --- a/_data/meltano/extractors/tap-f1/reubenf.yml +++ b/_data/meltano/extractors/tap-f1/reubenf.yml @@ -51,6 +51,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-mongodb/z3z1ma.yml b/_data/meltano/extractors/tap-mongodb/z3z1ma.yml index 8df28fd0e..cb41cfa88 100644 --- a/_data/meltano/extractors/tap-mongodb/z3z1ma.yml +++ b/_data/meltano/extractors/tap-mongodb/z3z1ma.yml @@ -36,6 +36,14 @@ settings: kind: array label: Database Includes name: database_includes +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-mysql/meltanolabs.yml b/_data/meltano/extractors/tap-mysql/meltanolabs.yml index d36c03687..930b62942 100644 --- a/_data/meltano/extractors/tap-mysql/meltanolabs.yml +++ b/_data/meltano/extractors/tap-mysql/meltanolabs.yml @@ -52,6 +52,14 @@ settings: kind: string label: Database name: database +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: If an array of schema names is provided, the tap will only process the specified MySQL schemas and ignore others. If left blank, the tap automatically determines ALL available MySQL schemas. diff --git a/_data/meltano/extractors/tap-postgres/meltanolabs.yml b/_data/meltano/extractors/tap-postgres/meltanolabs.yml index f9a71c35d..ec908a5ae 100644 --- a/_data/meltano/extractors/tap-postgres/meltanolabs.yml +++ b/_data/meltano/extractors/tap-postgres/meltanolabs.yml @@ -70,6 +70,14 @@ settings: - label: Log Based value: LOG_BASED value: FULL_TABLE +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: If an array of schema names is provided, the tap will only process the specified Postgres schemas and ignore others. If left blank, the tap automatically determines ALL available Postgres schemas. diff --git a/_data/meltano/extractors/tap-pulumi-cloud/meltanolabs.yml b/_data/meltano/extractors/tap-pulumi-cloud/meltanolabs.yml index 6844acdf6..2ce4bb6be 100644 --- a/_data/meltano/extractors/tap-pulumi-cloud/meltanolabs.yml +++ b/_data/meltano/extractors/tap-pulumi-cloud/meltanolabs.yml @@ -47,6 +47,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-qualified/z3z1ma.yml b/_data/meltano/extractors/tap-qualified/z3z1ma.yml index 9de336b54..936cd2e4a 100644 --- a/_data/meltano/extractors/tap-qualified/z3z1ma.yml +++ b/_data/meltano/extractors/tap-qualified/z3z1ma.yml @@ -51,6 +51,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-shortcut/edgarrmondragon.yml b/_data/meltano/extractors/tap-shortcut/edgarrmondragon.yml index f87e2ba96..4237baea4 100644 --- a/_data/meltano/extractors/tap-shortcut/edgarrmondragon.yml +++ b/_data/meltano/extractors/tap-shortcut/edgarrmondragon.yml @@ -49,6 +49,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-showpad/z3z1ma.yml b/_data/meltano/extractors/tap-showpad/z3z1ma.yml index 0759656c2..b1a9af991 100644 --- a/_data/meltano/extractors/tap-showpad/z3z1ma.yml +++ b/_data/meltano/extractors/tap-showpad/z3z1ma.yml @@ -51,6 +51,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-smoke-test/meltano.yml b/_data/meltano/extractors/tap-smoke-test/meltano.yml index e088de18d..f5b2a5713 100644 --- a/_data/meltano/extractors/tap-smoke-test/meltano.yml +++ b/_data/meltano/extractors/tap-smoke-test/meltano.yml @@ -47,6 +47,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-snowflake/meltanolabs.yml b/_data/meltano/extractors/tap-snowflake/meltanolabs.yml index 22690744b..971194bee 100644 --- a/_data/meltano/extractors/tap-snowflake/meltanolabs.yml +++ b/_data/meltano/extractors/tap-snowflake/meltanolabs.yml @@ -56,6 +56,14 @@ settings: kind: string label: Database name: database +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/extractors/tap-stackexchange/meltanolabs.yml b/_data/meltano/extractors/tap-stackexchange/meltanolabs.yml index 7fa7d6c33..154bee980 100644 --- a/_data/meltano/extractors/tap-stackexchange/meltanolabs.yml +++ b/_data/meltano/extractors/tap-stackexchange/meltanolabs.yml @@ -47,6 +47,14 @@ settings: kind: string label: Batch Config Storage Root name: batch_config.storage.root +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: Custom API filter to apply to all requests documentation: https://api.stackexchange.com/docs/filters kind: string diff --git a/_data/meltano/extractors/tap-udemy-for-business/immuta.yml b/_data/meltano/extractors/tap-udemy-for-business/immuta.yml index 6174c056c..5f7a7e68d 100644 --- a/_data/meltano/extractors/tap-udemy-for-business/immuta.yml +++ b/_data/meltano/extractors/tap-udemy-for-business/immuta.yml @@ -55,6 +55,14 @@ settings: kind: password label: Client Secret name: client_secret +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean diff --git a/_data/meltano/loaders/target-clickhouse/shaped-ai.yml b/_data/meltano/loaders/target-clickhouse/shaped-ai.yml index 3a0d066d9..c2cf949bb 100644 --- a/_data/meltano/loaders/target-clickhouse/shaped-ai.yml +++ b/_data/meltano/loaders/target-clickhouse/shaped-ai.yml @@ -22,10 +22,54 @@ settings: kind: boolean label: Add Record Metadata name: add_record_metadata +- description: The cluster to create tables in. This is passed as the `clickhouse_cluster` + argument when creating a table. [documentation](https://clickhouse.com/docs/en/sql-reference/distributed-ddl) + can be found here. + kind: string + label: Cluster Name + name: cluster_name +- description: Database name + kind: string + label: Database + name: database + value: default - description: The default target database schema name to use for all streams. kind: string label: Default Target Schema name: default_target_schema +- description: Driver type + kind: options + label: Driver + name: driver + options: + - label: Http + value: http + - label: Native + value: native + - label: Asynch + value: asynch + value: http +- description: The engine type to use for the table. + kind: options + label: Engine Type + name: engine_type + options: + - label: Mergetree + value: MergeTree + - label: Replacingmergetree + value: ReplacingMergeTree + - label: Summingmergetree + value: SummingMergeTree + - label: Aggregatingmergetree + value: AggregatingMergeTree + - label: Replicatedmergetree + value: ReplicatedMergeTree + - label: Replicatedreplacingmergetree + value: ReplicatedReplacingMergeTree + - label: Replicatedsummingmergetree + value: ReplicatedSummingMergeTree + - label: Replicatedaggregatingmergetree + value: ReplicatedAggregatingMergeTree - description: "'True' to enable schema flattening and automatically expand nested properties." kind: boolean @@ -35,6 +79,11 @@ settings: kind: integer label: Flattening Max Depth name: flattening_max_depth +- description: Database host + kind: string + label: Host + name: host + value: localhost - description: The method to use when loading data into the destination. `append-only` will always write all input records whether that records already exists or not. `upsert` will update existing records and insert new records. `overwrite` will @@ -50,7 +99,33 @@ settings: - label: Overwrite value: overwrite value: append-only -- description: The SQLAlchemy connection string for the ClickHouse database +- description: Run 'OPTIMIZE TABLE' after data insert. Useful whentable engine removes + duplicate rows. + kind: boolean + label: Optimize After + name: optimize_after + value: false +- description: Username password + kind: password + label: Password + name: password +- description: Database connection port + kind: integer + label: Port + name: port + value: 8123 +- description: The `replica_name` for replicated tables. This is required when using + any of the replication engines. + kind: string + label: Replica Name + name: replica_name +- description: Should the connection be secure + kind: boolean + label: Secure + name: secure + value: false +- description: The SQLAlchemy connection string for the ClickHouse database. Used + if set, otherwise separate settings are used kind: string label: Sqlalchemy URL name: sqlalchemy_url @@ -67,6 +142,23 @@ settings: kind: string label: Table Name name: table_name +- description: The table path for replicated tables. This is required when using any + of the replication engines. Check out the + [documentation](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication#replicatedmergetree-parameters) + for more information. Use `$table_name` to substitute the table name. + kind: string + label: Table Path + name: table_path +- description: Database user + kind: string + label: Username + name: username + value: default +- description: Should secure connection need to verify SSL/TLS + kind: boolean + label: Verify + name: verify + value: true settings_group_validation: - - sqlalchemy_url settings_preamble: '' diff --git a/_data/meltano/loaders/target-csv/meltanolabs.yml b/_data/meltano/loaders/target-csv/meltanolabs.yml index 84b01fed4..c713a253a 100644 --- a/_data/meltano/loaders/target-csv/meltanolabs.yml +++ b/_data/meltano/loaders/target-csv/meltanolabs.yml @@ -36,6 +36,14 @@ settings: kind: string label: Escape Character name: escape_character +- description: 'One or more LCID locale strings to produce localized output for: https://faker.readthedocs.io/en/master/#localization' + kind: string + label: Faker Config Locale + name: faker_config.locale +- description: 'Value to seed the Faker generator for deterministic output: https://faker.readthedocs.io/en/master/#seeding-the-generator' + kind: string + label: Faker Config Seed + name: faker_config.seed - description: "The scheme with which output files will be named. Naming scheme may leverage any of the following substitutions: \n\n- `{stream_name}`- `{datestamp}`- `{timestamp}`" @@ -110,6 +118,11 @@ settings: label: Timestamp Timezone name: timestamp_timezone value: UTC +- description: Whether to validate the schema of the incoming streams. + kind: boolean + label: Validate Records + name: validate_records + value: true settings_group_validation: - [] settings_preamble: '' From 68127754c69df71c2737af6941eb1513b462fc38 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Feb 2024 15:34:32 -0600 Subject: [PATCH 3/6] chore(deps): bump aws-actions/configure-aws-credentials from 4.0.1 to 4.0.2 (#1670) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Edgar Ramírez Mondragón <16805946+edgarrmondragon@users.noreply.github.com> --- .github/workflows/build-api.yml | 17 +++++++++-------- .github/workflows/metadata-extract.yml | 6 +++--- .github/workflows/metadata-refresh.yml | 2 +- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-api.yml b/.github/workflows/build-api.yml index 99032fa0e..941131ddb 100644 --- a/.github/workflows/build-api.yml +++ b/.github/workflows/build-api.yml @@ -8,13 +8,14 @@ on: branches: - main workflow_dispatch: - hub-url: - description: The base URL of the Meltano Hub site - required: true - default: 'https://hub.meltano.com' - api-url: - description: The base URL of the Hub API - required: true + inputs: + hub-url: + description: The base URL of the Meltano Hub site + required: true + default: 'https://hub.meltano.com' + api-url: + description: The base URL of the Hub API + required: true jobs: build: @@ -83,7 +84,7 @@ jobs: steps: - uses: actions/checkout@v4.1.1 - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4.0.1 + uses: aws-actions/configure-aws-credentials@v4.0.2 with: role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} aws-region: us-west-2 diff --git a/.github/workflows/metadata-extract.yml b/.github/workflows/metadata-extract.yml index 24526e123..1bbc33959 100644 --- a/.github/workflows/metadata-extract.yml +++ b/.github/workflows/metadata-extract.yml @@ -74,7 +74,7 @@ jobs: - uses: actions/checkout@v4.1.1 - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4.0.1 + uses: aws-actions/configure-aws-credentials@v4.0.2 with: role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} aws-region: us-west-2 @@ -132,7 +132,7 @@ jobs: - uses: actions/checkout@v4.1.1 - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4.0.1 + uses: aws-actions/configure-aws-credentials@v4.0.2 with: role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} aws-region: us-west-2 @@ -211,7 +211,7 @@ jobs: - uses: actions/checkout@v4.1.1 - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4.0.1 + uses: aws-actions/configure-aws-credentials@v4.0.2 with: role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} aws-region: us-west-2 diff --git a/.github/workflows/metadata-refresh.yml b/.github/workflows/metadata-refresh.yml index 21866ee11..853ee20a3 100644 --- a/.github/workflows/metadata-refresh.yml +++ b/.github/workflows/metadata-refresh.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/checkout@v4.1.1 - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4.0.1 + uses: aws-actions/configure-aws-credentials@v4.0.2 with: role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} aws-region: us-west-2 From 290a5e505f4aa7fb1f5e6cdd18a0c3cd2d87baf0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Feb 2024 12:33:46 +0000 Subject: [PATCH 4/6] chore(deps-dev): bump black from 24.1.1 to 24.2.0 Bumps [black](https://github.com/psf/black) from 24.1.1 to 24.2.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/24.1.1...24.2.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- poetry.lock | 48 ++++++++++++++++++++++++------------------------ pyproject.toml | 2 +- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/poetry.lock b/poetry.lock index 92051316d..60fa28e95 100644 --- a/poetry.lock +++ b/poetry.lock @@ -19,33 +19,33 @@ tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy" [[package]] name = "black" -version = "24.1.1" +version = "24.2.0" description = "The uncompromising code formatter." optional = false python-versions = ">=3.8" files = [ - {file = "black-24.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2588021038bd5ada078de606f2a804cadd0a3cc6a79cb3e9bb3a8bf581325a4c"}, - {file = "black-24.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1a95915c98d6e32ca43809d46d932e2abc5f1f7d582ffbe65a5b4d1588af7445"}, - {file = "black-24.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fa6a0e965779c8f2afb286f9ef798df770ba2b6cee063c650b96adec22c056a"}, - {file = "black-24.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:5242ecd9e990aeb995b6d03dc3b2d112d4a78f2083e5a8e86d566340ae80fec4"}, - {file = "black-24.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fc1ec9aa6f4d98d022101e015261c056ddebe3da6a8ccfc2c792cbe0349d48b7"}, - {file = "black-24.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0269dfdea12442022e88043d2910429bed717b2d04523867a85dacce535916b8"}, - {file = "black-24.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3d64db762eae4a5ce04b6e3dd745dcca0fb9560eb931a5be97472e38652a161"}, - {file = "black-24.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:5d7b06ea8816cbd4becfe5f70accae953c53c0e53aa98730ceccb0395520ee5d"}, - {file = "black-24.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e2c8dfa14677f90d976f68e0c923947ae68fa3961d61ee30976c388adc0b02c8"}, - {file = "black-24.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a21725862d0e855ae05da1dd25e3825ed712eaaccef6b03017fe0853a01aa45e"}, - {file = "black-24.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07204d078e25327aad9ed2c64790d681238686bce254c910de640c7cc4fc3aa6"}, - {file = "black-24.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:a83fe522d9698d8f9a101b860b1ee154c1d25f8a82ceb807d319f085b2627c5b"}, - {file = "black-24.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:08b34e85170d368c37ca7bf81cf67ac863c9d1963b2c1780c39102187ec8dd62"}, - {file = "black-24.1.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7258c27115c1e3b5de9ac6c4f9957e3ee2c02c0b39222a24dc7aa03ba0e986f5"}, - {file = "black-24.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40657e1b78212d582a0edecafef133cf1dd02e6677f539b669db4746150d38f6"}, - {file = "black-24.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:e298d588744efda02379521a19639ebcd314fba7a49be22136204d7ed1782717"}, - {file = "black-24.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:34afe9da5056aa123b8bfda1664bfe6fb4e9c6f311d8e4a6eb089da9a9173bf9"}, - {file = "black-24.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:854c06fb86fd854140f37fb24dbf10621f5dab9e3b0c29a690ba595e3d543024"}, - {file = "black-24.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3897ae5a21ca132efa219c029cce5e6bfc9c3d34ed7e892113d199c0b1b444a2"}, - {file = "black-24.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:ecba2a15dfb2d97105be74bbfe5128bc5e9fa8477d8c46766505c1dda5883aac"}, - {file = "black-24.1.1-py3-none-any.whl", hash = "sha256:5cdc2e2195212208fbcae579b931407c1fa9997584f0a415421748aeafff1168"}, - {file = "black-24.1.1.tar.gz", hash = "sha256:48b5760dcbfe5cf97fd4fba23946681f3a81514c6ab8a45b50da67ac8fbc6c7b"}, + {file = "black-24.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6981eae48b3b33399c8757036c7f5d48a535b962a7c2310d19361edeef64ce29"}, + {file = "black-24.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d533d5e3259720fdbc1b37444491b024003e012c5173f7d06825a77508085430"}, + {file = "black-24.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61a0391772490ddfb8a693c067df1ef5227257e72b0e4108482b8d41b5aee13f"}, + {file = "black-24.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:992e451b04667116680cb88f63449267c13e1ad134f30087dec8527242e9862a"}, + {file = "black-24.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:163baf4ef40e6897a2a9b83890e59141cc8c2a98f2dda5080dc15c00ee1e62cd"}, + {file = "black-24.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e37c99f89929af50ffaf912454b3e3b47fd64109659026b678c091a4cd450fb2"}, + {file = "black-24.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9de21bafcba9683853f6c96c2d515e364aee631b178eaa5145fc1c61a3cc92"}, + {file = "black-24.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:9db528bccb9e8e20c08e716b3b09c6bdd64da0dd129b11e160bf082d4642ac23"}, + {file = "black-24.2.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d84f29eb3ee44859052073b7636533ec995bd0f64e2fb43aeceefc70090e752b"}, + {file = "black-24.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e08fb9a15c914b81dd734ddd7fb10513016e5ce7e6704bdd5e1251ceee51ac9"}, + {file = "black-24.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:810d445ae6069ce64030c78ff6127cd9cd178a9ac3361435708b907d8a04c693"}, + {file = "black-24.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:ba15742a13de85e9b8f3239c8f807723991fbfae24bad92d34a2b12e81904982"}, + {file = "black-24.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7e53a8c630f71db01b28cd9602a1ada68c937cbf2c333e6ed041390d6968faf4"}, + {file = "black-24.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:93601c2deb321b4bad8f95df408e3fb3943d85012dddb6121336b8e24a0d1218"}, + {file = "black-24.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0057f800de6acc4407fe75bb147b0c2b5cbb7c3ed110d3e5999cd01184d53b0"}, + {file = "black-24.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:faf2ee02e6612577ba0181f4347bcbcf591eb122f7841ae5ba233d12c39dcb4d"}, + {file = "black-24.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:057c3dc602eaa6fdc451069bd027a1b2635028b575a6c3acfd63193ced20d9c8"}, + {file = "black-24.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:08654d0797e65f2423f850fc8e16a0ce50925f9337fb4a4a176a7aa4026e63f8"}, + {file = "black-24.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca610d29415ee1a30a3f30fab7a8f4144e9d34c89a235d81292a1edb2b55f540"}, + {file = "black-24.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:4dd76e9468d5536abd40ffbc7a247f83b2324f0c050556d9c371c2b9a9a95e31"}, + {file = "black-24.2.0-py3-none-any.whl", hash = "sha256:e8a6ae970537e67830776488bca52000eaa37fa63b9988e8c487458d9cd5ace6"}, + {file = "black-24.2.0.tar.gz", hash = "sha256:bce4f25c27c3435e4dace4815bcb2008b87e167e3bf4ee47ccdc5ce906eb4894"}, ] [package.dependencies] @@ -499,4 +499,4 @@ dev = ["doc8", "flake8", "flake8-import-order", "rstcheck[sphinx]", "sphinx"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "2aac13dee53d31977f5644e004f8a63dd7e9d2f043223df292ac6670f209e4ef" +content-hash = "9d6b9aadbcf54edcc30babb50d472c82769cdfa2ace1411c580f22c0a556809b" diff --git a/pyproject.toml b/pyproject.toml index 88549ac5f..811fdc3a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ yamllint = "^1.34.0" requests = "^2.31.0" [tool.poetry.dev-dependencies] -black = "^24.1.1" +black = "^24.2.0" isort = "^5.13.2" [build-system] From 297ac30b3d0b920f8894697d2555c2e397da1caf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Feb 2024 12:53:29 +0000 Subject: [PATCH 5/6] chore(deps): bump yamllint from 1.34.0 to 1.35.0 Bumps [yamllint](https://github.com/adrienverge/yamllint) from 1.34.0 to 1.35.0. - [Changelog](https://github.com/adrienverge/yamllint/blob/master/CHANGELOG.rst) - [Commits](https://github.com/adrienverge/yamllint/compare/v1.34.0...v1.35.0) --- updated-dependencies: - dependency-name: yamllint dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 60fa28e95..0fbcadb8c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -480,13 +480,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "yamllint" -version = "1.34.0" +version = "1.35.0" description = "A linter for YAML files." optional = false python-versions = ">=3.8" files = [ - {file = "yamllint-1.34.0-py3-none-any.whl", hash = "sha256:33b813f6ff2ffad2e57a288281098392b85f7463ce1f3d5cd45aa848b916a806"}, - {file = "yamllint-1.34.0.tar.gz", hash = "sha256:7f0a6a41e8aab3904878da4ae34b6248b6bc74634e0d3a90f0fb2d7e723a3d4f"}, + {file = "yamllint-1.35.0-py3-none-any.whl", hash = "sha256:601b0adaaac6d9bacb16a2e612e7ee8d23caf941ceebf9bfe2cff0f196266004"}, + {file = "yamllint-1.35.0.tar.gz", hash = "sha256:9bc99c3e9fe89b4c6ee26e17aa817cf2d14390de6577cb6e2e6ed5f72120c835"}, ] [package.dependencies] @@ -499,4 +499,4 @@ dev = ["doc8", "flake8", "flake8-import-order", "rstcheck[sphinx]", "sphinx"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "9d6b9aadbcf54edcc30babb50d472c82769cdfa2ace1411c580f22c0a556809b" +content-hash = "a31d3e8bd0f0b231d70887305ce919bb9d247df12967e15aed3093159239f69f" diff --git a/pyproject.toml b/pyproject.toml index 811fdc3a5..2475690dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ authors = ["Meltano Team"] python = "^3.9" jsonschema = "^4.17.3" "ruamel.yaml" = "^0.18.6" -yamllint = "^1.34.0" +yamllint = "^1.35.0" requests = "^2.31.0" [tool.poetry.dev-dependencies] From ef21b1b366fa508c4fd4113c3eaab3f3ff802f89 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Feb 2024 12:53:30 +0000 Subject: [PATCH 6/6] chore(deps): bump yamllint from 1.35.0 to 1.35.1 Bumps [yamllint](https://github.com/adrienverge/yamllint) from 1.35.0 to 1.35.1. - [Changelog](https://github.com/adrienverge/yamllint/blob/master/CHANGELOG.rst) - [Commits](https://github.com/adrienverge/yamllint/compare/v1.35.0...v1.35.1) --- updated-dependencies: - dependency-name: yamllint dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0fbcadb8c..ee7a76d2f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -480,13 +480,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "yamllint" -version = "1.35.0" +version = "1.35.1" description = "A linter for YAML files." optional = false python-versions = ">=3.8" files = [ - {file = "yamllint-1.35.0-py3-none-any.whl", hash = "sha256:601b0adaaac6d9bacb16a2e612e7ee8d23caf941ceebf9bfe2cff0f196266004"}, - {file = "yamllint-1.35.0.tar.gz", hash = "sha256:9bc99c3e9fe89b4c6ee26e17aa817cf2d14390de6577cb6e2e6ed5f72120c835"}, + {file = "yamllint-1.35.1-py3-none-any.whl", hash = "sha256:2e16e504bb129ff515b37823b472750b36b6de07963bd74b307341ef5ad8bdc3"}, + {file = "yamllint-1.35.1.tar.gz", hash = "sha256:7a003809f88324fd2c877734f2d575ee7881dd9043360657cc8049c809eba6cd"}, ] [package.dependencies] @@ -499,4 +499,4 @@ dev = ["doc8", "flake8", "flake8-import-order", "rstcheck[sphinx]", "sphinx"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "a31d3e8bd0f0b231d70887305ce919bb9d247df12967e15aed3093159239f69f" +content-hash = "10ee8a0539ff8c11f7e20b8e346daf8d1455783dd47a3e63a7de2b6fc323ddb5" diff --git a/pyproject.toml b/pyproject.toml index 2475690dc..7a1ed8f8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ authors = ["Meltano Team"] python = "^3.9" jsonschema = "^4.17.3" "ruamel.yaml" = "^0.18.6" -yamllint = "^1.35.0" +yamllint = "^1.35.1" requests = "^2.31.0" [tool.poetry.dev-dependencies]