From 9a083f47ff3189991aad08736f111d4ecbb7df1c Mon Sep 17 00:00:00 2001 From: David Stancu Date: Wed, 17 Jul 2024 12:00:33 -0400 Subject: [PATCH 1/3] improve openapi-helper a little bit --- openapi/openapi-helper.bb | 56 ++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/openapi/openapi-helper.bb b/openapi/openapi-helper.bb index 1c5e077..10bfd97 100755 --- a/openapi/openapi-helper.bb +++ b/openapi/openapi-helper.bb @@ -47,18 +47,36 @@ :else form)) -(defn emit-components +(defn clean-databricks + [form] + (if (map? form) + (let [to-remove (filter #(str/starts-with? % "x-databricks") + (keys form))] + (apply dissoc form to-remove)) + form)) + +(defn select-refs "Collect referenced schema/parameter/response refs from OpenAPI components" [cs components] - (let [filtered-components (->> cs - ; #/components/kind/name split / - (map #(str/split % #"/")) - ; drop #, components - (map #(map keyword (drop 2 %))) - (map last))] - {:schemas (select-keys (:schemas components) filtered-components) - :parameters (select-keys (:parameters components) filtered-components) - :responses (select-keys (:responses components) filtered-components)})) + (let [component-keys (->> cs + ; #/components/kind/name split / + (map #(str/split % #"/")) + ; drop #, components + (map #(map keyword (drop 2 %))) + (map last))] + {:schemas (select-keys (:schemas components) component-keys) + :parameters (select-keys (:parameters components) component-keys) + :responses (select-keys (:responses components) component-keys)})) + +(defn emit-components-for + ([all form] + (emit-components-for all form (set (walk/postwalk collect-refs form)))) + ([all form refs] + (let [new-form (select-refs refs all) + new-refs (set (walk/postwalk collect-refs new-form))] + (if (> (count new-refs) (count refs)) + (emit-components-for all form new-refs) + new-form)))) (defn emit-subspec "Emit a subset of the larger spec" @@ -68,19 +86,12 @@ paths (if f (into {} (keep #(emit-matching f %)) paths) paths) - path-component-refs (set (walk/postwalk collect-refs paths)) - path-components (emit-components path-component-refs components) - component-refs (set (walk/postwalk collect-refs path-components)) - component-components (emit-components component-refs components) - - components (merge-with merge path-components component-components)] + components (emit-components-for components paths)] (-> parsed-spec (assoc :components components) (assoc :paths paths) - (dissoc :tags) - (dissoc :x-databricks-groups) - (walk/stringify-keys)))) + (dissoc :tags)))) (defn -main [args] @@ -93,8 +104,11 @@ (when-not spec (println "Usage\n" (cli/format-opts cli-spec)) (System/exit 1)) - - (let [new-spec (walk/postwalk repascalize (emit-subspec filter spec))] + + (let [new-spec (->> (emit-subspec filter spec) + (walk/stringify-keys) + (walk/postwalk repascalize) + (walk/postwalk clean-databricks))] (println (json/encode new-spec {:pretty true}))))) (-main *command-line-args*) From 133b38ad5e0682a9a9d5de0b5da80a37abbec41c Mon Sep 17 00:00:00 2001 From: David Stancu Date: Wed, 17 Jul 2024 12:02:37 -0400 Subject: [PATCH 2/3] enum was missing from openapi spec, add manually and bump versions --- Cargo.lock | 4 ++-- charts/databricks-kube-operator/Chart.yaml | 4 ++-- databricks-kube/Cargo.toml | 2 +- databricks-rust-jobs/Cargo.toml | 2 +- databricks-rust-jobs/src/models/new_task_cluster.rs | 2 ++ 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 12003c2..1be7ff0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -414,7 +414,7 @@ dependencies = [ [[package]] name = "databricks_kube" -version = "0.8.1" +version = "0.8.2" dependencies = [ "assert-json-diff", "async-stream", @@ -463,7 +463,7 @@ dependencies = [ [[package]] name = "databricks_rust_jobs" -version = "2.1.1" +version = "2.1.2" dependencies = [ "reqwest 0.12.5", "schemars", diff --git a/charts/databricks-kube-operator/Chart.yaml b/charts/databricks-kube-operator/Chart.yaml index 8b81137..67af2ac 100644 --- a/charts/databricks-kube-operator/Chart.yaml +++ b/charts/databricks-kube-operator/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 -appVersion: 0.8.1 +appVersion: 0.8.2 name: databricks-kube-operator description: A kube-rs operator for managing Databricks API resources -version: 0.9.2 +version: 0.9.3 home: https://github.com/mach-kernel/databricks-kube-operator sources: diff --git a/databricks-kube/Cargo.toml b/databricks-kube/Cargo.toml index 1187890..e500b03 100644 --- a/databricks-kube/Cargo.toml +++ b/databricks-kube/Cargo.toml @@ -5,7 +5,7 @@ path = "src/crdgen.rs" [package] name = "databricks_kube" default-run = "databricks_kube" -version = "0.8.1" +version = "0.8.2" edition = "2021" [dependencies] diff --git a/databricks-rust-jobs/Cargo.toml b/databricks-rust-jobs/Cargo.toml index 62abac7..88b8889 100644 --- a/databricks-rust-jobs/Cargo.toml +++ b/databricks-rust-jobs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "databricks_rust_jobs" -version = "2.1.1" +version = "2.1.2" authors = ["OpenAPI Generator team and contributors"] description = "The Jobs API allows you to create, edit, and delete jobs. You should never hard code secrets or store them in plain text. Use the [Secrets API](https://docs.databricks.com/dev-tools/api/latest/secrets.html) to manage secrets in the [Databricks CLI](https://docs.databricks.com/dev-tools/cli/index.html). Use the [Secrets utility](https://docs.databricks.com/dev-tools/databricks-utils.html#dbutils-secrets) to reference secrets in notebooks and jobs." # Override this license by providing a License Object in the OpenAPI. diff --git a/databricks-rust-jobs/src/models/new_task_cluster.rs b/databricks-rust-jobs/src/models/new_task_cluster.rs index a2f2167..98ce908 100644 --- a/databricks-rust-jobs/src/models/new_task_cluster.rs +++ b/databricks-rust-jobs/src/models/new_task_cluster.rs @@ -120,6 +120,8 @@ pub enum DataSecurityMode { LegacyPassthrough, #[serde(rename = "LEGACY_SINGLE_USER")] LegacySingleUser, + #[serde(rename = "LEGACY_SINGLE_USER_STANDARD")] + LegacySingleUserStandard, } impl Default for DataSecurityMode { From fedc4e9c6077810a130b20e5e053cd8079970389 Mon Sep 17 00:00:00 2001 From: David Stancu Date: Wed, 17 Jul 2024 12:05:03 -0400 Subject: [PATCH 3/3] also add to new_cluster model --- databricks-rust-jobs/src/models/new_cluster.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/databricks-rust-jobs/src/models/new_cluster.rs b/databricks-rust-jobs/src/models/new_cluster.rs index 64bf978..29b1283 100644 --- a/databricks-rust-jobs/src/models/new_cluster.rs +++ b/databricks-rust-jobs/src/models/new_cluster.rs @@ -118,6 +118,8 @@ pub enum DataSecurityMode { LegacyPassthrough, #[serde(rename = "LEGACY_SINGLE_USER")] LegacySingleUser, + #[serde(rename = "LEGACY_SINGLE_USER_STANDARD")] + LegacySingleUserStandard, } impl Default for DataSecurityMode {