Merge branch '071724/add-missing-data-security-mode'

mach-kernel · Jul 17, 2024 · b047c00 · b047c00
2 parents c42ec67 + fedc4e9
commit b047c00
Show file tree

Hide file tree

Showing 7 changed files with 45 additions and 27 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/charts/databricks-kube-operator/Chart.yaml b/charts/databricks-kube-operator/Chart.yaml
@@ -1,8 +1,8 @@
 apiVersion: v2
-appVersion: 0.8.1
+appVersion: 0.8.2
 name: databricks-kube-operator
 description: A kube-rs operator for managing Databricks API resources
-version: 0.9.2
+version: 0.9.3
 
 home: https://github.com/mach-kernel/databricks-kube-operator
 sources:

diff --git a/databricks-kube/Cargo.toml b/databricks-kube/Cargo.toml
@@ -5,7 +5,7 @@ path = "src/crdgen.rs"
 [package]
 name = "databricks_kube"
 default-run = "databricks_kube"
-version = "0.8.1"
+version = "0.8.2"
 edition = "2021"
 
 [dependencies]

diff --git a/databricks-rust-jobs/Cargo.toml b/databricks-rust-jobs/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "databricks_rust_jobs"
-version = "2.1.1"
+version = "2.1.2"
 authors = ["OpenAPI Generator team and contributors"]
 description = "The Jobs API allows you to create, edit, and delete jobs. You should never hard code secrets or store them in plain text. Use the [Secrets API](https://docs.databricks.com/dev-tools/api/latest/secrets.html) to manage secrets in the [Databricks CLI](https://docs.databricks.com/dev-tools/cli/index.html). Use the [Secrets utility](https://docs.databricks.com/dev-tools/databricks-utils.html#dbutils-secrets) to reference secrets in notebooks and jobs."
 # Override this license by providing a License Object in the OpenAPI.

diff --git a/databricks-rust-jobs/src/models/new_cluster.rs b/databricks-rust-jobs/src/models/new_cluster.rs
@@ -118,6 +118,8 @@ pub enum DataSecurityMode {
     LegacyPassthrough,
     #[serde(rename = "LEGACY_SINGLE_USER")]
     LegacySingleUser,
+    #[serde(rename = "LEGACY_SINGLE_USER_STANDARD")]
+    LegacySingleUserStandard,
 }
 
 impl Default for DataSecurityMode {

diff --git a/databricks-rust-jobs/src/models/new_task_cluster.rs b/databricks-rust-jobs/src/models/new_task_cluster.rs
@@ -120,6 +120,8 @@ pub enum DataSecurityMode {
     LegacyPassthrough,
     #[serde(rename = "LEGACY_SINGLE_USER")]
     LegacySingleUser,
+    #[serde(rename = "LEGACY_SINGLE_USER_STANDARD")]
+    LegacySingleUserStandard,
 }
 
 impl Default for DataSecurityMode {

diff --git a/openapi/openapi-helper.bb b/openapi/openapi-helper.bb
@@ -47,18 +47,36 @@
     :else
     form))
 
-(defn emit-components
+(defn clean-databricks
+  [form]
+  (if (map? form)
+    (let [to-remove (filter #(str/starts-with? % "x-databricks")
+                            (keys form))]
+      (apply dissoc form to-remove))
+    form))
+
+(defn select-refs
   "Collect referenced schema/parameter/response refs from OpenAPI components"
   [cs components]
-  (let [filtered-components (->> cs
-                                 ; #/components/kind/name split /
-                                 (map #(str/split % #"/"))
-                                 ; drop #, components
-                                 (map #(map keyword (drop 2 %)))
-                                 (map last))] 
-    {:schemas    (select-keys (:schemas components) filtered-components)
-     :parameters (select-keys (:parameters components) filtered-components)
-     :responses  (select-keys (:responses components) filtered-components)}))
+  (let [component-keys (->> cs
+                            ; #/components/kind/name split /
+                            (map #(str/split % #"/"))
+                            ; drop #, components
+                            (map #(map keyword (drop 2 %)))
+                            (map last))]
+    {:schemas    (select-keys (:schemas components) component-keys)
+     :parameters (select-keys (:parameters components) component-keys)
+     :responses  (select-keys (:responses components) component-keys)}))
+
+(defn emit-components-for
+  ([all form]
+   (emit-components-for all form (set (walk/postwalk collect-refs form))))
+  ([all form refs]
+   (let [new-form (select-refs refs all)
+         new-refs (set (walk/postwalk collect-refs new-form))]
+     (if (> (count new-refs) (count refs))
+       (emit-components-for all form new-refs)
+       new-form))))
 
 (defn emit-subspec
   "Emit a subset of the larger spec"
@@ -68,19 +86,12 @@
         paths (if f
                 (into {} (keep #(emit-matching f %)) paths)
                 paths)
-        path-component-refs (set (walk/postwalk collect-refs paths))
-        path-components (emit-components path-component-refs components)
 
-        component-refs (set (walk/postwalk collect-refs path-components))
-        component-components (emit-components component-refs components)
-
-        components (merge-with merge path-components component-components)]
+        components (emit-components-for components paths)]
     (-> parsed-spec
         (assoc :components components)
         (assoc :paths paths)
-        (dissoc :tags)
-        (dissoc :x-databricks-groups)
-        (walk/stringify-keys))))
+        (dissoc :tags))))
 
 (defn -main
   [args]
@@ -93,8 +104,11 @@
     (when-not spec
       (println "Usage\n" (cli/format-opts cli-spec))
       (System/exit 1))
-
-    (let [new-spec (walk/postwalk repascalize (emit-subspec filter spec))]
+
+    (let [new-spec (->> (emit-subspec filter spec)
+                        (walk/stringify-keys)
+                        (walk/postwalk repascalize)
+                        (walk/postwalk clean-databricks))]
       (println (json/encode new-spec {:pretty true})))))
 
 (-main *command-line-args*)