0.9.6 Write index.html and meta-data.json twice

... once with and once without prefix.
Madek · Mar 9, 2018 · b4507ec · b4507ec
1 parent bbee5a2
commit b4507ec
Show file tree

Hide file tree

Showing 8 changed files with 86 additions and 50 deletions.
diff --git a/app/dev/releases.json b/app/dev/releases.json
@@ -1 +1 @@
-[{"version_major":0,"version_minor":9,"version_patch":5,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Download and store the meta-data schema (public vocabularies with meta-keys).\n* Export a index.html file for every entry.\n\nFixes:\n------\n\n* Sort meta-data in the same way as webapp.\n"},{"version_major":0,"version_minor":8,"version_patch":2,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Improve UI and UI when selecting the meta-key prefix.\n\nFixes:\n------\n\n* Fix recursive download of sets \"ArityException\"\n"},{"version_major":0,"version_minor":7,"version_patch":3,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Rename from \"Madek Exporter\" to \"Madek-Exporter\"\n\nFixes\n=====\n\n* Fix sign in as API-client.\n* Extend auth-info with type, show auth-info when connected as ApiClient.\n* Hide secrets by default: do not show \"internal\" connection parameters\n  (password etc) in connection panel.\n"},{"version_major":0,"version_minor":6,"version_patch":0,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Improve the UX of the sign-in page: swapping tags deletes existing sign in info.\n\nFixes\n=====\n\n* Fix sign in with login and password.\n"},{"version_major":0,"version_minor":5,"version_patch":0,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Rename the application from \"Madek APP\" to \"Madek Exporter\".\n* Include release info.\n\nFixes\n=====\n\n* Fix export of meta-data of the Type MetaDatum::TextType.\n* Disconnect on the connection page will now delete the saved token, too.\n* Fix spelling and unnecessary involved language in many places.\n"}]
+[{"version_major":0,"version_minor":9,"version_patch":6,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Write meta-data.json and index.html twice: once with and once without prefix.\n\nFixes:\n------\n\n* Fix build on JDK9.\n"},{"version_major":0,"version_minor":9,"version_patch":5,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Download and store the meta-data schema (public vocabularies with meta-keys).\n* Export a index.html file for every entry.\n\nFixes:\n------\n\n* Sort meta-data in the same way as webapp.\n"},{"version_major":0,"version_minor":8,"version_patch":2,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Improve UI and UI when selecting the meta-key prefix.\n\nFixes:\n------\n\n* Fix recursive download of sets \"ArityException\"\n"},{"version_major":0,"version_minor":7,"version_patch":3,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Rename from \"Madek Exporter\" to \"Madek-Exporter\"\n\nFixes\n=====\n\n* Fix sign in as API-client.\n* Extend auth-info with type, show auth-info when connected as ApiClient.\n* Hide secrets by default: do not show \"internal\" connection parameters\n  (password etc) in connection panel.\n"},{"version_major":0,"version_minor":6,"version_patch":0,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Improve the UX of the sign-in page: swapping tags deletes existing sign in info.\n\nFixes\n=====\n\n* Fix sign in with login and password.\n"},{"version_major":0,"version_minor":5,"version_patch":0,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Rename the application from \"Madek APP\" to \"Madek Exporter\".\n* Include release info.\n\nFixes\n=====\n\n* Fix export of meta-data of the Type MetaDatum::TextType.\n* Disconnect on the connection page will now delete the saved token, too.\n* Fix spelling and unnecessary involved language in many places.\n"}]
diff --git a/app/prod/releases.json b/app/prod/releases.json
@@ -1 +1 @@
-[{"version_major":0,"version_minor":9,"version_patch":5,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Download and store the meta-data schema (public vocabularies with meta-keys).\n* Export a index.html file for every entry.\n\nFixes:\n------\n\n* Sort meta-data in the same way as webapp.\n"},{"version_major":0,"version_minor":8,"version_patch":2,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Improve UI and UI when selecting the meta-key prefix.\n\nFixes:\n------\n\n* Fix recursive download of sets \"ArityException\"\n"},{"version_major":0,"version_minor":7,"version_patch":3,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Rename from \"Madek Exporter\" to \"Madek-Exporter\"\n\nFixes\n=====\n\n* Fix sign in as API-client.\n* Extend auth-info with type, show auth-info when connected as ApiClient.\n* Hide secrets by default: do not show \"internal\" connection parameters\n  (password etc) in connection panel.\n"},{"version_major":0,"version_minor":6,"version_patch":0,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Improve the UX of the sign-in page: swapping tags deletes existing sign in info.\n\nFixes\n=====\n\n* Fix sign in with login and password.\n"},{"version_major":0,"version_minor":5,"version_patch":0,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Rename the application from \"Madek APP\" to \"Madek Exporter\".\n* Include release info.\n\nFixes\n=====\n\n* Fix export of meta-data of the Type MetaDatum::TextType.\n* Disconnect on the connection page will now delete the saved token, too.\n* Fix spelling and unnecessary involved language in many places.\n"}]
+[{"version_major":0,"version_minor":9,"version_patch":6,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Write meta-data.json and index.html twice: once with and once without prefix.\n\nFixes:\n------\n\n* Fix build on JDK9.\n"},{"version_major":0,"version_minor":9,"version_patch":5,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Download and store the meta-data schema (public vocabularies with meta-keys).\n* Export a index.html file for every entry.\n\nFixes:\n------\n\n* Sort meta-data in the same way as webapp.\n"},{"version_major":0,"version_minor":8,"version_patch":2,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Improve UI and UI when selecting the meta-key prefix.\n\nFixes:\n------\n\n* Fix recursive download of sets \"ArityException\"\n"},{"version_major":0,"version_minor":7,"version_patch":3,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Rename from \"Madek Exporter\" to \"Madek-Exporter\"\n\nFixes\n=====\n\n* Fix sign in as API-client.\n* Extend auth-info with type, show auth-info when connected as ApiClient.\n* Hide secrets by default: do not show \"internal\" connection parameters\n  (password etc) in connection panel.\n"},{"version_major":0,"version_minor":6,"version_patch":0,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Improve the UX of the sign-in page: swapping tags deletes existing sign in info.\n\nFixes\n=====\n\n* Fix sign in with login and password.\n"},{"version_major":0,"version_minor":5,"version_patch":0,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Rename the application from \"Madek APP\" to \"Madek Exporter\".\n* Include release info.\n\nFixes\n=====\n\n* Fix export of meta-data of the Type MetaDatum::TextType.\n* Disconnect on the connection page will now delete the saved token, too.\n* Fix spelling and unnecessary involved language in many places.\n"}]
diff --git a/cider-ci.yml b/cider-ci.yml
@@ -50,6 +50,7 @@ jobs:
                   script_key: build-windows
                 build-mac-os has passed:
                   script_key: build-mac-os
+              timeout: 10 Minutes
               body: |
                 set -euxo
                 export PATH=~/.rubies/$RUBY/bin:$PATH

diff --git a/jvm_main/src/madek/app/server/export.clj b/jvm_main/src/madek/app/server/export.clj
@@ -1,6 +1,6 @@
 (ns madek.app.server.export
   (:require
-    [madek.app.server.export.files :as files :refer [download-media-files]]
+    [madek.app.server.export.files :as files :refer [download-media-files path-prefix]]
     [madek.app.server.export.index-html :as index-html]
     [madek.app.server.export.meta-data :as meta-data :refer [meta-data write-meta-data]]
     [madek.app.server.export.meta-data-schema :as meta-data-schema]
@@ -47,28 +47,6 @@
       (nio-path target)
       (make-array java.nio.file.attribute.FileAttribute 0))))
 
-;### title and prefix #########################################################
-
-(defn get-prefix [prefix-meta-key media-resource]
-  (-> media-resource
-      (roa/relation :meta-data)
-      (roa/get {:meta_keys (cheshire/generate-string [(str prefix-meta-key)])})
-      roa/coll-seq
-      first
-      (roa/get {})
-      roa/data
-      :value str))
-
-(defn useableFileName [s]
-  (.replaceAll s "[^a-zA-Z0-9 ]" ""))
-
-(defn path-prefix [prefix-meta-key media-resource]
-  (let [prefix-part-one (if-not (presence prefix-meta-key) ""
-                          (if-let [mk-value (get-prefix
-                                              prefix-meta-key media-resource)]
-                            (str (useableFileName mk-value) "_") ""))]
-    (str prefix-part-one (-> media-resource roa/data :id))))
-
 
 ;### DL Media-Entry ###########################################################
 
@@ -110,10 +88,11 @@
                               :path entry-dir-path
                               :download_started-at (str (time/now))))
              (io/make-parents entry-dir-path)
-             (write-meta-data entry-dir-path meta-data id)
+             (write-meta-data entry-dir-path meta-data id entry-prefix-path)
              (index-html/write entry-dir-path
                                meta-data
-                               (-> media-entry roa/data (assoc :type :media-entry)))
+                               (-> media-entry roa/data (assoc :type :media-entry))
+                               entry-prefix-path)
              (when-not skip-media-files?
                (download-media-files entry-dir-path media-entry))
              (set-item-to-finished id)))))))
@@ -187,10 +166,11 @@
                          :path target-dir-path
                          :download_started-at (str (time/now))))
         (io/make-parents target-dir-path)
-        (write-meta-data target-dir-path meta-data id)
+        (write-meta-data target-dir-path meta-data id path-prefix)
         (index-html/write target-dir-path
                           meta-data
-                          (-> collection roa/data (assoc :type :collection)))
+                          (-> collection roa/data (assoc :type :collection))
+                          path-prefix)
         (download-media-entries-for-set
           id target-dir-path skip-media-files? prefix-meta-key api-entry-point api-http-opts)
         (when recursive?

diff --git a/jvm_main/src/madek/app/server/export/files.clj b/jvm_main/src/madek/app/server/export/files.clj
@@ -1,22 +1,47 @@
 (ns madek.app.server.export.files
   (:require
     [madek.app.server.state :as state]
-    [json-roa.client.core :as roa]
-    [madek.app.server.utils :refer [deep-merge]]
+    [madek.app.server.utils :refer [deep-merge presence]]
 
+    [json-roa.client.core :as roa]
     [cheshire.core :as cheshire]
     [clojure.java.io :as io]
 
     [clj-logging-config.log4j :as logging-config]
     [clojure.tools.logging :as logging]
     [logbug.catcher :as catcher]
+    [logbug.debug :as debug :refer [identity-with-logging I> I>>]]
     [logbug.thrown :as thrown]
     )
 
   (:import
     [java.io File]
     ))
 
+;### title and prefix #########################################################
+
+(defn get-prefix [prefix-meta-key media-resource]
+  (-> media-resource
+      (roa/relation :meta-data)
+      (roa/get {:meta_keys (cheshire/generate-string [(str prefix-meta-key)])})
+      roa/coll-seq
+      first
+      (roa/get {})
+      roa/data
+      :value str))
+
+(defn useableFileName [s]
+  (.replaceAll s "[^a-zA-Z0-9 ]" ""))
+
+(defn path-prefix [prefix-meta-key media-resource]
+  (let [prefix-part-one (if-not (presence prefix-meta-key) ""
+                          (if-let [mk-value (get-prefix
+                                              prefix-meta-key media-resource)]
+                            (str (useableFileName mk-value) "_") ""))]
+    (str prefix-part-one (-> media-resource roa/data :id))))
+
+
+
 ;### DL Previews ##############################################################
 
 (defn download-previews [target-dir media-file item-id media-file-id]
@@ -84,3 +109,10 @@
           )))))
 
 
+;### Debug ####################################################################
+;(logging-config/set-logger! :level :debug)
+;(logging-config/set-logger! :level :info)
+;(debug/debug-ns 'ring.middleware.resource)
+;(debug/debug-ns *ns*)
+;(debug/debug-ns 'json-roa.client.core)
+;(debug/debug-ns 'uritemplate-clj.core)
diff --git a/jvm_main/src/madek/app/server/export/index_html.clj b/jvm_main/src/madek/app/server/export/index_html.clj
@@ -1,5 +1,6 @@
 (ns madek.app.server.export.index-html
   (:require
+    [madek.app.server.export.files :as files :refer [path-prefix]]
     [madek.app.server.state :as state]
     [madek.app.server.utils :refer [deep-merge]]
 
@@ -88,11 +89,12 @@
       [:p "URL: " (url media-resource)]
       (html-meta-data meta-data)]]))
 
-(defn write [target-dir meta-data media-resource]
+(defn write [target-dir meta-data media-resource prefix-path]
   (io/make-parents target-dir)
-  (let [path (str target-dir File/separator "index.html")
-        html (html media-resource meta-data)]
-    (spit path html)))
+  (let [html (html media-resource meta-data)]
+    (doseq [path [(str target-dir File/separator "index.html")
+                  (str target-dir File/separator prefix-path "_index.html")]]
+      (spit path html))))
 
 ;### Debug ####################################################################
 ;(debug/re-apply-last-argument #'write)

diff --git a/jvm_main/src/madek/app/server/export/meta_data.clj b/jvm_main/src/madek/app/server/export/meta_data.clj
@@ -55,21 +55,22 @@
                        meta-datum))
                 meta-data)))
 
-(defn write-meta-data [target-dir meta-data item-id]
-  (let [path (str target-dir File/separator "meta-data.json")]
-    (io/make-parents path)
-    (spit path (cheshire/generate-string meta-data {:pretty true}))
-    (swap! state/db
-           (fn [db params]
-             (deep-merge db params))
-           {:download
-            {:items
-             {item-id
-              {:title (title meta-data)
-               :meta-data
-               {:path path
-                :data meta-data }
-               }}}})))
+(defn write-meta-data [target-dir meta-data item-id prefix-path]
+  (let [content (cheshire/generate-string meta-data {:pretty true})]
+    (doseq [path [(str target-dir File/separator "meta-data.json")
+                  (str target-dir File/separator prefix-path "_meta-data.json")]]
+      (io/make-parents path)
+      (spit path content)
+      (swap! state/db
+             (fn [db params]
+               (deep-merge db params))
+             {:download
+              {:items
+               {item-id
+                {:title (title meta-data)
+                 :meta-data
+                 {:path path
+                  :data meta-data }}}}}))))
 
 ;### Debug ####################################################################
 ;(logging-config/set-logger! :level :debug)

diff --git a/releases.yml b/releases.yml
@@ -1,5 +1,25 @@
 ####### 0.9.x #######################################################################
 
+- version_major: 0
+  version_minor: 9
+  version_patch: 6
+  version_pre: NULL
+  version_build: NULL
+  name: NULL
+
+  description: |
+
+    Changes
+    =======
+
+    * Write meta-data.json and index.html twice: once with and once without prefix.
+
+    Fixes:
+    ------
+
+    * Fix build on JDK9.
+
+
 - version_major: 0
   version_minor: 9
   version_patch: 5
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		[{"version_major":0,"version_minor":9,"version_patch":5,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Download and store the meta-data schema (public vocabularies with meta-keys).\n* Export a index.html file for every entry.\n\nFixes:\n------\n\n* Sort meta-data in the same way as webapp.\n"},{"version_major":0,"version_minor":8,"version_patch":2,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Improve UI and UI when selecting the meta-key prefix.\n\nFixes:\n------\n\n* Fix recursive download of sets \"ArityException\"\n"},{"version_major":0,"version_minor":7,"version_patch":3,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Rename from \"Madek Exporter\" to \"Madek-Exporter\"\n\nFixes\n=====\n\n* Fix sign in as API-client.\n* Extend auth-info with type, show auth-info when connected as ApiClient.\n* Hide secrets by default: do not show \"internal\" connection parameters\n (password etc) in connection panel.\n"},{"version_major":0,"version_minor":6,"version_patch":0,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Improve the UX of the sign-in page: swapping tags deletes existing sign in info.\n\nFixes\n=====\n\n* Fix sign in with login and password.\n"},{"version_major":0,"version_minor":5,"version_patch":0,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Rename the application from \"Madek APP\" to \"Madek Exporter\".\n* Include release info.\n\nFixes\n=====\n\n* Fix export of meta-data of the Type MetaDatum::TextType.\n* Disconnect on the connection page will now delete the saved token, too.\n* Fix spelling and unnecessary involved language in many places.\n"}]
		[{"version_major":0,"version_minor":9,"version_patch":6,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Write meta-data.json and index.html twice: once with and once without prefix.\n\nFixes:\n------\n\n* Fix build on JDK9.\n"},{"version_major":0,"version_minor":9,"version_patch":5,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Download and store the meta-data schema (public vocabularies with meta-keys).\n* Export a index.html file for every entry.\n\nFixes:\n------\n\n* Sort meta-data in the same way as webapp.\n"},{"version_major":0,"version_minor":8,"version_patch":2,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Improve UI and UI when selecting the meta-key prefix.\n\nFixes:\n------\n\n* Fix recursive download of sets \"ArityException\"\n"},{"version_major":0,"version_minor":7,"version_patch":3,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Rename from \"Madek Exporter\" to \"Madek-Exporter\"\n\nFixes\n=====\n\n* Fix sign in as API-client.\n* Extend auth-info with type, show auth-info when connected as ApiClient.\n* Hide secrets by default: do not show \"internal\" connection parameters\n (password etc) in connection panel.\n"},{"version_major":0,"version_minor":6,"version_patch":0,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Improve the UX of the sign-in page: swapping tags deletes existing sign in info.\n\nFixes\n=====\n\n* Fix sign in with login and password.\n"},{"version_major":0,"version_minor":5,"version_patch":0,"version_pre":null,"version_build":null,"name":null,"description":"\nChanges\n=======\n\n* Rename the application from \"Madek APP\" to \"Madek Exporter\".\n* Include release info.\n\nFixes\n=====\n\n* Fix export of meta-data of the Type MetaDatum::TextType.\n* Disconnect on the connection page will now delete the saved token, too.\n* Fix spelling and unnecessary involved language in many places.\n"}]