Publish json to swift

cloud-info/ams-wrapper.sh

@@ -9,31 +9,6 @@ GOCDB_ID=$(python -c "from __future__ import print_function; \
                                                 '$GOCDB_SERVICE_TYPE',
                                                 timeout=60)['gocdb_id'], end='')")
 
-if test "$AMS_TOKEN_FILE" != ""; then
-	AMS_TOKEN=$(cat "$AMS_TOKEN_FILE")
-elif test "$HOSTCERT" != "" -a "$HOSTKEY" != ""; then
-	AMS_TOKEN=$(python -c "from argo_ams_library import ArgoMessagingService; \
-			   ams = ArgoMessagingService(endpoint='$AMS_HOST', \
-                                                      project='$AMS_PROJECT', \
-                                                      cert='$HOSTCERT', \
-                                                      key='$HOSTKEY'); \
-                           print(ams.token)")
-fi
-
-if test "$SITE_NAME" = ""; then
-	SITE_NAME="$(yq -r .site.name "$CLOUD_INFO_CONFIG" | tr "." "-")"
-fi
-
-SITE_TOPIC=$(echo "$SITE_NAME" | tr "." "-")
-AMS_TOPIC="SITE_${SITE_TOPIC}_ENDPOINT_${GOCDB_ID}"
-
-# exit if TOPIC is not available.
-curl -f "https://$AMS_HOST/v1/projects/$AMS_PROJECT/topics/$AMS_TOPIC?key=$AMS_TOKEN" >/dev/null 2>&1 ||
-	(
-		echo "Topic $AMS_TOPIC is not avaiable, aborting!"
-		false
-	)
-
 # Attempt to generate the site configuration
 AUTO_CONFIG_PATH="$(mktemp -d)"
 
@@ -73,13 +48,10 @@ else
 		--ignore-share-errors \
 		--format glue21 >cloud-info.out
 	# Produce the json output also
-	RCLONE_CONFIG_S3="$(yq -r '.s3' <"$CHECKIN_SECRETS_FILE")"
-	if test "$RCLONE_CONFIG_S3" != "null"; then
-		cloud-info-provider-service --yaml-file "$CLOUD_INFO_CONFIG" \
-			--middleware "$CLOUD_INFO_MIDDLEWARE" \
-			--ignore-share-errors \
-			--format glue21json >site.json
-	fi
+	cloud-info-provider-service --yaml-file "$CLOUD_INFO_CONFIG" \
+		--middleware "$CLOUD_INFO_MIDDLEWARE" \
+		--ignore-share-errors \
+		--format glue21json >cloud-info.json
 fi
 
 # Fail if there are no shares
@@ -89,32 +61,55 @@ grep -q GLUE2ShareID cloud-info.out ||
 		false
 	)
 
-# Publishing on our own as message is too large for some providers
-ARGO_URL="https://$AMS_HOST/v1/projects/$AMS_PROJECT/topics/$AMS_TOPIC:publish?key=$AMS_TOKEN"
+# Publish to AMS
+if test "$AMS_TOKEN_FILE" != ""; then
+	AMS_TOKEN=$(cat "$AMS_TOKEN_FILE")
+elif test "$HOSTCERT" != "" -a "$HOSTKEY" != ""; then
+	AMS_TOKEN=$(python -c "from argo_ams_library import ArgoMessagingService; \
+			   ams = ArgoMessagingService(endpoint='$AMS_HOST', \
+						      project='$AMS_PROJECT', \
+						      cert='$HOSTCERT', \
+						      key='$HOSTKEY'); \
+			   print(ams.token)")
+fi
+
+if test "$SITE_NAME" = ""; then
+	SITE_NAME="$(yq -r .site.name "$CLOUD_INFO_CONFIG" | tr "." "-")"
+fi
+SITE_TOPIC=$(echo "$SITE_NAME" | tr "." "-")
+AMS_TOPIC="SITE_${SITE_TOPIC}_ENDPOINT_${GOCDB_ID}"
+curl -f "https://$AMS_HOST/v1/projects/$AMS_PROJECT/topics/$AMS_TOPIC?key=$AMS_TOKEN" >/dev/null 2>&1 &&
+	(
+		# Publishing to AMS on our own to ensure message fits
+		ARGO_URL="https://$AMS_HOST/v1/projects/$AMS_PROJECT/topics/$AMS_TOPIC:publish?key=$AMS_TOKEN"
 
-printf '{"messages":[{"attributes":{},"data":"' >ams-payload
-grep -v "UNKNOWN" cloud-info.out | grep -v "^#" | grep -v ": $" | gzip | base64 -w 0 >>ams-payload
-printf '"}]}' >>ams-payload
+		printf '{"messages":[{"attributes":{},"data":"' >ams-payload
+		grep -v "UNKNOWN" cloud-info.out | grep -v "^#" | grep -v ": $" | gzip | base64 -w 0 >>ams-payload
+		printf '"}]}' >>ams-payload
 
-curl -X POST "$ARGO_URL" -H "content-type: application/json" -d @ams-payload
+		curl -X POST "$ARGO_URL" -H "content-type: application/json" -d @ams-payload
+	)
 
-if [ -f site.json ]; then
-	# Put this info into S3, configure rclone config with
-	# a provider named "s3" using env variables
-	export RCLONE_CONFIG_S3_TYPE=s3
-	RCLONE_CONFIG_S3_ACCESS_KEY_ID="$(yq -r '.s3.access_key_id' <"$CHECKIN_SECRETS_FILE")"
-	export RCLONE_CONFIG_S3_ACCESS_KEY_ID
-	RCLONE_CONFIG_S3_SECRET_ACCESS_KEY="$(yq -r '.s3.secret_access_key' <"$CHECKIN_SECRETS_FILE")"
-	export RCLONE_CONFIG_S3_SECRET_ACCESS_KEY
-	RCLONE_CONFIG_S3_ENDPOINT="$(yq -r '.s3.endpoint' <"$CHECKIN_SECRETS_FILE")"
-	export RCLONE_CONFIG_S3_ENDPOINT
-	S3_BUCKET_NAME="$(yq -r '.s3.bucket' <"$CHECKIN_SECRETS_FILE")"
-	export S3_BUCKET_NAME
-	RCLONE_CONFIG_S3_PROVIDER="$(yq -r '.s3.provider' <"$CHECKIN_SECRETS_FILE")"
-	export RCLONE_CONFIG_S3_PROVIDER
-	export RCLONE_CONFIG_S3_ACL=private
-	export RCLONE_CONFIG_S3_NO_CHECK_BUCKET=true
-	rclone copy site.json "s3:$S3_BUCKET_NAME/$SITE_NAME"
+# Publish to object
+if test -s cloud-info.json; then
+	if test "$SWIFT_SITE_NAME" != ""; then
+		OIDC_ACCESS_TOKEN=$(yq -r '.checkin.access_token' <"$ACCESS_TOKEN_FILE")
+		export OIDC_ACCESS_TOKEN
+		export EGI_VO="$SWIFT_VO_NAME"
+		SWIFT_URL=$(fedcloud openstack \
+			--site "$SWIFT_SITE_NAME" \
+			catalog show swift -f json |
+			jq -r '(.endpoints[] | select(.interface=="public")).url')
+		export RCLONE_CONFIG_REMOTE_TYPE="swift"
+		export RCLONE_CONFIG_REMOTE_ENV_AUTH="false"
+		export RCLONE_CONFIG_REMOTE_STORAGE_URL="$SWIFT_URL"
+		eval "$(fedcloud site env --site "$SWIFT_SITE_NAME")"
+		export RCLONE_CONFIG_REMOTE_AUTH_URL="$OS_AUTH_URL"
+		OS_AUTH_TOKEN=$(fedcloud openstack --site "$SWIFT_SITE_NAME" token issue -c id -f value)
+		export RCLONE_CONFIG_REMOTE_AUTH_TOKEN="$OS_AUTH_TOKEN"
+		rclone mkdir "remote:$SWIFT_CONTAINER_NAME"
+		rclone copy cloud-info.json "remote:$SWIFT_CONTAINER_NAME/$SITE_NAME"
+	fi
 fi
 
 rm -rf "$VO_CONFIG_PATH"

deploy/playbook.yaml

@@ -11,3 +11,7 @@
     - role: catchall
       vars:
         site_config_dir: ../sites/
+        swift:
+          site: NCG-INGRID-PT
+          vo: vo.access.egi.eu
+          container: cloud-info

deploy/roles/catchall/defaults/main.yaml

@@ -26,6 +26,12 @@ egi_group: "1999"
 # No site information as default
 sites: []
 
+# Empty swift configuration as default
+swift:
+  site:
+  vo:
+  container:
+
 cloud_info_cron:
   minute: "4,34"
   hour: "*"

deploy/roles/catchall/templates/cloud-info.env.j2

@@ -12,3 +12,6 @@ OS_PROTOCOL={{ site.protocol | default('openid') }}
 OS_REGION={{ site.region }}
 {% endif %}
 SITE_NAME={{ site.gocdb }}
+SWIFT_SITE_NAME={{ swift.site }}
+SWIFT_VO_NAME={{ swift.vo }}
+SWIFT_CONTAINER_NAME={{ swift.container }}

vo-mappings.yaml

@@ -57,7 +57,6 @@ vos:
   vo.enes.org: "urn:mace:egi.eu:group:vo.enes.org:role=vm_operator#aai.egi.eu"
   vo.envrihub.eu: "urn:mace:egi.eu:group:vo.envrihub.eu:role=vm_operator#aai.egi.eu"
   vo.envri-fair.eu: "urn:mace:egi.eu:group:vo.envri-fair.eu:role=vm_operator#aai.egi.eu"
-  vo.environmental.egi.eu: "urn:mace:egi.eu:group:vo.environmental.egi.eu:role=vm_operator#aai.egi.eu"
   vo.europlanet-vespa.eu: "urn:mace:egi.eu:group:vo.europlanet-vespa.eu:role=vm_operator#aai.egi.eu"
   vo.eurosea.marine.ie: "urn:mace:egi.eu:group:vo.eurosea.marine.ie:role=member#aai.egi.eu"
   vo.geoss.eu: "urn:mace:egi.eu:group:vo.geoss.eu:role=vm_operator#aai.egi.eu"