Configure Galaxy to be able to access the secondary HTCondor cluster

files/galaxy/tpv/destinations.yml.j2

@@ -318,6 +318,8 @@ destinations:
   # LOCAL CONDOR DESTINATIONS #
   #############################
 
+{# Save condor destinations to replicate them in the secondary cluster. #}
+{% set condor %}
   condor_docker:
     inherits: basic_docker_destination
     runner: condor
@@ -430,3 +432,17 @@ destinations:
       GPU_AVAILABLE: 1
     params:
       requirements: 'GalaxyGroup == "compute_gpu"'
+{%- endset %}{{ condor }}
+
+{# Generate secondary cluster destinations. #}
+{% for name, destination in (condor | from_yaml).items() %}
+  {% if destination.runner is defined and destination.runner == "condor" %}
+  secondary_{{ name }}:
+    inherits: {{ name }}
+    runner: condor_secondary
+    scheduling:
+      require:
+        - condor-secondary
+
+  {% endif %}
+{% endfor %}

group_vars/htcondor-secondary-submit-host.yml

@@ -31,7 +31,7 @@
 nspawn_enable: false
 nspawn_start: false
 
 nspawn_ssh: yes
 nspawn_ssh_config_path: /etc/ssh/sshd_config
 nspawn_ssh_config:
   Port: "2222"
@@ -59,8 +59,13 @@
   rsa: "[email protected] AAAAHHNzaC1yc2EtY2VydC12MDFAb3BlbnNzaC5jb20AAAAgQuoZdMdY7Boli6KRlNmUk2zQar66du6kmLR4PkWoHI8AAAADAQABAAABgQDxIgnu5/BIIzVbWM442uzwyuXPQx0jqaVfffjexwewfc2mMNnVAG7mRzcWcgr+o/lFiogkqGulkvO/a+EARs5WsWrsq5bGFcHx9DPgaT+7smu+K3t0WNiqPEykXUJppyMtaYftZSWaCSl51pg3QT29pvl95sjC3URFrQhoaLHJ2fESYJI4WuMBig/rCLaRsF6xaR38LIVfrZdA7UjG5Ncr+FXF6cpNtdvDvJXkGyuYft2p9LMh7vycdiSUL/CluC6tptgQ82JGxgyHDaxwJvyzzDukjKTUkR59ctuGthq9M/M384sCn8Z3PsGE4hkRGdL/HJZhasPIkkwJK8ekucnBR2LzlCqiTrn7HFbndqgePhJWHq0tBN53o2Vmczrbq6UVROAFDCpPKeaWa6hcdqzp0fpHrgSZEyKmMxThV8sj69E/mCh2PEgWjCXsPoJLwrW9TuPUMeXx2+Cwf4/JaFSvLBT19ADFKkzgAPeEVdJg2ZkjWZVO+xxb9e6Ahux+g7EAAAAAAAAAAAAAAAIAAAAVbWFuYWdlci1zZWNvbmRhcnktcnNhAAAAAAAAAABlKAbkAAAAAHfm2ysAAAAAAAAAAAAAAAAAAAEXAAAAB3NzaC1yc2EAAAADAQABAAABAQDLQD6fG38uwFj91GSe6YnRnBuTjXWZN6Pck1JRCTWtufwKV0SZNczD+qUdnFfZrCx/wBVK8R6zL2VWS9hcFK1LuE8HK86f8qG/gcB6yFt/0I/PWoSjcbUMPQTzFIy8yxvdIoPTlj/P6+uNgweTvMFI4+UOuCI71IhB/liTHn1/2dXQM94SFd4VQeg+3Tc6gDxEqRSS6dLIq0uvR8//luIpoW38yh2ozwHmjMKTvHnbduGqHlES4qz9cU9iZkWoPzSp+qoxCOijHvwzL5vD0/k4hZ/iJyTzDHQLDra3Kaa8ykWdERCxjpMp1y9dVQ23lVxp+UUAt3RHOCU1/KuNM9PrAAABFAAAAAxyc2Etc2hhMi01MTIAAAEAOXRiZFs14HScCvi+9Hp51z310t0r3SHtHhE50vEjdAsnej+/yHUtlmaYJ1l4B8SWYHHUV94KxZbQP79D8PYQextXoYldA4Vp29ow33sYF8Mr4p7Jbjyyr4GVP9RAymiZz9cuiiRqjs8qwn/UCOw4A6yaix4k0EYTJa9bpbD3MsCxJbKEdZxxL72kIKf4sGuDrd77hEtRwJFUSqz+XD96TYG+BIomnrk35Oy+oyMGhoIlSM0//cS0pHUW8fpGnRIJJbBjDETwOmaMEF5yynRSeJyWnEBPpjcA+1i7wILSxYxZb5ckGxCABYxNvlIhF8Bvdcyb4eo7B/WeEdzr5C+Pzg== /etc/ssh/ssh_host_rsa_key.pub"
 nspawn_ssh_authorized_keys:
   - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDV7gfNbNN5O8vH6/tM/iOFXKBP2YKRHXOmdfV8ogvu9BdVV0IPmDzk2EooVpThDE1VMv1hz3811tvBhHRJ6IgNhVIV/61w/+RazQD/AU27X8bX+Hb9EQ/bP4DW+6ySd/z5vdDLzpH5dbiMhzPEDkXVsylUT+hkQnas6cHspDhHmtKQ5MWOgDe3D/IEudTDJQe8hxxaU4TaZUmFzn7eYp9HvuK8qW0yCy4NWOxJJHA+G5wSCyLuKnaKo4AitUIzSKF1AB94oq7b96KONhPxgRptAk4OYIUTdNFbrI5HDaSNzHLnF5FbjQvG+Eu6m5nY5yvJMogE+jiuWeIXCZTCFljg287FUo0ohmbZpd802L6VXun14VumRC+rRgPrvBALo/CsyCsPIoBSTKhVElxKVOcRjmTLNfrUZM0GQxqJhIvah8BV+JTExkipPwkrKTdMAWIXvCoehxV+WMpBWqtEEzAzEoqJpaiec7HfriwsHTGESZWAPYEbFjzbHXQZtqBkbOvtokPMRmTWfWKxaplCMN6ddJeeY6faorD0w/e6lszWES1Q1ieajiPKDy37UvybKKvPTk4o3MzyzYOS4c8HQj+jnGeR5Q3ETuyz4psLyOfuBtIrfOeuxV42rFDmkYM3IrrRR+F9oklFG6Ig8DVfgQEzSG36NkgvpF4OdFvigYqXvw== cloud@vgcn"
 nspawn_ssh_host_trust_container: yes
 
+nspawn_condor_systemd_run: "/usr/bin/systemd-run --uid={{ galaxy_user.uid }} --gid={{ galaxy_group.gid }} --pipe --quiet --machine {{ nspawn_name }}"
+nspawn_condor_rm_command: "{{ nspawn_condor_systemd_run }} /usr/bin/condor_rm"
+nspawn_condor_ssh_to_job_command: "{{ nspawn_condor_systemd_run }} /usr/bin/condor_ssh_to_job"
+nspawn_condor_submit_command: "{{ nspawn_condor_systemd_run }} /usr/bin/condor_submit"
+
 ssh_allow_tcp_forwarding: "local"
 sshd_custom_options:
   - "PermitOpen 127.0.0.1:{{ nspawn_ssh_config.Port }}"

htcondor.yml

@@ -177,6 +177,21 @@
         key: "[127.0.0.1]:{{ nspawn_ssh_config.Port }} {{ nspawn_ssh_host_key.content | b64decode }}"
       when: nspawn_ssh_host_trust_container
 
+    - name: Allow the Galaxy user to run HTCondor commands in the container.
+      # Uses /etc/sudoers. Ideally this would be solved using what is requested
+      # in this issue https://github.com/systemd/systemd/issues/10997, but the
+      # issue is still open.
+      community.general.sudoers:
+        name: htcondor-nspawn
+        user: "{{ galaxy_user.name }}"
+        nopassword: true
+        validation: required
+        setenv: true
+        commands:
+          - "{{ nspawn_condor_rm_command }} *"
+          - "{{ nspawn_condor_ssh_to_job_command }} *"
+          - "{{ nspawn_condor_submit_command }} *"
+
 - name: HTCondor cluster.
   hosts: htcondor:!sn06.galaxyproject.eu
   handlers:

templates/galaxy/config/job_conf.yml

@@ -15,6 +15,12 @@ galaxy_jobconf:
     #workers: 3
     - id: condor
       load: galaxy.jobs.runners.condor:CondorJobRunner
+    - id: condor_secondary
+      load: galaxy.jobs.runners.condor:CondorJobRunner
+      params:
+        condor_rm_cmd: "sudo {{ nspawn_condor_rm_command }}"
+        condor_ssh_to_job_cmd: "sudo {{ nspawn_condor_ssh_to_job_command }}"
+        condor_submit_cmd: "sudo {{ nspawn_condor_submit_command }}"
     - id: local
       load: galaxy.jobs.runners.local:LocalJobRunner
     - id: pulsar_embedded

templates/galaxy/config/job_conf.yml.j2

@@ -136,3 +136,17 @@ limits:
     window: {{ limit['window'] }}
 {% endif %}
 {% endfor %}
+{# Replicate destination limits for the secondary HTCondor cluster #}
+{% for limit in galaxy_jobconf['limits'] | sort(attribute='type') %}
+{% if limit['type'].startswith('destination_') and 'id' in limit
+and limit['id'].startswith('condor_') %}
+  - type: {{ limit['type'] }}
+    value: {{ limit['value'] }}
+    id: secondary_{{ limit['id'] }}
+{% if 'tag' in limit %}
+    tag: {{ limit['tag'] }}
+{% elif 'window' in limit %}
+    window: {{ limit['window'] }}
+{% endif %}
+{% endif %}
+{% endfor %}