Skip to content

Commit

Permalink
mulled: also consider non strict channel priority
Browse files Browse the repository at this point in the history
for the creation of conda environments we try

1. strict channel priotity
2. if this fails try without

https://github.com/galaxyproject/galaxy/blob/5f484fca6255fbbc8a66016601c3fcab20271c81/lib/galaxy/tool_util/deps/conda_util.py#L282

For the creation of mulled containers we don't do this.
I stumbled over this a few times when creating containers
for older tools, where a bioconda package was used that
is now in conda-forge (and with strict priority only the
newer conda-forge packages can be installed)

Also restructured slightly. Before the `build` was appended
to the `involucro_args` early. Therefore some inserts
were needed in the middle of the list.

Fix verbose option which was ignored before.
  • Loading branch information
bernt-matthias committed Jan 21, 2025
1 parent 5f484fc commit 6b3bc4f
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 13 deletions.
10 changes: 9 additions & 1 deletion lib/galaxy/tool_util/deps/mulled/invfile.lua
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,13 @@ for i = 1, #channels do
channel_args = channel_args .. " -c '" .. channels[i] .. "'"
end

local strict_channel_priority = VAR.STRICT_CHANNEL_PRIORITY
if strict_channel_priority == '' then
strict_channel_priority = ''
else
strict_channel_priority = '--strict-channel-priority'
end

local target_args = ''
local targets = VAR.TARGETS:split(",")
for i = 1, #targets do
Expand Down Expand Up @@ -90,8 +97,9 @@ inv.task('build')
.run('/bin/sh', '-c', preinstall
.. conda_bin .. ' install '
.. channel_args .. ' '
.. strict_channel_priority .. ' '
.. target_args
.. ' --strict-channel-priority -p /usr/local --copy --yes '
.. ' -p /usr/local --copy --yes '
.. verbose
.. postinstall)
.wrap('build/dist')
Expand Down
44 changes: 32 additions & 12 deletions lib/galaxy/tool_util/deps/mulled/mulled_build.py
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,6 @@ def mull_targets(
conda_install = f"""conda install {verbose} --yes {" ".join(f"'{spec}'" for spec in specs)}"""
involucro_args.extend(["-set", f"PREINSTALL=if {mamba_test} ; then {conda_install} ; fi"])

involucro_args.append(command)
if test_files:
test_bind = []
for test_file in test_files:
Expand All @@ -333,12 +332,23 @@ def mull_targets(
if os.path.exists(test_file.split(":")[0]):
test_bind.append(test_file)
if test_bind:
involucro_args.insert(6, "-set")
involucro_args.insert(7, f"TEST_BINDS={','.join(test_bind)}")
cmd = involucro_context.build_command(involucro_args)
print(f"Executing: {shlex_join(cmd)}")
involucro_args.append("-set")
involucro_args.append(f"TEST_BINDS={','.join(test_bind)}")

involucro_args_list = []
for strict_channel_priority in [True, False]:
involucro_args_list.append(involucro_args[:])
if strict_channel_priority:
involucro_args_list[-1].extend(["-set", "STRICT_CHANNEL_PRIORITY=1"])

for involucro_args in involucro_args_list:
involucro_args.append(command)

if dry_run:
cmd = involucro_context.build_command(involucro_args_list[0])
print(f"Executing: {shlex_join(cmd)}")
return 0

ensure_installed(involucro_context, True)
if singularity:
if not os.path.exists(singularity_image_dir):
Expand All @@ -349,12 +359,20 @@ def mull_targets(
"base_image": dest_base_image or DEFAULT_BASE_IMAGE,
}
sin_def.write(fill_template)
with PrintProgress():
ret = involucro_context.exec_command(involucro_args)
if singularity:
# we can not remove this folder as it contains the image wich is owned by root
pass
# shutil.rmtree('./singularity_import')

for involucro_args in involucro_args_list:
cmd = involucro_context.build_command(involucro_args)
print(f"Executing: {shlex_join(cmd)}")

with PrintProgress():
ret = involucro_context.exec_command(involucro_args)
log.error(f"{ret=}")
if singularity:
# we can not remove this folder as it contains the image wich is owned by root
pass
# shutil.rmtree('./singularity_import')
if not ret:
break
return ret


Expand Down Expand Up @@ -407,7 +425,7 @@ def __init__(self, involucro_bin=None, shell_exec=None, verbose="3"):
def build_command(self, involucro_args):
return [self.involucro_bin, f"-v={self.verbose}"] + involucro_args

def exec_command(self, involucro_args):
def exec_command(self, involucro_args) -> int:
cmd = self.build_command(involucro_args)
# Create ./build dir manually, otherwise Docker will do it as root
created_build_dir = False
Expand Down Expand Up @@ -585,6 +603,8 @@ def args_to_mull_targets_kwds(args):
kwds["singularity_image_dir"] = args.singularity_image_dir
if hasattr(args, "invfile"):
kwds["invfile"] = args.invfile
if hasattr(args, "verbose"):
kwds["verbose"] = args.verbose

kwds["involucro_context"] = context_from_args(args)

Expand Down

0 comments on commit 6b3bc4f

Please sign in to comment.