Skip to content

Commit

Permalink
Merge pull request #7 from rail-berkeley/add-local-to-easy-launch
Browse files Browse the repository at this point in the history 
Add local and here_no_doodad modes to easy launch
  • Loading branch information
@Jendker
Jendker authored Apr 22, 2021
2 parents 24df365 + 6ddf3bb commit bbf305e
Show file tree
Hide file tree
Showing 2 changed files with 113 additions and 102 deletions.
33 changes: 12 additions & 21 deletions doodad/wrappers/easy_launch/run_experiment.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import pickle
import base64
import argparse
from pathlib import Path


ARGS_DATA = 'DOODAD_ARGS_DATA'
Expand Down Expand Up @@ -101,32 +102,22 @@ def encode_args(call_args, cloudpickle=False):
if run_mode == 'azure':
try:
import urllib.request
request = urllib.request.Request(
"http://169.254.169.254/metadata/instance?api-version=2020-06-01"
)
request.add_header("Metadata", "true")
azure_metadata = urllib.request.urlopen(request).read().decode()
doodad_config.extra_launch_info['azure_metadata2'] = (
azure_metadata
)
except Exception as e:
print("Could not get Azure instance metadata 2. Error was...")
print(e)
try:
import urllib.request
import json
request = urllib.request.Request(
"http://169.254.169.254/metadata/instance?api-version=2020-06-01"
)
request.add_header("Metadata", True)
azure_metadata = urllib.request.urlopen(request).read().decode()
doodad_config.extra_launch_info['azure_metadata3'] = (
azure_metadata
azure_metadata = json.loads(
urllib.request.urlopen(request).read().decode()
)
doodad_config.extra_launch_info['azure_resource_group_name'] = (
azure_metadata['compute']['resourceGroupName']
)
except Exception as e:
print("Could not get Azure instance metadata 3. Error was...")
print("Could not get Azure instance metadata. Error was...")
print(e)
doodad_config = doodad_config._replace(
output_directory=output_dir,
)

doodad_config = doodad_config._replace(
output_directory=output_dir,
)
Path(output_dir).mkdir(parents=True, exist_ok=True)
method_call(doodad_config, variant)
182 changes: 101 additions & 81 deletions doodad/wrappers/easy_launch/sweep_function.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,24 +24,42 @@ def sweep_function(
use_gpu=False,
gpu_id=0,
name_runs_by_id=True,
add_time_to_run_id=True,
start_run_id=0
):
"""
Usage:
```
def foo(doodad_config, variant):
def function(doodad_config, variant):
x = variant['x']
y = variant['y']
with open(doodad_config.output_directory + '/foo_output.txt', "w") as f:
f.write('sum = %f' % x + y)
z = variant['z']
with open(doodad_config.output_directory + '/function_output.txt', "w") as f:
f.write('sum = {}'.format(x+y+z))
params = {
params_to_sweep = {
'x': [1, 4],
'y': [3, 4],
}
sweep_function(foo, variant, log_path='my-experiment')
default_params = {
'z': 10,
}
sweep_function(
function,
params_to_sweep,
default_params=default_params,
log_path='my-experiment',
mode='azure',
)
```
:param method_call: A function
:param mode: Mode to run this in. Supported modes:
- 'azure'
- 'here_no_doodad'
- 'local'
- 'gcp' (untested)
:param method_call: A function that takes in two parameters:
doodad_config: metadata.DoodadConfig
variant: dictionary
:param params:
:param log_path:
:param name_runs_by_id: If true, then each run will be in its own
Expand All @@ -53,16 +71,17 @@ def foo(doodad_config, variant):
run2/
...
```
otherwise, all the runs will be saved to `log_path/` directly (possibly with
some collision-logic that's mode-dependent).
otherwise, all the runs will be saved to `log_path/` directly. The way path
collisions are handled depends on the mode.
:param add_time_to_run_id: If true, append the time to the run id name
:param start_run_id:
:return: How many
"""
if add_date_to_logname:
datestamp = time.strftime("%y-%m-%d")
log_path = '%s_%s' % (datestamp, log_path)
target = osp.join(REPO_DIR, 'doodad/wrappers/easy_launch/run_experiment.py')
sweeper, output_mount = _create_sweeper_and_output_mount(mode)
sweeper, output_mount = _create_sweeper_and_output_mount(mode, log_path)
git_infos = metadata.generate_git_infos()

doodad_config = metadata.DoodadConfig(
Expand All @@ -74,89 +93,97 @@ def foo(doodad_config, variant):
extra_launch_info={},
)

def _create_final_log_path(base_path, run_id):
if name_runs_by_id:
path_suffix = '/run{}'.format(start_run_id + run_id)
if add_time_to_run_id:
timestamp = time.strftime("%Hh-%Mm-%Ss")
path_suffix += '_{}'.format(timestamp)
else:
path_suffix = ''
return base_path + path_suffix

if mode == 'here_no_doodad':
return run_method_here_no_doodad(
return _run_method_here_no_doodad(
method_call,
doodad_config,
params,
default_params,
log_path,
_create_final_log_path,
)

def postprocess_config_and_run_mode(config, run_mode, config_idx):
if name_runs_by_id:
path_suffix = '/run{}'.format(start_run_id + config_idx)
else:
path_suffix = ''
new_log_path = log_path + path_suffix
new_log_path = _create_final_log_path(log_path, config_idx)
args = {
'method_call': method_call,
'output_dir': output_mount.mount_point,
'doodad_config': doodad_config,
'variant': config,
'mode': mode,
}
if isinstance(run_mode, ddmode.AzureMode):
run_mode.log_path = new_log_path
if isinstance(run_mode, ddmode.GCPMode):
run_mode.gcp_log_path = new_log_path
if isinstance(run_mode, ddmode.LocalMode):
args['output_dir'] = _create_final_log_path(
args['output_dir'], config_idx
)
args_encoded, cp_version = run_experiment.encode_args(args, cloudpickle=use_cloudpickle)
new_config = {
run_experiment.ARGS_DATA: args_encoded,
run_experiment.USE_CLOUDPICKLE: str(int(use_cloudpickle)),
run_experiment.CLOUDPICKLE_VERSION :cp_version,
}
if isinstance(run_mode, ddmode.AzureMode):
run_mode.log_path = new_log_path
if isinstance(run_mode, ddmode.GCPMode):
run_mode.gcp_log_path = new_log_path
if isinstance(run_mode, ddmode.EC2Mode):
run_mode.s3_log_path = new_log_path
return new_config, run_mode

run_sweep(default_params, log_path, mode, params,
postprocess_config_and_run_mode, sweeper, target)
def _run_sweep():
if mode == 'azure':
sweeper.run_sweep_azure(
target,
params,
default_params=default_params,
log_path=log_path,
add_date_to_logname=False,
postprocess_config_and_run_mode=postprocess_config_and_run_mode,
)
elif mode == 'gcp':
sweeper.run_sweep_gcp(
target,
params,
default_params=default_params,
log_prefix=log_path,
add_date_to_logname=False,
postprocess_config_and_run_mode=postprocess_config_and_run_mode,
)
elif mode == 'local':
sweeper.run_sweep_local(
target,
params,
default_params=default_params,
postprocess_config_and_run_mode=postprocess_config_and_run_mode,
)
else:
raise ValueError('Unknown mode: {}'.format(mode))

_run_sweep()


def run_method_here_no_doodad(
method_call, doodad_config, params, default_params, log_path
def _run_method_here_no_doodad(
method_call, doodad_config, params, default_params, log_path,
create_final_log_path
):
sweeper = Sweeper(params, default_params)
for xid, config in enumerate(sweeper):
new_log_path = create_final_log_path(log_path, xid)
doodad_config = doodad_config._replace(
output_directory=osp.join(LOCAL_LOG_DIR, log_path, 'run{}'.format(xid)),
output_directory=osp.join(LOCAL_LOG_DIR, new_log_path),
)
save_doodad_config(doodad_config)
method_call(doodad_config, config)


def run_sweep(default_params, log_path, mode, params,
postprocess_config_and_run_mode, sweeper, target):
if mode == 'azure':
sweeper.run_sweep_azure(
target,
params,
default_params=default_params,
log_path=log_path,
add_date_to_logname=False,
postprocess_config_and_run_mode=postprocess_config_and_run_mode,
)
elif mode == 'gcp':
sweeper.run_sweep_gcp(
target,
params,
default_params=default_params,
log_prefix=log_path,
add_date_to_logname=False,
postprocess_config_and_run_mode=postprocess_config_and_run_mode,
)
elif mode == 'local':
sweeper.run_sweep_local(
target,
params,
default_params=default_params,
postprocess_config_and_run_mode=postprocess_config_and_run_mode,
)
else:
raise ValueError('Unknown mode: {}'.format(mode))


def _create_mounts():
NON_CODE_MOUNTS = [
doodad.MountLocal(**non_code_mapping)
Expand All @@ -172,7 +199,7 @@ def _create_mounts():
return mounts


def _create_sweeper_and_output_mount(mode):
def _create_sweeper_and_output_mount(mode, log_path):
mounts = _create_mounts()
az_mount = doodad.MountAzure(
'',
Expand All @@ -188,7 +215,7 @@ def _create_sweeper_and_output_mount(mode):
azure_tenant_id=AZ_TENANT_ID,
azure_storage_container=AZ_CONTAINER,
mount_out_azure=az_mount,
local_output_dir=LOCAL_LOG_DIR,
local_output_dir=osp.join(LOCAL_LOG_DIR, log_path), # TODO: how to make this vary in local mode?
)
# TODO: the sweeper should probably only have one output mount that is
# set rather than read based on the mode
Expand All @@ -205,39 +232,32 @@ def _create_sweeper_and_output_mount(mode):
return sweeper, output_mount


def foo(doodad_config, variant):
x = variant['x']
y = variant['y']
print("sum", x+y)
print("OUTPUT_DIR_IS", doodad_config.output_directory)
with open(doodad_config.output_directory + '/foo_output.txt', "w") as f:
f.write('sum = {}\n'.format(x + y))

from doodad.wrappers.easy_launch.metadata import save_doodad_config
save_doodad_config(doodad_config)


def function(doodad_config, variant):
def example_function(doodad_config, variant):
x = variant['x']
y = variant['y']
z = variant['z']
with open(doodad_config.output_directory + '/function_output.txt', "w") as f:
f.write('sum = {}'.format(x+y+z))
save_doodad_config(doodad_config)


if __name__ == "__main__":
params_to_sweep = {
'x': [1, 4],
'y': [3, 4],
'y': [100],
}
default_params = {
'z': 10,
}
sweep_function(
function,
params_to_sweep,
default_params=default_params,
log_path='my-experiment',
mode='azure',
# mode='here_no_doodad', # this should also work
)
for mode in [
'here_no_doodad',
'local',
'azure',
]:
sweep_function(
example_function,
params_to_sweep,
default_params=default_params,
log_path='test_easy_launch_{}_mode'.format(mode),
mode=mode,
)

0 comments on commit bbf305e

Please sign in to comment.