Merge pull request justinjfu#9 from rail-berkeley/misc-updates

Misc updates

doodad/wrappers/easy_launch/README.md

@@ -3,7 +3,7 @@ Easy-launch is a wrapper around doodad that makes it easier to launch experiment
 Usage:
 
 ```python
-from doodad.wrappers.easy_launch.python_function import sweep_function
+from doodad.wrappers.easy_launch.core import sweep_function
 
 
 def function(doodad_config, variant):
@@ -78,11 +78,12 @@ NON_CODE_DIRS_TO_MOUNT = [
     ),
 ]
 LOCAL_LOG_DIR = '/home/user/logs'
+DEFAULT_DOCKER = 'user/image'
 ```
 
-After that, the settings are specific to the different mode you want to use
+And example docker file is provided in [docker_example/Dockerfile](docker_example/Dockerfile).)
 
-## BRC (sss and htp mode)
+## BRC (sss and htp mode - deprecated)
 _Disclaimer: this workflow is not polished, but it works. Hopefully we'll update this BRC workflow to be more similar to [this doodad version](https://github.com/rail-berkeley/doodad), but that's a work in progress._
 
 Setting up doodad + BRC takes a bit of extra leg-work, because you can't launched code from outside BRC using a API. Instead, you need to log in to the BRC node and launch code from inside of it.

doodad/wrappers/easy_launch/sweep_function.py → doodad/wrappers/easy_launch/core.py

@@ -26,7 +26,7 @@ def sweep_function(
         name_runs_by_id=True,
         add_time_to_run_id=True,
         start_run_id=0,
-        docker_img=config.DEFAULT_DOCKER,
+        docker_image=config.DEFAULT_DOCKER,
 ):
     """
     Usage:
@@ -82,7 +82,7 @@ def function(doodad_config, variant):
         datestamp = time.strftime("%y-%m-%d")
         log_path = '%s_%s' % (datestamp, log_path)
     target = osp.join(REPO_DIR, 'doodad/wrappers/easy_launch/run_experiment.py')
-    sweeper, output_mount = _create_sweeper_and_output_mount(mode, log_path, docker_img)
+    sweeper, output_mount = create_sweeper_and_output_mount(mode, log_path, docker_image)
     git_infos = metadata.generate_git_infos()
 
     doodad_config = metadata.DoodadConfig(
@@ -192,7 +192,7 @@ def _run_method_here_no_doodad(
         method_call(doodad_config, param)
 
 
-def _create_mounts():
+def create_mounts():
     NON_CODE_MOUNTS = [
         doodad.MountLocal(**non_code_mapping)
         for non_code_mapping in config.NON_CODE_DIRS_TO_MOUNT
@@ -207,15 +207,15 @@ def _create_mounts():
     return mounts
 
 
-def _create_sweeper_and_output_mount(mode, log_path, docker_img):
-    mounts = _create_mounts()
+def create_sweeper_and_output_mount(mode, log_path, docker_image):
+    mounts = create_mounts()
     az_mount = doodad.MountAzure(
         '',
         mount_point='/output',
     )
     sweeper = DoodadSweeper(
         mounts=mounts,
-        docker_img=docker_img,
+        docker_img=docker_image,
         azure_subscription_id=config.AZ_SUB_ID,
         azure_storage_connection_str=config.AZ_CONN_STR,
         azure_client_id=config.AZ_CLIENT_ID,
@@ -246,6 +246,7 @@ def example_function(doodad_config, variant):
     z = variant['z']
     with open(doodad_config.output_directory + '/function_output.txt', "w") as f:
         f.write('sum = {}'.format(x+y+z))
+    print('x, y, z = ', x, y, z)
     save_doodad_config(doodad_config)
 
 

doodad/wrappers/easy_launch/docker_example/Dockerfile

@@ -0,0 +1,98 @@
+# We need the CUDA base dockerfile to enable GPU rendering
+# on hosts with GPUs.
+# The image below is a pinned version of nvidia/cuda:9.1-cudnn7-devel-ubuntu16.04 (from Jan 2018)
+# If updating the base image, be sure to test on GPU since it has broken in the past.
+FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu16.04
+
+SHELL ["/bin/bash", "-c"]
+
+##########################################################
+### System dependencies
+##########################################################
+
+# Now let's download python 3 and all the dependencies
+RUN apt-get update -q \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    cmake \
+    curl \
+    git \
+    libav-tools \
+    libgl1-mesa-dev \
+    libgl1-mesa-glx \
+    libglew-dev \
+    libosmesa6-dev \
+    net-tools \
+    software-properties-common \
+    swig \
+    unzip \
+    vim \
+    wget \
+    xpra \
+    xserver-xorg-dev \
+    zlib1g-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+
+# Not sure why this is needed
+ENV LANG C.UTF-8
+
+# Not sure what this is fixing
+COPY ./files/Xdummy /usr/local/bin/Xdummy
+RUN chmod +x /usr/local/bin/Xdummy
+
+# Workaround for https://bugs.launchpad.net/ubuntu/+source/nvidia-graphics-drivers-375/+bug/1674677
+COPY ./files/10_nvidia.json /usr/share/glvnd/egl_vendor.d/10_nvidia.json
+
+# Not sure why this is needed
+ENV LD_LIBRARY_PATH /usr/local/nvidia/lib64:${LD_LIBRARY_PATH}
+
+##########################################################
+### MuJoCo
+##########################################################
+# Note: ~ is an alias for /root
+RUN mkdir -p /root/.mujoco \
+    && wget https://www.roboti.us/download/mujoco200_linux.zip -O mujoco.zip \
+    && unzip mujoco.zip -d /root/.mujoco \
+    && rm mujoco.zip
+RUN mkdir -p /root/.mujoco \
+    && wget https://www.roboti.us/download/mjpro150_linux.zip -O mujoco.zip \
+    && unzip mujoco.zip -d /root/.mujoco \
+    && rm mujoco.zip
+COPY ./files/mjkey.txt /root/.mujoco/mjkey.txt
+RUN ln -s /root/.mujoco/mujoco200_linux /root/.mujoco/mujoco200
+ENV LD_LIBRARY_PATH /root/.mujoco/mjpro150/bin:${LD_LIBRARY_PATH}
+ENV LD_LIBRARY_PATH /root/.mujoco/mujoco200/bin:${LD_LIBRARY_PATH}
+ENV LD_LIBRARY_PATH /root/.mujoco/mujoco200_linux/bin:${LD_LIBRARY_PATH}
+
+
+
+##########################################################
+### Example Python Installation
+##########################################################
+ENV PATH /opt/conda/bin:$PATH
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /tmp/miniconda.sh && \
+    /bin/bash /tmp/miniconda.sh -b -p /opt/conda && \
+    rm /tmp/miniconda.sh && \
+    ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
+    echo ". /opt/conda/etc/profile.d/conda.sh" >> /etc/bash.bashrc
+
+RUN conda update -y --name base conda && conda clean --all -y
+
+RUN conda create --name customenv python=3.6.5 pip
+RUN echo "source activate customenv" >> ~/.bashrc
+# Use the railrl pip
+ENV OLDPATH $PATH
+ENV PATH /opt/conda/envs/customenv/bin:$PATH
+
+# Install packages here
+RUN pip install cloudpickle==0.5.2
+RUN pip install torch==1.1.0
+RUN pip install torchvision
+
+##########################################################
+### gym sometimes has this patchelf issue
+##########################################################
+RUN curl -o /usr/local/bin/patchelf https://s3-us-west-2.amazonaws.com/openai-sci-artifacts/manual-builds/patchelf_0.9_amd64.elf \
+    && chmod +x /usr/local/bin/patchelf
+RUN pip install gym[all]==0.12.5

doodad/wrappers/easy_launch/docker_example/files/10_nvidia.json

@@ -0,0 +1,6 @@
+{
+    "file_format_version" : "1.0.0",
+    "ICD" : {
+        "library_path" : "libEGL_nvidia.so.0"
+    }
+}