add scripts for zero-shot generalization evaluation

README.md

@@ -201,11 +201,11 @@ Coming soon.
   ```
 
 - Zero-shot generalization evaluation:
-  - To evaluate camera view generalization experiments, run [scripts/run_maniskill2_camera_view.sh](scripts/run_maniskill2_camera_view.sh). The script evaluates the given `checkpoint` of the given `model` on the given `task` with four different camera views. See the script for more details. For example:
+  - To evaluate camera view generalization experiments, run [scripts/run_maniskill2_camera_view.sh](scripts/run_maniskill2_camera_view.sh). The script evaluates the given `checkpoint` of the given `model` on the given `task` with four different camera views, using the specified `seed`. See the script for more details. For example:
   ```bash
   bash scripts/run_maniskill2_camera_view.sh ${path/to/checkpoint} ${task} ${model} ${seed}
   ```
-  - To evaluate visual changes generalization experiments, run [scripts/run_maniskill2_visual_changes.sh](scripts/run_maniskill2_visual_changes.sh). The script evaluates the given `checkpoint` of the given `model` with different lighting conditions, noise levels and background colors. See the script for more details. Note that currently only `StackCube` task is supported. For example:
+  - To evaluate visual changes generalization experiments, run [scripts/run_maniskill2_visual_changes.sh](scripts/run_maniskill2_visual_changes.sh). The script evaluates the given `checkpoint` of the given `model` with different lighting conditions, noise levels and background colors, using the specified `seed`. See the script for more details. Note that currently only `StackCube` task is supported. For example:
   ```bash
   bash scripts/run_maniskill2_visual_changes.sh ${path/to/checkpoint} ${model} ${seed}
   ```

scripts/run_maniskill2_camera_view.sh

@@ -0,0 +1,61 @@
+ckpt_path=$1
+task=$2 
+model=$3
+seed=$4
+
+task_name=${task%-v0}
+if [ "$task_name" = "PegInsertionSide" ];then
+    model_env_id=${task_name}-3steps-MultiView
+  else
+    model_env_id=${task_name}-MultiView
+fi
+
+echo "Starting validation for task: ${task} with model: ${model} and checkpoint path: ${ckpt_path}"
+echo "Model environment ID: ${model_env_id}"
+echo "Seed: ${seed}"
+
+if [[ "$model" == *"pcd"* ]]; then
+    echo "Running validation for PCD camera views..."
+    # Change the camera view for PCD by specifying the `camera_ids` parameter
+    # Mapping of camera names to IDs:
+    # 4:left_camera_5
+    # 5:down_camera_5
+    # 6:left_camera_10
+    # 7:down_camera_10
+    for camera in 4 5 6 7
+    do
+    echo "Validating with camera ID: ${camera}"
+    python src/validate.py exp_maniskill2_act_policy=base \
+        exp_maniskill2_act_policy/maniskill2_model@maniskill2_task=${task} \
+        exp_maniskill2_act_policy/maniskill2_model@maniskill2_model=${model} \
+        ckpt_path=${ckpt_path} \
+        model.env_id=${model_env_id} \
+        data.train.camera_ids=${camera} seed=${seed}
+    done
+else
+    echo "Running validation for RGB or RGBD camera views..."
+    # Change the camera view for RGB and RGBD by specifying the `camera_names` parameter
+    for camera in left_camera_5 left_camera_10 down_camera_5 down_camera_10
+    do 
+    echo "Validating with camera view: ${camera}"
+    python src/validate.py exp_maniskill2_act_policy=base \
+        exp_maniskill2_act_policy/maniskill2_model@maniskill2_task=${task} \
+        exp_maniskill2_act_policy/maniskill2_model@maniskill2_model=${model} \
+        ckpt_path=${ckpt_path} \
+        model.env_id=${model_env_id} \
+        data.train.camera_names=${camera} seed=${seed}
+    done
+fi
+
+echo "Validation completed."
+
+
+
+
+
+
+
+
+
+
+

scripts/run_maniskill2_visual_changes.sh

@@ -0,0 +1,75 @@
+ckpt_path=$1
+model=$2
+seed=$3
+
+echo "Starting validation with model: ${model} and checkpoint path: ${ckpt_path}"
+echo "Seed: ${seed}"
+# 1) lighting intensity evaluation
+light_intensities=("0.03" "0.05" "0.15" "0.6" "1.8" "3")
+for light in "${light_intensities[@]}"
+do
+echo "Running validation for light intensity: ${light}"
+python src/validate.py exp_maniskill2_act_policy=base \
+    exp_maniskill2_act_policy/maniskill2_model@maniskill2_task=StackCube-v0 \
+    exp_maniskill2_act_policy/maniskill2_model@maniskill2_model=${model} \
+    ckpt_path=${ckpt_path} \
+    model.env_id=StackCube-light-${light} \
+    task_name=${model}-light-${light} \
+    seed=${seed}
+done
+
+# 2) noise level evaluation
+noise_levels=("2" "16" "32" "64")
+for noise in "${noise_levels[@]}"
+do
+echo "Running validation for noise level: ${noise}"
+python src/validate.py exp_maniskill2_act_policy=base \
+    exp_maniskill2_act_policy/maniskill2_model@maniskill2_task=StackCube-v0 \
+    exp_maniskill2_act_policy/maniskill2_model@maniskill2_model=${model} \
+    ckpt_path=${ckpt_path} \
+    model.env_id=StackCube-v0 \
+    model.shader_dir="rt" \
+    model.rt_samples_per_pixel=${noise}  model.rt_use_denoiser=false \
+    task_name=${model}-noise-${noise} \
+    seed=${seed}
+done
+
+# background color evaluation
+colors=("0.2" "0.6" "1.0")
+
+for red_color in "${colors[@]}"
+do
+echo "Running validation for red background with R value: ${red_color}"
+python src/validate.py exp_maniskill2_act_policy=base \
+    exp_maniskill2_act_policy/maniskill2_model@maniskill2_task=StackCube-v0 \
+    exp_maniskill2_act_policy/maniskill2_model@maniskill2_model=${model} \
+    ckpt_path=${ckpt_path} \
+    model.env_id=StackCube-background-red-${red_color} \
+    task_name=${model}-red_color-${red_color} \
+    seed=${seed}
+done
+
+for green_color in "${colors[@]}"
+do
+echo "Running validation for green background with G value: ${green_color}"
+python src/validate.py exp_maniskill2_act_policy=base \
+    exp_maniskill2_act_policy/maniskill2_model@maniskill2_task=StackCube-v0 \
+    exp_maniskill2_act_policy/maniskill2_model@maniskill2_model=${model} \
+    ckpt_path=${ckpt_path} \
+    model.env_id=StackCube-background-green-${green_color} \
+    task_name=${model}-green_color-${green_color} \
+    seed=${seed}
+done
+
+
+
+
+
+
+
+
+
+
+
+
+