Add evaluation notebook with tournament results. Organise configurati…

…on files

README.md

@@ -1,13 +1,26 @@
 
 # The source code of the "Graph neural networks and deep reinforcement learning in job-shop scheduling"
 
-To perform any of the experiments you must define the configuration file following examples in `configuration/experiments` folders.
+**Edit:** Organise the configuration files for better reproducibility of results. Added only configurations from 
+the final steps of the experiments!
+
+To perform any of the experiments you must define the configuration file following examples in `configuration/experiments/**/emperiment.yml` files.
 Then to execute the experiment just call `cli.py`, i.e.
 
 ```
 python cli.py --configuration PATH_TO_EXPERIMENT_FILE
 ```
 
+The results of the tournament runs are stored in the pickled version of the class from `environment/statistics.py`.
+To load the file use the `load` method of Statistics class.
+
+```
+from environment.statistics import Statistics
+
+statistics = Statistics.load(PATH_TO_PICKLED_FILE)
+```
+
 The requirements of the work are present in `requirements.txt`.
 
-Trained models and experiment results are available [here](https://campuscvut-my.sharepoint.com/:u:/g/personal/hayeuyur_cvut_cz/EQ7TgHnCjbVIvDmWBltOo5ABQH5YcKSm6CRa0k33InaY8A?e=ul41Pa).
+Trained models and experiment results are available [here](https://campuscvut-my.sharepoint.com/:u:/g/personal/hayeuyur_cvut_cz/EQ7TgHnCjbVIvDmWBltOo5ABQH5YcKSm6CRa0k33InaY8A?e=ul41Pa). Additionally, in `evaluation` folder you can find
+the results of the tournaments of the trained models.

diploma_thesis/a.txt

@@ -0,0 +1 @@
+Running 8 simulations

diploma_thesis/configuration/experiments/jsp/post-submission/experiments/1/0/experiment.yml

@@ -0,0 +1,163 @@
+
+
+default_mods: &default_mods
+  - 'util/train_schedule/on_store_128.yml'
+  - 'agent/dqn/ddqn.yml'
+  - 'agent/dqn/prioritized.yml'
+  - 'util/optimizer/adam_stationary.yml'
+  - 'util/optimizer/grad_norm.yml'
+
+###############################################################################################
+
+dqn_1: &dqn_1
+  base_path: 'configuration/experiments/jsp/post-submission/experiments/1/flexible_machine.yml'
+  template: '../../../../../../mods/machine/model/flexible_custom_gat'
+  mod_dirs:
+    - 'configuration/mods/machine/mods'
+  mods:
+    - *default_mods
+
+
+dqn_2: &dqn_2
+  base_path: 'configuration/experiments/jsp/post-submission/experiments/1/mr_machine.yml'
+  template: '../../../../../../mods/machine/model/marl_dqn/baseline'
+  mod_dirs:
+    - 'configuration/mods/machine/mods'
+  mods:
+    - *default_mods
+
+###############################################################################################
+
+reward: &reward
+  - kind: 'surrogate_tardiness'
+    parameters:
+      winq_factor: 0.2
+      span: 20
+      critical_level_factor: 64
+
+##############################################################################################
+
+# 8 runs
+
+long_single_source_run: &long_single_source_run
+  parameters:
+    mods:
+      __inout_factory__:
+        - [ ['utilization/90.yml']]
+    nested:
+      parameters:
+        dispatch:
+          seed:
+            - '__range__': [ 2000, 2300 ]
+
+
+###############################################################################################
+
+
+task:
+  kind: 'multi_task'
+  n_workers: 2
+  n_threads: 30
+  debug: False
+  store_run_statistics: False
+  output_dir: 'results/jsp/experiments/FINAL/BEST/0'
+
+  tasks:
+    - kind: 'multi_value'
+      parameters:
+        base:
+          name: 'model'
+          output_dir: '1'
+          log_stdout: False
+          seed: 123
+
+          machine_agent:
+            kind: 'mod'
+            parameters:
+              base_path: 'configuration/mods/machine_agent/model.yml'
+              mods: [ ]
+
+          work_center_agent:
+            kind: 'static'
+            parameters:
+              model:
+                kind: 'static'
+                parameters:
+                  rule: 'et'
+              encoder:
+                kind: 'plain'
+
+          tape:
+            machine_reward:
+              kind: 'surrogate_tardiness'
+
+            work_center_reward:
+              kind: 'no'
+
+          simulator:
+            kind: 'td'
+
+          run:
+            kind: 'mod'
+            parameters:
+              base_path: 'configuration/experiments/jsp/post-submission/experiments/1/run.yml'
+              mod_dirs:
+                - 'configuration/mods/run/mods'
+              mods: []
+              nested:
+                parameters:
+                  simulations:
+                    - name: ''
+                      kind: 'multi_value'
+                      parameters:
+                        base:
+                          kind: 'mod'
+                          parameters:
+                            base_path: 'configuration/experiments/jsp/post-submission/experiments/1/simulation.yml'
+                            mod_dirs:
+                              - 'configuration/mods/simulation/mods'
+                            mods: [ ]
+                        values:
+        values:
+          __concat__:
+
+            - output_dir: 'Graph'
+              machine_agent:
+                parameters:
+                  - *dqn_1
+              graph:
+                transition_model:
+                  kind: 'base'
+                  parameters:
+                    forward:
+                      kind: 'complete'
+
+                    schedule:
+                      kind: 'machine_compressed'
+
+                memory: 0
+                is_machine_set_in_work_center_connected: False
+                is_work_center_set_in_shop_floor_connected: False
+
+            - output_dir: 'MARL'
+              machine_agent:
+                parameters:
+                  - *dqn_2
+              graph:
+                transition_model:
+                  kind: 'no'
+
+          tape:
+            machine_reward:
+              *reward
+
+          run:
+            parameters:
+              nested:
+                parameters:
+                  simulations:
+                    __0__:
+                      parameters:
+                        values:
+                          __concat__:
+                            - *long_single_source_run

diploma_thesis/configuration/experiments/jsp/post-submission/experiments/1/1/experiment.yml

@@ -0,0 +1,163 @@
+
+
+default_mods: &default_mods
+  - 'util/train_schedule/on_store_128.yml'
+  - 'agent/dqn/ddqn.yml'
+  - 'agent/dqn/prioritized.yml'
+  - 'util/optimizer/adam_stationary.yml'
+  - 'util/optimizer/grad_norm.yml'
+
+###############################################################################################
+
+dqn_1: &dqn_1
+  base_path: 'configuration/experiments/jsp/post-submission/experiments/1/flexible_machine.yml'
+  template: '../../../../../../mods/machine/model/flexible_custom_gat'
+  mod_dirs:
+    - 'configuration/mods/machine/mods'
+  mods:
+    - *default_mods
+
+
+dqn_2: &dqn_2
+  base_path: 'configuration/experiments/jsp/post-submission/experiments/1/mr_machine.yml'
+  template: '../../../../../../mods/machine/model/marl_dqn/baseline'
+  mod_dirs:
+    - 'configuration/mods/machine/mods'
+  mods:
+    - *default_mods
+
+###############################################################################################
+
+reward: &reward
+  - kind: 'surrogate_tardiness'
+    parameters:
+      winq_factor: 0.2
+      span: 20
+      critical_level_factor: 64
+
+##############################################################################################
+
+# 8 runs
+
+long_single_source_run: &long_single_source_run
+  parameters:
+    mods:
+      __inout_factory__:
+        - [ ['utilization/90.yml']]
+    nested:
+      parameters:
+        dispatch:
+          seed:
+            - '__range__': [ 2300, 2600 ]
+
+
+###############################################################################################
+
+
+task:
+  kind: 'multi_task'
+  n_workers: 2
+  n_threads: 30
+  debug: False
+  store_run_statistics: False
+  output_dir: 'results/jsp/experiments/FINAL/BEST/1'
+
+  tasks:
+    - kind: 'multi_value'
+      parameters:
+        base:
+          name: 'model'
+          output_dir: '1'
+          log_stdout: False
+          seed: 123
+
+          machine_agent:
+            kind: 'mod'
+            parameters:
+              base_path: 'configuration/mods/machine_agent/model.yml'
+              mods: [ ]
+
+          work_center_agent:
+            kind: 'static'
+            parameters:
+              model:
+                kind: 'static'
+                parameters:
+                  rule: 'et'
+              encoder:
+                kind: 'plain'
+
+          tape:
+            machine_reward:
+              kind: 'surrogate_tardiness'
+
+            work_center_reward:
+              kind: 'no'
+
+          simulator:
+            kind: 'td'
+
+          run:
+            kind: 'mod'
+            parameters:
+              base_path: 'configuration/experiments/jsp/post-submission/experiments/1/run.yml'
+              mod_dirs:
+                - 'configuration/mods/run/mods'
+              mods: []
+              nested:
+                parameters:
+                  simulations:
+                    - name: ''
+                      kind: 'multi_value'
+                      parameters:
+                        base:
+                          kind: 'mod'
+                          parameters:
+                            base_path: 'configuration/experiments/jsp/post-submission/experiments/1/simulation.yml'
+                            mod_dirs:
+                              - 'configuration/mods/simulation/mods'
+                            mods: [ ]
+                        values:
+        values:
+          __concat__:
+
+            - output_dir: 'Graph'
+              machine_agent:
+                parameters:
+                  - *dqn_1
+              graph:
+                transition_model:
+                  kind: 'base'
+                  parameters:
+                    forward:
+                      kind: 'complete'
+
+                    schedule:
+                      kind: 'machine_compressed'
+
+                memory: 0
+                is_machine_set_in_work_center_connected: False
+                is_work_center_set_in_shop_floor_connected: False
+
+            - output_dir: 'MARL'
+              machine_agent:
+                parameters:
+                  - *dqn_2
+              graph:
+                transition_model:
+                  kind: 'no'
+
+          tape:
+            machine_reward:
+              *reward
+
+          run:
+            parameters:
+              nested:
+                parameters:
+                  simulations:
+                    __0__:
+                      parameters:
+                        values:
+                          __concat__:
+                            - *long_single_source_run