Add evaluation notebook

diploma_thesis/agents/base/state.py

@@ -93,9 +93,11 @@ def to_pyg_graph(self) -> pyg.data.Data | pyg.data.HeteroData:
             if isinstance(key, str):
                 data[key] = self.data[key]
 
+            # Nested
             if len(key) == 2:
                 data[key[0]][key[1]] = self.data[key]
 
+            # Edge
             if len(key) == 4:
                 data[key[:3]][key[3]] = self.data[key]
 

diploma_thesis/agents/machine/state/custom_encoder_v2.py

@@ -72,7 +72,7 @@ def __encode__(self, parameters: StateEncoder.Input) -> State:
                 completion_rate.view(-1),
                 critical_ratios.view(-1),
 
-                placeholder + wait_times,
+                placeholder + wait_times / self.norm_factor,
                 placeholder + machine_util_rate,
                 placeholder + arriving_jobs,
                 placeholder + will_arrive_jobs,

diploma_thesis/agents/machine/state/deep_marl_indirect.py

@@ -7,7 +7,7 @@
 from environment import JobReductionStrategy
 from .encoder import StateEncoder
 
-# TODO: - Update normalized state
+
 
 class DEEPMARLIndirectStateEncoder(StateEncoder):
 

diploma_thesis/agents/utils/nn/loss.py

@@ -13,7 +13,7 @@ class Configuration:
 
     def __init__(self, configuration: Configuration):
         super().__init__()
-        
+
         self.configuration = configuration
         self.loss = self.__make_loss__()
 

diploma_thesis/agents/utils/return_estimator/gae.py

@@ -37,8 +37,6 @@ def update_returns(self, records: List[Record]) -> List[Record]:
             records[i].info[Record.ADVANTAGE_KEY] = coef ** i * advantage + next_advantage
             records[i].info[Record.RETURN_KEY] = records[i].info[Record.ADVANTAGE_KEY] + value
 
-
-
         return records[:-1]
 
     @staticmethod

diploma_thesis/agents/utils/return_estimator/n_step.py

@@ -37,6 +37,8 @@ def __init__(self, configuration: Configuration):
 
         self.configuration = configuration
 
+        if self.configuration.n > 1:
+            self.value_fetch_method = ValueFetchMethod.ACTION
 
     @property
     def discount_factor(self) -> float:
@@ -75,7 +77,7 @@ def update_returns(self, records: List[Record]) -> List[Record]:
                 off_policy_weights += [1]
 
         for i in range(len(records)):
-            g = records[i].info[Record.VALUE_KEY]
+            g = self.get_value(records[i])
             n = min(self.configuration.n, len(records) - i)
 
             weights = off_policy_weights[i:i+n]

diploma_thesis/agents/utils/rl/rl.py

@@ -105,7 +105,7 @@ def __train_step__(self, model: Policy):
 
         torch.cuda.empty_cache()
 
-        print(f'Train step: { time.time() - start } Optimizer Step: { self.optimizer.step_count }')
+        print(f'Train step: { time.time() - start } Optimizer Step: { self.optimizer.step_count } Learning Rate: { self.optimizer.learning_rate}')
 
     def clear(self):
         self.loss_cache = []

diploma_thesis/agents/utils/rl/storage.py

@@ -27,9 +27,9 @@ def store(self, sample: TrainingSample):
             record.info['episode'] = sample.episode_id
 
             # Remove extra fields for training record
-            if record.state.graph is not None and Graph.JOB_INDEX_MAP in record.state.graph.data.keys():
-                del record.state.graph.data[Graph.JOB_INDEX_MAP]
-                del record.next_state.graph.data[Graph.JOB_INDEX_MAP]
+            for graph in [record.state.graph, record.next_state.graph]:
+                if graph is not None and Graph.JOB_INDEX_MAP in graph.data.keys():
+                    del graph.data[Graph.JOB_INDEX_MAP]
 
         if self.is_episodic:
             self.memory.store([records])

diploma_thesis/configuration/experiments/jsp/GRAPH-BEST/experiments/1/0/experiment.yml

@@ -0,0 +1,207 @@
+# Evaluate the effectivenes of basic DQNs on the JSP environment
+base_model: &base_model 'configuration/experiments/jsp/GRAPH-BEST/experiments/1/flexible_machine.yml'
+
+
+default_mods: &default_mods
+  - 'util/train_schedule/on_store_256.yml'
+  - 'agent/dqn/ddqn.yml'
+#  - 'agent/dqn/dueling.yml'
+  - 'agent/dqn/prioritized.yml'
+#  - 'agent/dqn/steps/3.yml'
+  - 'util/optimizer/adam_stationary.yml'
+  - 'util/optimizer/grad_norm.yml'
+#  - 'util/infrastructure/cuda.yml'
+
+###############################################################################################
+
+dqn_1: &dqn_1
+  base_path: *base_model
+  template: 'custom_gat'
+  mod_dirs:
+    - 'configuration/mods/machine/mods'
+  mods:
+    - *default_mods
+
+
+###############################################################################################
+
+dqn_2: &dqn_2
+  base_path: *base_model
+  template: 'custom_gin'
+  mod_dirs:
+    - 'configuration/mods/machine/mods'
+  mods:
+    - *default_mods
+
+###############################################################################################
+
+dqn_3: &dqn_3
+  base_path: *base_model
+  template: 'hierarchical_gin'
+  mod_dirs:
+    - 'configuration/mods/machine/mods'
+  mods:
+    - *default_mods
+
+###############################################################################################
+
+dqn_4: &dqn_4
+  base_path: *base_model
+  template: 'hierarchical_gat'
+  mod_dirs:
+    - 'configuration/mods/machine/mods'
+  mods:
+    - *default_mods
+
+###############################################################################################
+
+reward: &reward
+  - kind: 'surrogate_tardiness'
+    parameters:
+      winq_factor: 0.2
+      span: 80
+      critical_level_factor: 64
+
+##############################################################################################
+
+# 8 runs
+
+long_single_source_run: &long_single_source_run
+  parameters:
+    mods:
+      __inout_factory__:
+        - [ ['utilization/90.yml' ]]
+    nested:
+      parameters:
+        dispatch:
+          seed:
+            - '__range__': [ 2000, 2200 ]
+
+
+###############################################################################################
+
+
+task:
+  kind: 'multi_task'
+  n_workers: 1
+  n_threads: 30
+  debug: False
+  store_run_statistics: False
+  output_dir: 'results/jsp/experiments/FINAL/BEST/0'
+
+  tasks:
+    - kind: 'multi_value'
+      parameters:
+        base:
+          name: 'model'
+          output_dir: '1'
+          log_stdout: False
+          seed: 123
+
+          machine_agent:
+            kind: 'mod'
+            parameters:
+              base_path: 'configuration/mods/machine_agent/model.yml'
+              mods: [ ]
+
+          work_center_agent:
+            kind: 'static'
+            parameters:
+              model:
+                kind: 'static'
+                parameters:
+                  rule: 'et'
+              encoder:
+                kind: 'plain'
+
+          tape:
+            machine_reward:
+              kind: 'surrogate_tardiness'
+
+            work_center_reward:
+              kind: 'no'
+
+          simulator:
+            kind: 'td'
+
+
+          run:
+            kind: 'mod'
+            parameters:
+              base_path: 'configuration/experiments/jsp/GRAPH-BEST/experiments/1/run.yml'
+              mod_dirs:
+                - 'configuration/mods/run/mods'
+              mods: []
+              nested:
+                parameters:
+                  simulations:
+                    - name: ''
+                      kind: 'multi_value'
+                      parameters:
+                        base:
+                          kind: 'mod'
+                          parameters:
+                            base_path: 'configuration/experiments/jsp/GRAPH-BEST/experiments/1/simulation.yml'
+                            mod_dirs:
+                              - 'configuration/mods/simulation/mods'
+                            mods: [ ]
+                        values:
+        values:
+          __concat__:
+
+#            - output_dir: 'Global'
+#              machine_agent:
+#                parameters:
+#                  - *dqn_1
+#                  - *dqn_2
+#                  - *dqn_3
+#                  - *dqn_4
+#              graph:
+#                transition_model:
+#                  kind: 'base'
+#                  parameters:
+#                    forward:
+#                      kind: 'complete'
+#
+#                    schedule:
+#                      kind: 'complete'
+#
+#                memory: 0
+#                is_machine_set_in_work_center_connected: False
+#                is_work_center_set_in_shop_floor_connected: False
+
+            - output_dir: 'Local'
+              machine_agent:
+                parameters:
+                  - *dqn_1
+#                  - *dqn_2
+#                  - *dqn_3
+#                  - *dqn_4
+              graph:
+                transition_model:
+                  kind: 'base'
+                  parameters:
+                    forward:
+                      kind: 'complete'
+
+                    schedule:
+                      kind: 'machine_compressed'
+
+                memory: 0
+                is_machine_set_in_work_center_connected: True
+                is_work_center_set_in_shop_floor_connected: True
+
+          tape:
+            machine_reward:
+              *reward
+
+          run:
+            parameters:
+              nested:
+                parameters:
+                  simulations:
+                    __0__:
+                      parameters:
+                        values:
+                          __concat__:
+                            - *long_single_source_run

diploma_thesis/configuration/experiments/jsp/GRAPH-BEST/experiments/1/flexible_machine.yml

@@ -0,0 +1,79 @@
+
+kind: 'rl'
+parameters:
+
+  model:
+    kind: 'deep_rule'
+    parameters:
+      policy:
+        kind: 'flexible_action'
+        parameters:
+          policy_method: 'independent'
+
+          model:
+            __model__: ''
+
+          action_selector:
+            kind: 'phase_selector'
+            parameters:
+              default:
+                 kind: 'greedy'
+              phases:
+                - phase:
+                    kind: 'warm_up'
+                    parameters:
+                      step: 0
+                  action_selector:
+                    kind: 'uniform'
+                - phase:
+                    kind: 'warm_up'
+                    parameters:
+                      step: 1
+                  action_selector:
+                    kind: 'epsilon_greedy'
+                    parameters:
+                      epsilon: 0.4
+                - phase:
+                    kind: 'training'
+                  action_selector:
+                    kind: 'epsilon_greedy'
+                    parameters:
+                      epsilon: 0.4
+                      min_epsilon: 0.05
+                      decay_factor: 0.999
+                      decay_steps: 125
+
+  __encoder__: ''
+
+  trainer:
+    kind: 'dqn'
+    parameters:
+      decay: 1.0
+      update_steps: 50
+      epochs: 5
+
+      memory:
+        kind: 'replay'
+        parameters:
+          size: 16394
+          batch_size: 1024
+
+      loss:
+        kind: 'mse'
+        parameters:
+          reduction: 'none'
+
+      optimizer:
+        model:
+          kind: 'adam'
+          parameters:
+            lr: 0.001
+        scheduler:
+          kind: 'exponential'
+          parameters:
+            gamma: 0.999
+
+      return:
+        kind: 'no'
+        parameters:
+          discount_factor: 0.95

diploma_thesis/configuration/experiments/jsp/GRAPH-BEST/experiments/1/run.yml

@@ -0,0 +1,23 @@
+
+# JSP training based on the DEEP-MARL thesis
+
+kind: 'plain'
+parameters:
+  timeline:
+    warmup:
+      - 5000
+      - 5000
+
+    duration: 20000000
+
+  machine_train_schedule:
+    pretrain_steps: 10
+    train_interval: 25
+    max_training_steps: 100000000
+
+  work_center_train_schedule:
+    pretrain_steps: 0
+    train_interval: 100
+    max_training_steps: 0
+
+  n_workers: 10