From 8f383d0deea4d87fbc39b37e9c6ee9fa520e8b50 Mon Sep 17 00:00:00 2001
From: Peter Gadfort <gadfort@zeroasic.com>
Date: Thu, 25 Apr 2024 14:12:35 -0400
Subject: [PATCH] add _merge_with_init to schema object to handle inconsistent
 schemas better

---
 siliconcompiler/schema/schema_obj.py | 68 +++++++++++++++++++++++-----
 tests/schema/test_schema.py          | 38 ++++++++++++++++
 2 files changed, 94 insertions(+), 12 deletions(-)

diff --git a/siliconcompiler/schema/schema_obj.py b/siliconcompiler/schema/schema_obj.py
index 9d0953ea9..233c06720 100644
--- a/siliconcompiler/schema/schema_obj.py
+++ b/siliconcompiler/schema/schema_obj.py
@@ -56,12 +56,17 @@ def __init__(self, cfg=None, manifest=None, logger=None):
 
         self._init_logger(logger)
 
-        if cfg is not None:
-            self.cfg = Schema._dict_to_schema(copy.deepcopy(cfg))
-        elif manifest is not None:
+        if manifest is not None:
             # Normalize value to string in case we receive a pathlib.Path
-            manifest = str(manifest)
-            self.cfg = Schema._read_manifest(manifest)
+            cfg = Schema.__read_manifest_file(str(manifest))
+
+        if cfg is not None:
+            try:
+                self.cfg = Schema._dict_to_schema(copy.deepcopy(cfg))
+            except (TypeError, ValueError) as e:
+                raise ValueError('Attempting to read manifest with '
+                                 f'incompatible schema version: {e}') \
+                    from e
         else:
             self.cfg = self._init_schema_cfg()
 
@@ -110,9 +115,54 @@ def _dict_to_schema(cfg):
                 Schema._dict_to_schema_set(cfg[category], category)
         return cfg
 
+    ###########################################################################
+    def _merge_with_init_schema(self):
+        new_schema = Schema()
+
+        for keylist in self.allkeys():
+            if keylist[0] in ('history', 'library'):
+                continue
+
+            if 'default' in keylist:
+                continue
+
+            # only read in valid keypaths without 'default'
+            key_valid = new_schema.valid(*keylist, default_valid=True)
+            if not key_valid:
+                self.logger.warning(f'Keypath {keylist} is not valid')
+            if not key_valid:
+                continue
+
+            for val, step, index in self._getvals(*keylist, return_defvalue=False):
+                new_schema.set(*keylist, val, step=step, index=index)
+
+                # update other pernode fields
+                # TODO: only update these if clobber is successful
+                step_key = Schema.GLOBAL_KEY if not step else step
+                idx_key = Schema.GLOBAL_KEY if not index else index
+                for field in self.getdict(*keylist)['node'][step_key][idx_key].keys():
+                    if field == 'value':
+                        continue
+                    new_schema.set(*keylist,
+                                   self.get(*keylist, step=step, index=index, field=field),
+                                   step=step, index=index, field=field)
+
+        if 'library' in self.cfg:
+            # Handle libraries seperately
+            for library in self.cfg['library'].keys():
+                lib_schema = Schema(cfg=self.getdict('library', library))
+                lib_schema._merge_with_init_schema()
+                new_schema.cfg['library'][library] = lib_schema.cfg
+
+        if 'history' in self.cfg:
+            # Copy over history
+            new_schema.cfg['history'] = self.cfg['history']
+
+        self.cfg = new_schema.cfg
+
     ###########################################################################
     @staticmethod
-    def _read_manifest(filepath):
+    def __read_manifest_file(filepath):
         if not os.path.isfile(filepath):
             raise ValueError(f'Manifest file not found {filepath}')
 
@@ -133,12 +183,6 @@ def _read_manifest(filepath):
         finally:
             fin.close()
 
-        try:
-            Schema._dict_to_schema(localcfg)
-        except (TypeError, ValueError) as e:
-            raise ValueError(f'Attempting to read manifest with incompatible schema version: {e}') \
-                from e
-
         return localcfg
 
     def get(self, *keypath, field='value', job=None, step=None, index=None):
diff --git a/tests/schema/test_schema.py b/tests/schema/test_schema.py
index 38760de6c..2c4560308 100644
--- a/tests/schema/test_schema.py
+++ b/tests/schema/test_schema.py
@@ -4,6 +4,7 @@
 
 from siliconcompiler.schema import Schema
 from siliconcompiler.schema.schema_cfg import scparam
+from siliconcompiler import Chip
 
 
 def test_list_of_lists():
@@ -96,3 +97,40 @@ def test_list_of_tuples():
     # for a list type
     schema.set(*keypath, ['import', '0'])
     assert schema.get(*keypath) == expected
+
+
+def test_merge_with_init_old_has_values():
+    old_schema = Schema().cfg
+
+    scparam(old_schema, ['test'], sctype='[[str]]', shorthelp='Test')
+
+    new_schema = Schema(cfg=old_schema)
+    assert new_schema.getdict('test')
+
+    new_schema._merge_with_init_schema()
+
+    with pytest.raises(ValueError):
+        new_schema.getdict('test')
+
+
+def test_merge_with_init_new_has_values():
+    old_schema = Schema().cfg
+
+    del old_schema['package']
+
+    new_schema = Schema(cfg=old_schema)
+    with pytest.raises(ValueError):
+        new_schema.getdict('package')
+
+    new_schema._merge_with_init_schema()
+
+    assert new_schema.getdict('package')
+
+
+def test_merge_with_init_with_lib():
+    chip = Chip('')
+    chip.load_target('asic_demo')
+
+    chip.schema._merge_with_init_schema()
+
+    assert 'sky130hd' in chip.getkeys('library')