diff --git a/.ci_support/environment.yml b/.ci_support/environment.yml index 5d0afea2d9..d27c6a8f6b 100644 --- a/.ci_support/environment.yml +++ b/.ci_support/environment.yml @@ -9,11 +9,11 @@ dependencies: - h5py =3.10.0 - jinja2 =3.1.2 - numpy =1.26.2 -- pandas =2.1.3 +- pandas =2.1.4 - pint =0.23 -- psutil =5.9.5 -- pyfileindex =0.0.18 -- pysqa =0.1.7 +- psutil =5.9.7 +- pyfileindex =0.0.19 +- pysqa =0.1.8 - pytables =3.9.2 - sqlalchemy =2.0.23 - tqdm =4.66.1 diff --git a/pyiron_base/interfaces/lockable.py b/pyiron_base/interfaces/lockable.py new file mode 100644 index 0000000000..4116b63ef4 --- /dev/null +++ b/pyiron_base/interfaces/lockable.py @@ -0,0 +1,351 @@ +""" +A small mixin to lock attribute and method access at runtime. + +Sometimes we wish to restrict users of pyiron from changing certain things past certain stages of object lifetime, e.g. +the input of jobs should only be changed before it is run, but still need to be able to change them internally. This +can be implemented with :class:`~Lockable` and the decorator :func:`~sentinel`. It should be thought of as a well +defined escape hatch that is rarely necessary. Users should never be expected to unlock an object ever again after it +has been locked by them or pyiron. + +The context manager functionality is implemented in a separate class rather than directly on Lockable to conserve dunder +name space real estate and let subclasses be context managers on their own. + +Through out the code inside methods of `Lockable` will use `object.__setattr__` +and `object.__getattribute__` to avoid any overloading attribute access that +sibling classes may bring in. +""" + +from typing import Optional, Literal +from functools import wraps +import warnings + +from pyiron_base.interfaces.has_groups import HasGroups + + +class Locked(Exception): + pass + + +class LockedWarning(UserWarning): + pass + + +def sentinel(meth): + """ + Wrap a method to fail if `read_only` is `True` on the owning object. + + Use together with :class:`Lockable`. + + Args: + meth (method): method to call if `read_only` is `False`. + + Returns: + wrapped method + """ + + def dispatch_or_error(self, *args, **kwargs): + try: + method = object.__getattribute__(self, "_lock_method") + except AttributeError: + method = None + if method not in ("error", "warning"): + method = "error" + if self.read_only and method == "error": + raise Locked( + "Object is currently locked! Use unlocked() if you know what you are doing." + ) + elif self.read_only and method == "warning": + warnings.warn( + f"{meth.__name__} called on {type(self)}, but object is locked!", + category=LockedWarning, + ) + return meth(self, *args, **kwargs) + + # if sentinel is applied to __setattr__ we must ensure that `read_only` + # stays available, otherwise we can't unlock again later + if meth.__name__ == "__setattr__": + + @wraps(meth) + def f(self, *args, **kwargs): + if len(args) > 0: + target = args[0] + else: + target = kwargs["name"] + if target in ("read_only", "_read_only"): + return meth(self, *args, **kwargs) + return dispatch_or_error(self, *args, **kwargs) + + else: + f = wraps(meth)(dispatch_or_error) + return f + + +class _UnlockContext: + """ + Context manager that unlocks and relocks a :class:`Lockable`. + + This is an implementation detail of :class:`Lockable`. + """ + + __slots__ = ("owner",) + + def __init__(self, owner): + self.owner = owner + + def __enter__(self): + self.owner.read_only = False + return self.owner + + def __exit__(self, *_): + self.owner.lock() + return False # never handle exceptions + + +def _iterate_lockable_subs(lockable_groups): + """ + Yield sub nodes and groups that are lockable. Recurse into groups. + + If the given object is not a :class:`HasGroups` yield nothing. + """ + if not isinstance(lockable_groups, HasGroups): + return + + subs = lockable_groups.list_all() + for n in subs["nodes"]: + node = lockable_groups[n] + if isinstance(node, Lockable): + yield node + + for g in subs["groups"]: + group = lockable_groups[g] + yield group + yield from _iterate_lockable_subs(g) + + +class Lockable: + """ + A small mixin to lock attribute and method access at runtime. + + The mixin maintains an :attr:`~.read_only` and offers a context manager to temporarily unset it. It does **not** + restrict access to any attributes or methods on its own. Instead sub classes are expected to mark methods they wish + protected with :func:`.sentinel`. Wrapped methods will then raise :exc:`.Locked` if :attr:`~.read_only` is set. + + If the subclass also implements :class:`.HasGroups`, locking it will iterate over all nodes and (recursively) + groups and lock them if possible and vice-versa for unlocking. + + Once an object has been locked it should generally not be expected to be (permanently) unlocked again, especially + not explicitely by the user. + + Subclasses need to initialize this class by calling the inherited `__init__`, if explicitely overriding it. When + not explicitely overriding it (as in the examples below), take care that either the other super classes call + `super().__init__` or place this class before them in the inheritance order. Also be sure to initialize it before + using methods and properties decorated with :func:`.sentinel`. + + Subclasses may override :meth:`_on_lock` and :meth:`_on_unlock` if they wish to customize locking/unlocking + behaviour, provided that they call `super()` in their overloads. + + Let's start with a simple example; a list that can be locked + + >>> class LockList(Lockable, list): + ... __setitem__ = sentinel(list.__setitem__) + ... clear = sentinel(list.clear) + >>> l = LockList([1,2,3]) + >>> l + [1, 2, 3] + + Deriving adds the read only flag + + >>> l.read_only + False + + While it is not set, we may mutate the object + + >>> l[2] = 4 + >>> l[2] + 4 + + Once it is locked, the wrapped methods will raise errors + + >>> l.lock() + >>> l.read_only + True + >>> l[1] + 2 + >>> l[1] = 4 + Traceback (most recent call last): + ... + lockable.Locked: Object is currently locked! Use unlocked() if you know what you are doing. + + You can lock an object multiple times to no effect + + >>> l.lock() + + From now on every modification should be done with the :meth:`~.unlocked()` context manager. It returns the + unlocked object itself. + + >>> with l.unlocked(): + ... l[1] = 4 + >>> l[1] + 4 + >>> with l.unlocked() as lopen: + ... print(l is lopen) + ... l[1] = 4 + True + + :func:`~.sentinel` can be used for methods, item and attribute access. + + >>> l.clear() + Traceback (most recent call last): + ... + lockable.Locked: Object is currently locked! Use unlocked() if you know what you are doing. + >>> with l.unlocked(): + ... l.clear() + >>> l + [] + + When used together with :class:`.HasGroups`, objects will be locked recursively. + + >>> class LockGroupDict(Lockable, dict, HasGroups): + ... __setitem__ = sentinel(dict.__setitem__) + ... + ... def _list_groups(self): + ... return [k for k, v in self.items() if isinstance(v, LockGroupDict)] + ... + ... def _list_nodes(self): + ... return [k for k, v in self.items() if not isinstance(v, LockGroupDict)] + + >>> d = LockGroupDict(a=dict(c=1, d=2), b=LockGroupDict(c=1, d=2)) + >>> d.lock() + + Since the first item is a plain dict, it can still be mutated. + + >>> type(d['a']) + + >>> d['a']['c'] = 23 + >>> d['a']['c'] + 23 + + Where as the second will be locked from now on + + >>> type(d['b']) + + >>> d['b']['c'] = 23 + Traceback (most recent call last): + ... + lockable.Locked: Object is currently locked! Use unlocked() if you know what you are doing. + >>> d['b']['c'] + 1 + + but we can unlock it as usual + + >>> with d.unlocked(): + ... d['b']['d'] = 23 + >>> d['b']['d'] + 23 + + To use this class with properties, simply decorate the setter + + >>> class MyLock(Lockable): + ... def __init__(self, foo): + ... super().__init__() + ... self._foo = foo + ... @property + ... def foo(self): + ... return self._foo + ... @foo.setter + ... @sentinel + ... def foo(self, value): + ... self._foo = value + >>> ml = MyLock(42) + >>> ml.foo + 42 + >>> ml.foo = 23 + >>> ml.lock() + >>> ml.foo = 42 + Traceback (most recent call last): + ... + lockable.Locked: Object is currently locked! Use unlocked() if you know what you are doing. + + It's possible to change the errors raised into a warning and allow + modification by passing `lock_method` to :meth:`~.Lockable.__init__` or + `method` to :meth:`~.lock`. + + >>> mw = LockList(lock_method="warning") + >>> mw.append(0) + >>> mw.lock() + >>> mw[0] = 1 # will print the warning + >>> mw[0] + 1 + + >>> mw = LockList() + >>> mw.append(0) + >>> mw.lock(method='warning') + >>> mw[0] = 1 # will print the warning + >>> mw[0] + 1 + + """ + + def __init__(self, *args, lock_method: str = "error", **kwargs): + object.__setattr__(self, "_read_only", False) + object.__setattr__(self, "_lock_method", lock_method) + super().__init__(*args, **kwargs) + + @property + def read_only(self) -> bool: + """ + bool: False if the object can currently be written to + + Setting this value will trigger :meth:`._on_lock` and :meth:`._on_unlock` if it changes. + """ + return object.__getattribute__(self, "_read_only") + + @read_only.setter + def read_only(self, value: bool): + changed = self._read_only != value + if changed: + self._read_only = value + if value: + self._on_lock() + else: + self._on_unlock() + + def _on_lock(self): + for it in _iterate_lockable_subs(self): + it.lock() + + def _on_unlock(self): + for it in _iterate_lockable_subs(self): + it.read_only = False + + def lock(self, method: Optional[Literal["error", "warning"]] = None): + """ + Set :attr:`~.read_only`. + + Objects may be safely locked multiple times without further effect. + + Args: + method (str, either "error" or "warning"): if "error" raise an :class:`.Locked` exception if modification is + attempted; if "warning" raise a :class:`.LockedWarning` warning; default is "error" or the value + passed to the constructor. + + Raises: + ValueError: if `method` is not an allowed value + """ + if method not in ["error", "warning", None]: + raise ValueError(f"Unrecognized lock method {method}!") + if method is not None: + object.__setattr__(self, "_lock_method", method) + self.read_only = True + + def unlocked(self) -> _UnlockContext: + """ + Unlock the object temporarily. + + Context manager returns this object again and relocks it after the `with` statement finished. + + .. note:: `lock()` vs. `unlocked()` + + There is a small asymmetry between these two methods. :meth:`.lock` can only be done once (meaningfully), while :meth:`.unlocked` is a context manager and can be called multiple times. + """ + return _UnlockContext(self) diff --git a/pyiron_base/jobs/job/extension/server/generic.py b/pyiron_base/jobs/job/extension/server/generic.py index e01103643f..77074ac02c 100644 --- a/pyiron_base/jobs/job/extension/server/generic.py +++ b/pyiron_base/jobs/job/extension/server/generic.py @@ -12,6 +12,7 @@ from typing import Union from pyiron_base.state import state +from pyiron_base.interfaces.lockable import Lockable, sentinel from pyiron_base.utils.instance import static_isinstance from pyiron_base.jobs.job.extension.server.runmode import Runmode @@ -27,7 +28,9 @@ __date__ = "Sep 1, 2017" -class Server: # add the option to return the job id and the hold id to the server object +class Server( + Lockable +): # add the option to return the job id and the hold id to the server object """ Generic Server object to handle the execution environment for the job @@ -95,6 +98,7 @@ def __init__( run_mode="modal", new_hdf=True, ): + super().__init__() self._cores = cores self._threads = threads self._active_queue = None @@ -130,6 +134,7 @@ def send_to_db(self): return self._send_to_db @send_to_db.setter + @sentinel def send_to_db(self, send): """ Set the boolean option to decide which jobs should be store in the external/public database @@ -144,6 +149,7 @@ def accept_crash(self): return self._accept_crash @accept_crash.setter + @sentinel def accept_crash(self, accept): self._accept_crash = accept @@ -158,6 +164,7 @@ def structure_id(self): return self._structure_id @structure_id.setter + @sentinel def structure_id(self, structure_id): """ Set the structure ID to be linked to an external/public database @@ -178,6 +185,7 @@ def queue(self): return self._active_queue @queue.setter + @sentinel def queue(self, new_scheduler): """ Set a queue for the current simulation, by choosing one of the options @@ -236,6 +244,7 @@ def queue_id(self): return self._queue_id @queue_id.setter + @sentinel def queue_id(self, qid): """ Set the queue ID @@ -256,6 +265,7 @@ def threads(self): return self._threads @threads.setter + @sentinel def threads(self, number_of_threads): """ The number of threads selected for the current simulation @@ -276,6 +286,7 @@ def gpus(self): return self._gpus @gpus.setter + @sentinel def gpus(self, number_of_gpus): """ Total number of GPUs to use for this calculation. @@ -296,6 +307,7 @@ def cores(self): return self._cores @cores.setter + @sentinel def cores(self, new_cores): """ The number of cores selected for the current simulation @@ -335,6 +347,7 @@ def run_time(self): return self._run_time @run_time.setter + @sentinel def run_time(self, new_run_time): """ The run time in seconds selected for the current simulation @@ -366,6 +379,7 @@ def memory_limit(self): return self._memory_limit @memory_limit.setter + @sentinel def memory_limit(self, limit): if state.queue_adapter is not None and self._active_queue is not None: memory_max = state.queue_adapter.check_queue_parameters( @@ -394,6 +408,7 @@ def run_mode(self): return self._run_mode @run_mode.setter + @sentinel def run_mode(self, new_mode): """ Set the run mode of the job @@ -421,6 +436,7 @@ def new_hdf(self): return self._new_hdf @new_hdf.setter + @sentinel def new_hdf(self, new_hdf_bool): """ New_hdf5 defines whether a subjob should be stored in the same HDF5 file or in a new one. @@ -468,6 +484,7 @@ def executor(self) -> Union[Executor, None]: return self._executor @executor.setter + @sentinel def executor(self, exe: Union[Executor, None]): """ Executor to execute the job object this server object is attached to in the background. @@ -499,6 +516,8 @@ def future(self) -> Union[Future, None]: """ return self._future + # We don't wrap future in sentinel, to allow it later to be dropped to + # None, once execution is finished @future.setter def future(self, future_obj: Future): """ diff --git a/pyiron_base/jobs/job/generic.py b/pyiron_base/jobs/job/generic.py index eb20b6e9ca..4d08c53e78 100644 --- a/pyiron_base/jobs/job/generic.py +++ b/pyiron_base/jobs/job/generic.py @@ -1276,7 +1276,7 @@ def set_input_to_read_only(self): This function enforces read-only mode for the input classes, but it has to be implemented in the individual classes. """ - pass + self.server.lock() def _run_if_busy(self): """ diff --git a/pyiron_base/jobs/master/generic.py b/pyiron_base/jobs/master/generic.py index 39b4b38039..d252a32298 100644 --- a/pyiron_base/jobs/master/generic.py +++ b/pyiron_base/jobs/master/generic.py @@ -143,6 +143,7 @@ def job_object_dict(self): @wraps(GenericJob.set_input_to_read_only) def set_input_to_read_only(self): + super().set_input_to_read_only() self._input.read_only = True def first_child_name(self): diff --git a/pyiron_base/jobs/master/interactivewrapper.py b/pyiron_base/jobs/master/interactivewrapper.py index a02b775ca8..61ea211fc3 100644 --- a/pyiron_base/jobs/master/interactivewrapper.py +++ b/pyiron_base/jobs/master/interactivewrapper.py @@ -59,6 +59,7 @@ def ref_job(self, ref_job): self.append(ref_job) def set_input_to_read_only(self): + super().set_input_to_read_only() self.input.read_only = True set_input_to_read_only.__doc__ = GenericMaster.set_input_to_read_only.__doc__ diff --git a/pyiron_base/jobs/script.py b/pyiron_base/jobs/script.py index 2b6777b570..7df709cf13 100644 --- a/pyiron_base/jobs/script.py +++ b/pyiron_base/jobs/script.py @@ -271,6 +271,7 @@ def set_input_to_read_only(self): This function enforces read-only mode for the input classes, but it has to be implement in the individual classes. """ + super().set_input_to_read_only() self.input.read_only = True def to_hdf(self, hdf=None, group_name=None): diff --git a/pyiron_base/storage/datacontainer.py b/pyiron_base/storage/datacontainer.py index 6689ab65ea..b111b33c6e 100644 --- a/pyiron_base/storage/datacontainer.py +++ b/pyiron_base/storage/datacontainer.py @@ -18,6 +18,7 @@ from pyiron_base.storage.helper_functions import write_dict_to_hdf from pyiron_base.interfaces.has_groups import HasGroups from pyiron_base.interfaces.has_hdf import HasHDF +from pyiron_base.interfaces.lockable import Lockable, sentinel __author__ = "Marvin Poul" __copyright__ = ( @@ -47,7 +48,7 @@ def _normalize(key): return key -class DataContainer(MutableMapping, HasGroups, HasHDF): +class DataContainer(MutableMapping, Lockable, HasGroups, HasHDF): """ Mutable sequence with optional keys. @@ -253,12 +254,18 @@ def __new__(cls, *args, **kwargs): object.__setattr__(instance, "_store", []) object.__setattr__(instance, "_indices", {}) object.__setattr__(instance, "table_name", None) - object.__setattr__(instance, "_read_only", False) object.__setattr__(instance, "_lazy", False) return instance - def __init__(self, init=None, table_name=None, lazy=False, wrap_blacklist=()): + def __init__( + self, + init=None, + table_name=None, + lazy=False, + wrap_blacklist=(), + lock_method="warning", + ): """ Create new container. @@ -270,6 +277,7 @@ def __init__(self, init=None, table_name=None, lazy=False, wrap_blacklist=()): wrap_blacklist (tuple of types): any values in `init` that are instances of the given types are *not* wrapped in :class:`.DataContainer` """ + super().__init__(lock_method=lock_method) self.table_name = table_name self._lazy = lazy if init is not None: @@ -318,10 +326,8 @@ def __getitem__(self, key): else: raise ValueError("{} is not a valid key, must be str or int".format(key)) + @sentinel def __setitem__(self, key, val): - if self.read_only: - self._read_only_error() - key = _normalize(key) if isinstance(key, tuple): @@ -348,10 +354,8 @@ def __setitem__(self, key, val): else: raise ValueError("{} is not a valid key, must be str or int".format(key)) + @sentinel def __delitem__(self, key): - if self.read_only: - self._read_only_error() - key = _normalize(key) if isinstance(key, tuple): @@ -388,6 +392,7 @@ def __getattr__(self, name): def _is_class_var(cls, name): return any(name in c.__dict__ for c in cls.__mro__) + @sentinel def __setattr__(self, name, val): # Search instance variables (self.__dict___) and class variables # (self.__class__.__dict__ + iterating over mro to find variables on @@ -399,6 +404,7 @@ def __setattr__(self, name, val): else: self[name] = val + @sentinel def __delattr__(self, name): # see __setattr__ if name in self.__dict__ or self._is_class_var(name): @@ -430,28 +436,6 @@ def __repr__(self): else: return name + "([" + ", ".join("{!r}".format(v) for v in self._store) + "])" - @property - def read_only(self): - """ - bool: if set, raise warning when attempts are made to modify the container - """ - return self._read_only - - @read_only.setter - def read_only(self, val): - # can't mark a read-only list as writeable - if self._read_only and not val: - self._read_only_error() - else: - self._read_only = bool(val) - - @classmethod - def _read_only_error(cls): - warnings.warn( - "The input in {} changed, while the state of the job was already " - "finished.".format(cls.__name__) - ) - def to_builtin(self, stringify=False): """ Convert the container back to builtin dict's and list's recursively. @@ -597,6 +581,7 @@ def _wrap_val(cls, val, blacklist): else: return val + @sentinel def update(self, init, wrap=False, blacklist=(), **kwargs): """ Add all elements or key-value pairs from init to this container. If wrap is @@ -640,6 +625,7 @@ def update(self, init, wrap=False, blacklist=(), **kwargs): else: super().update(init, **kwargs) + @sentinel def append(self, val): """ Add new value to the container without a key. @@ -649,6 +635,7 @@ def append(self, val): """ self._store.append(val) + @sentinel def extend(self, vals): """ Append vals to the end of this DataContainer. @@ -660,6 +647,7 @@ def extend(self, vals): for v in vals: self.append(v) + @sentinel def insert(self, index, val, key=None): """ Add a new element to the container at the specified position, with an optional @@ -680,6 +668,7 @@ def insert(self, index, val, key=None): self._store.insert(index, val) + @sentinel def mark(self, index, key): """ Add a key to an existing item at index. If key already exists, it is @@ -706,6 +695,7 @@ def mark(self, index, key): self._indices[key] = index + @sentinel def clear(self): """ Remove all items from DataContainer. @@ -713,6 +703,7 @@ def clear(self): self._store.clear() self._indices.clear() + @sentinel def create_group(self, name): """ Add a new empty subcontainer under the given key. @@ -884,6 +875,7 @@ def groups(self): def _list_groups(self): return list(self.groups()) + @sentinel def read(self, file_name, wrap=True): """ Parse file as dictionary and add its keys to this container. @@ -927,6 +919,11 @@ def _force_load(self, recursive=True): if recursive and isinstance(v, DataContainer): v._force_load() + # Lockable overload + def _on_unlock(self): + warnings.warn("Unlock previously locked object!") + super()._on_unlock() + def __init_subclass__(cls): # called whenever a subclass of DataContainer is defined, then register all subclasses with the same function # that the DataContainer is registered diff --git a/pyproject.toml b/pyproject.toml index 1231fc790b..50c5b2bc0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,11 +30,11 @@ dependencies = [ "h5py==3.10.0", "jinja2==3.1.2", "numpy==1.26.2", - "pandas==2.1.3", + "pandas==2.1.4", "pint==0.23", - "psutil==5.9.5", - "pyfileindex==0.0.18", - "pysqa==0.1.7", + "psutil==5.9.7", + "pyfileindex==0.0.19", + "pysqa==0.1.8", "sqlalchemy==2.0.23", "tables==3.9.2", "tqdm==4.66.1", diff --git a/tests/generic/test_datacontainer.py b/tests/generic/test_datacontainer.py index 2056ff8798..cf8de9a9d7 100644 --- a/tests/generic/test_datacontainer.py +++ b/tests/generic/test_datacontainer.py @@ -313,7 +313,8 @@ def test_del_item(self): def test_del_attr(self): class SubDataContainer(DataContainer): - def __init__(self): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) object.__setattr__(self, "attr", 42) s = SubDataContainer() del s.attr