Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Constraints demo #362

Draft
wants to merge 3 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 20 additions & 5 deletions datalad_metalad/dump.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,18 @@

from datalad.distribution.dataset import datasetmethod
from datalad.interface.base import (
Interface,
build_doc,
eval_results,
)
from datalad.support.constraints import (
EnsureNone,
from datalad_next.commands import (
EnsureCommandParameterization,
ValidatedInterface,
)
from datalad_next.constraints import (
EnsureStr,
EnsureNone,
)
from datalad_next.constraints.dataset import EnsureDataset
from datalad.support.param import Parameter
from datalad.ui import ui
from dataladmetadatamodel.datasettree import datalad_root_record_name
Expand Down Expand Up @@ -413,7 +417,7 @@ def dump_from_uuid_set(mapper: str,


@build_doc
class Dump(Interface):
class Dump(ValidatedInterface):
"""Dump a dataset's aggregated metadata for dataset and file metadata
Two types of metadata are supported:
Expand All @@ -432,6 +436,16 @@ class Dump(Interface):
(The tree-format is the default format and does not require a prefix).
"""

# Define parameter constraints
_validator_ = EnsureCommandParameterization(
param_constraints=dict(
dataset=EnsureDataset(installed=True, purpose='meta-dump'),
path=EnsureStr(),
),
validate_defaults=("dataset",)
)


# Use a custom renderer to emit a self-contained metadata record. The
# emitted record can be fed into meta-add for example.
result_renderer = 'tailored'
Expand Down Expand Up @@ -481,7 +495,6 @@ class Dump(Interface):
args=("path",),
metavar="DATASET_FILE_PATH_PATTERN",
doc="path to query metadata for",
constraints=EnsureStr() | EnsureNone(),
nargs="?"),
recursive=Parameter(
args=("-r", "--recursive",),
Expand All @@ -501,6 +514,8 @@ def __call__(
path="",
recursive=False):

# dataset is a DatasetParameter() from datalad-next
dataset = dataset.ds.path
metadata_store_path, tree_version_list, uuid_set = get_metadata_objects(
dataset,
default_mapper_family)
Expand Down
53 changes: 38 additions & 15 deletions datalad_metalad/extract.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,22 @@
from uuid import UUID

from dataclasses import dataclass

from datalad_next.commands import (
EnsureCommandParameterization,
ValidatedInterface,
)
from datalad.distribution.dataset import Dataset
from datalad.distribution.dataset import (
datasetmethod,
EnsureDataset,
from datalad.distribution.dataset import datasetmethod
from datalad_next.constraints import (
EnsurePath,
EnsureStr,
EnsureMapping,
EnsureNone,
EnsureListOf,
NoConstraint,
)
from datalad_next.constraints.dataset import EnsureDataset
from datalad.interface.base import (
Interface,
build_doc,
eval_results,
)
Expand All @@ -54,10 +62,6 @@
MetadataExtractorBase,
)

from datalad.support.constraints import (
EnsureNone,
EnsureStr,
)
from datalad.support.param import Parameter

from dataladmetadatamodel.metadatapath import MetadataPath
Expand Down Expand Up @@ -91,7 +95,7 @@ class ExtractionArguments:


@build_doc
class Extract(Interface):
class Extract(ValidatedInterface):
"""Run a metadata extractor on a dataset or file.

This command distinguishes between dataset-level extraction and
Expand Down Expand Up @@ -131,6 +135,23 @@ class Extract(Interface):
on the whether file-level- or dataset-level extraction is requested.
"""

# Define parameter constraints
extractorargs_constraints = EnsureMapping(key=EnsureStr(),
value=EnsureStr(),
delimiter='') | \
EnsureListOf(item_constraint=NoConstraint(),
min_len=0)
_validator_ = EnsureCommandParameterization(
param_constraints=dict(
path=EnsurePath(lexists=True),
dataset=EnsureDataset(installed=True, purpose='meta-extract'),
extractor=EnsureStr(),
extractorargs=extractorargs_constraints,
),
validate_defaults=("dataset",),
tailor_for_dataset=dict(path="dataset")
)

result_renderer = "tailored"

_examples_ = [
Expand Down Expand Up @@ -180,20 +201,20 @@ class Extract(Interface):
specified.
You might provide an absolute file path, but it has to contain
the dataset path as prefix.""",
constraints=EnsureStr() | EnsureNone()),
),
dataset=Parameter(
args=("-d", "--dataset"),
doc="""Dataset to extract metadata from. If no dataset
is given, the dataset is determined by the current work
directory.""",
constraints=EnsureDataset() | EnsureNone()),
),
context=Parameter(
args=("-c", "--context"),
doc="""Context, a JSON-serialized dictionary that provides
constant data which has been gathered before, so meta-extract
will not have re-gather this data. Keys and values are strings.
meta-extract will look for the following key: 'dataset_version'.""",
constraints=EnsureDataset() | EnsureNone()),
),
get_context=Parameter(
args=("--get-context",),
action="store_true",
Expand All @@ -219,7 +240,8 @@ class Extract(Interface):
prevent interpretation of the key of the first extractor argument
as path for a file-level extraction.""",
nargs="*",
constraints=EnsureStr() | EnsureNone()))
)
)

@staticmethod
@datasetmethod(name="meta_extract")
Expand Down Expand Up @@ -247,7 +269,8 @@ def __call__(
if isinstance(context, str)
else context))

source_dataset = check_dataset(dataset or curdir, "extract metadata")
# dataset is a DatasetParameter from the parameter validation
source_dataset = dataset.ds
source_dataset_version = context.get("dataset_version", None)
if source_dataset_version is None:
source_dataset_version = source_dataset.repo.get_hexsha()
Expand Down
1 change: 1 addition & 0 deletions setup.cfg
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ install_requires =
datalad-metadata-model >=0.3.10
pytest
pyyaml
datalad-next @ git+https://github.com/mih/datalad-next@resolvepath#egg=datalad-next,
test_requires =
coverage
pytest
Expand Down