diff --git a/README.md b/README.md index 683d669..6f474a5 100644 --- a/README.md +++ b/README.md @@ -8,19 +8,32 @@ A web interface to the superman tools. Starting from a fresh download of the source files, a few steps are required before starting the server for the first time. -### 1: Install Dependencies +Easy Guide: + + mkdir superman + cd superman + git clone https://github.com/all-umass/superman-web + git clone https://github.com/all-umass/superman + cd superman + pip3 install -e . + cd ../superman-web + pip3 install matplotlib tornado==4.4.2 pyyaml h5py pandas pywt sklearn + -Python (2.7 or 3.4+) is the main requirement for running the server. -Several Python packages are needed, available from PyPI via `pip`: +### 1: Install Dependencies - pip install --only-binary :all: superman matplotlib tornado pyyaml h5py pandas +While 2.7 and 3.4+ are supported, I've only tested with 3.4+ -If you're not running Linux, `superman` may require special care to install. -See [the superman docs](https://github.com/all-umass/superman#installation) for instructions. +Python (3.4+) is the main requirement for running the server. +Several Python packages are needed, available from PyPI via `pip`: -For running tests, you'll want: + pip3 install matplotlib tornado==4.4.2 pyyaml h5py pandas pywt sklearn + +It will complain about `xylib` and `metakit`, but will only disable the ability to parse specific file types. +Neither of these packages are available on pip3 at the moment. - pip install pytest mock coverage +Make sure that you have set up your superman repo and installed it. +[Superman docs](https://github.com/all-umass/superman#installation) ### 2: Configure @@ -33,7 +46,7 @@ In the same way, copy `datasets-template.yml` to `datasets.yml` and update the listings to match your local datasets. -### 3: Add Datasets +### 3: Add Datasets (Optional) Datasets are the basic unit of data in the superman server. Add one by modifying the `datasets.yml` configuration file, @@ -54,16 +67,22 @@ with any currently running server. Or simply run the server directly, and handle the details yourself: - python superman_server.py + python3 superman_server.py To stop the server without restarting it, use: ./restart_server.sh --kill +### 5: Testing (Optional) + +For running tests, you'll want: + + pip3 install pytest mock coverage + If you want to verify that everything is working as intended, try running the test suite (located in the `test/` directory): - python -m pytest + python3 -m pytest To generate a nice code coverage report: diff --git a/backend/dataset_loaders.py b/backend/dataset_loaders.py index 31a43b1..aa92b07 100644 --- a/backend/dataset_loaders.py +++ b/backend/dataset_loaders.py @@ -18,42 +18,43 @@ def load_datasets(config_fh, custom_loaders, public_only=False, user_added=False config = yaml.safe_load(config_fh) for kind, entries in config.items(): - for name, info in entries.items(): - # skip this entry if it shouldn't be included - is_public = info.get('public', True) - if public_only and not is_public: - continue - - if 'files' in info: - files = info['files'] - else: - files = [info['file']] - - if 'loader' in info: - # look up the loader function from the module namespace - loader_fn = getattr(custom_loaders, info['loader']) - else: - # construct a loader from the meta_mapping and the default template - meta_mapping = [(k, getattr(web_datasets, cls), mname) - for k, cls, mname in info.get('metadata', [])] - if info.get('vector', False): - loader_fn = _generic_vector_loader(meta_mapping) + if entries: + for name, info in entries.items(): + # skip this entry if it shouldn't be included + is_public = info.get('public', True) + if public_only and not is_public: + continue + + if 'files' in info: + files = info['files'] else: - loader_fn = _generic_traj_loader(meta_mapping) + files = [info['file']] - if kind == 'LIBS': - ds = WebLIBSDataset(name, loader_fn, *files) - elif info.get('vector', False): - ds = WebVectorDataset(name, kind, loader_fn, *files) - else: - ds = WebTrajDataset(name, kind, loader_fn, *files) - - if 'description' in info: - ds.description = info['description'] - if 'urls' in info: - ds.urls = info['urls'] - ds.is_public = is_public - ds.user_added = user_added + if 'loader' in info: + # look up the loader function from the module namespace + loader_fn = getattr(custom_loaders, info['loader']) + else: + # construct a loader from the meta_mapping and the default template + meta_mapping = [(k, getattr(web_datasets, cls), mname) + for k, cls, mname in info.get('metadata', [])] + if info.get('vector', False): + loader_fn = _generic_vector_loader(meta_mapping) + else: + loader_fn = _generic_traj_loader(meta_mapping) + + if kind == 'LIBS': + ds = WebLIBSDataset(name, loader_fn, *files) + elif info.get('vector', False): + ds = WebVectorDataset(name, kind, loader_fn, *files) + else: + ds = WebTrajDataset(name, kind, loader_fn, *files) + + if 'description' in info: + ds.description = info['description'] + if 'urls' in info: + ds.urls = info['urls'] + ds.is_public = is_public + ds.user_added = user_added def try_load(filepath, data_name): diff --git a/restart_server.sh b/restart_server.sh index ba7cedd..b7d1b7f 100755 --- a/restart_server.sh +++ b/restart_server.sh @@ -2,7 +2,9 @@ # Parse result of geoiplookup to fit concisely on a line. function ip_info() { - geoiplookup $1 | sed 1d | cut -d: -f2 | cut -d' ' -f3- | cut -d, -f2,3 | xargs + geoiplookup $1 | sed 1d | cut -d: -f2 | \ + cut -d' ' -f3- | cut -d, -f2,3 | \ + tr "'" '^' | xargs } export -f ip_info @@ -12,7 +14,7 @@ function find_server_pid() { function start_server() { echo "Starting new server..." - nohup python superman_server.py &>logs/errors.out & + nohup python3 superman_server.py &>logs/errors.out & $follow_log || echo "Use 'tail -f logs/server.log' to check on it" sleep 1 if [[ -z "$(find_server_pid)" ]]; then