From 809427b88961f1362190500b91e4bc0fb1dc4906 Mon Sep 17 00:00:00 2001 From: Valerie Date: Tue, 24 Sep 2024 14:18:26 -0700 Subject: [PATCH] Prototype into Develop 9/19 (#26) Almanac, Exposure log, Narrative Log, backend classy, observation gaps, DDV Link, Accessing data from consdb to plot, etc Co-authored-by: Steve Pothier --- notebooks_tsqr/NightLog.ipynb | 809 +++++------------- notebooks_tsqr/NightLog.yaml | 16 +- notebooks_tsqr/README.md | 61 +- ..._logrep.yaml => TEMPLATE_logrep.HIDE_yaml} | 0 notebooks_tsqr/consdb/access_consdb.ipynb | 81 +- notebooks_tsqr/consdb/assorted_plots.ipynb | 158 ++++ notebooks_tsqr/consdb/assorted_plots.yaml | 18 + notebooks_tsqr/dashboard.ipynb | 135 +++ notebooks_tsqr/{efd.yaml => efd.HIDE_yaml} | 0 notebooks_tsqr/exposurelog.ipynb | 739 ---------------- notebooks_tsqr/narrativelog.ipynb | 731 ---------------- notebooks_tsqr/sources_dashboard.ipynb | 74 -- .../lsst/ts/logging_and_reporting/almanac.py | 81 ++ .../ts/logging_and_reporting/dashboard.py | 100 +++ .../ts/logging_and_reporting/exceptions.py | 78 ++ .../lsst/ts/logging_and_reporting/reports.py | 135 +++ .../logging_and_reporting/source_adapters.py | 589 +++++++------ python/lsst/ts/logging_and_reporting/utils.py | 54 ++ .../lsst/ts/logging_and_reporting/version.py | 2 +- requirements.txt | 8 +- times-square.yaml | 2 - 21 files changed, 1401 insertions(+), 2470 deletions(-) rename notebooks_tsqr/{TEMPLATE_logrep.yaml => TEMPLATE_logrep.HIDE_yaml} (100%) create mode 100644 notebooks_tsqr/consdb/assorted_plots.ipynb create mode 100644 notebooks_tsqr/consdb/assorted_plots.yaml create mode 100644 notebooks_tsqr/dashboard.ipynb rename notebooks_tsqr/{efd.yaml => efd.HIDE_yaml} (100%) delete mode 100644 notebooks_tsqr/exposurelog.ipynb delete mode 100644 notebooks_tsqr/narrativelog.ipynb delete mode 100644 notebooks_tsqr/sources_dashboard.ipynb create mode 100644 python/lsst/ts/logging_and_reporting/almanac.py create mode 100644 python/lsst/ts/logging_and_reporting/dashboard.py create mode 100644 python/lsst/ts/logging_and_reporting/exceptions.py create mode 100644 python/lsst/ts/logging_and_reporting/reports.py diff --git a/notebooks_tsqr/NightLog.ipynb b/notebooks_tsqr/NightLog.ipynb index e5657f3..332b0b5 100644 --- a/notebooks_tsqr/NightLog.ipynb +++ b/notebooks_tsqr/NightLog.ipynb @@ -5,7 +5,7 @@ "id": "0", "metadata": {}, "source": [ - "# Night Log" + "# Initialization" ] }, { @@ -15,11 +15,20 @@ "metadata": {}, "outputs": [], "source": [ - "# Parameters. Set defaults here.\n", + "# Parameters. \n", "# Times Square replaces this cell with the user's parameters.\n", - "record_limit = '999'\n", - "day_obs = 'TODAY' # TODAY, YESTERDAY, YYYY-MM-DD\n", - "number_of_days = '1' # Total number of days of data to display (ending on day_obs)" + "\n", + "# The run-time defaults for all of these parameters are in NightLog.yaml\n", + "# Under Times Square, the run-time defaults always override values given here.\n", + "# Values here are used for local tests.\n", + "\n", + "# day_obs values: TODAY, YESTERDAY, YYYY-MM-DD\n", + "# Report on observing nights that start upto but not included this day.\n", + "day_obs = '2024-09-04' # Value to use for local testing (Summit)\n", + "#!day_obs = 'TODAY' # TODO Change to 'TODAY' to test with default before push \n", + "\n", + "# Total number of days of data to display (ending on day_obs)\n", + "number_of_days = '1' # TODO Change to '1' to test with default before push " ] }, { @@ -33,12 +42,28 @@ "import requests\n", "from collections import defaultdict\n", "import pandas as pd\n", - "from pprint import pp\n", + "from pprint import pp, pformat\n", "from urllib.parse import urlencode\n", - "from IPython.display import display, Markdown, display_markdown\n", "from matplotlib import pyplot as plt\n", "import os\n", - "from datetime import datetime, date, timedelta" + "from datetime import datetime, date, timedelta\n", + "\n", + "# When running under Times Square, install pkg from github.\n", + "# Otherwise use what is installed locally (intended to be dev editiable pkg)\n", + "if os.environ.get('EXTERNAL_INSTANCE_URL'):\n", + " print('Installing \"lsst.ts.logging_and_reporting\" from github using \"prototype\" branch....')\n", + " !pip install --upgrade git+https://github.com/lsst-ts/ts_logging_and_reporting.git@prototype >/dev/null\n", + "import lsst.ts.logging_and_reporting.source_adapters as sad\n", + "import lsst.ts.logging_and_reporting.almanac as alm\n", + "import lsst.ts.logging_and_reporting.reports as rep \n", + "import lsst.ts.logging_and_reporting.utils as ut\n", + "from lsst.ts.logging_and_reporting.reports import md,mdlist\n", + "\n", + "try:\n", + " import lsst.ts.logging_and_reporting.version\n", + " lrversion = lsst.ts.logging_and_reporting.version.__version__\n", + "except:\n", + " lrversion = 'LIVE'" ] }, { @@ -48,33 +73,21 @@ "metadata": {}, "outputs": [], "source": [ - "limit = int(record_limit)\n", - "\n", - "match day_obs.lower():\n", - " case 'today':\n", - " date = datetime.now().date()\n", - " case 'yesterday':\n", - " date = datetime.now().date()-timedelta(days=1)\n", - " case _:\n", - " date = datetime.strptime(dd, '%Y-%m-%d').date()\n", + "# Normalize Parameters (both explicit Times Squares params, in implicit ones)\n", + "limit = 500 # YAGNI for Auto get more if this isn't enough to get all requested DAYS\n", "\n", + "date = ut.get_datetime_from_day_obs_str(day_obs)\n", + "# date: is EXLUSIVE (upto, but not including)\n", "days = int(number_of_days)\n", "\n", "# Thus: [min_day_obs,max_day_obs)\n", - "min_day_obs = (date - timedelta(days=days-1)).strftime('%Y%m%d') # Inclusive\n", - "max_day_obs = (date + timedelta(days=1)).strftime('%Y%m%d') # prep for Exclusive\n", + "# Format: string, YYYY-MM-DD\n", + "min_day_obs = (date - timedelta(days=days-1)).strftime('%Y-%m-%d') # Inclusive\n", + "max_day_obs = (date + timedelta(days=1)).strftime('%Y-%m-%d') # prep for Exclusive\n", "\n", "response_timeout = 3.05 # seconds, how long to wait for connection\n", "read_timeout = 20 # seconds\n", - "timeout = (float(response_timeout), float(read_timeout))\n", - "\n", - "summit = 'https://summit-lsp.lsst.codes'\n", - "usdf = 'https://usdf-rsp-dev.slac.stanford.edu'\n", - "tucson = 'https://tucson-teststand.lsst.codes'\n", - "\n", - "# Use server=tucson for dev testing.\n", - "# Use server=usdf for push to develop. TODO\n", - "server = os.environ.get('EXTERNAL_INSTANCE_URL', usdf)" + "timeout = (float(response_timeout), float(read_timeout))" ] }, { @@ -84,607 +97,199 @@ "metadata": {}, "outputs": [], "source": [ - "print(f'Report from {server} over {number_of_days} nights'\n", - " f' from {min_day_obs} to {date}. ')" + "# Set default env to \"usdf\" and try before PUSH to repo.\n", + "summit = 'https://summit-lsp.lsst.codes'\n", + "usdf = 'https://usdf-rsp-dev.slac.stanford.edu'\n", + "tucson = 'https://tucson-teststand.lsst.codes'\n", + "\n", + "# The default provided here is for local testing.\n", + "# Under Times Square it is ignored.\n", + "server = os.environ.get('EXTERNAL_INSTANCE_URL', usdf) # TODO try with \"usdf\" before push (else \"summit\")" + ] + }, + { + "cell_type": "markdown", + "id": "5", + "metadata": {}, + "source": [ + "# Overview" + ] + }, + { + "cell_type": "markdown", + "id": "6", + "metadata": {}, + "source": [ + "The only environment that has everything needed for this page is\n", + "https://summit-lsp.lsst.codes\n", + "\n", + "However, Times Square **does not** run on the Summit. It **does** run on USDF-dev. USDF doesn't fully support all the data sources we need so some functionality is currently missing on this page." ] }, { "cell_type": "code", "execution_count": null, - "id": "5", + "id": "7", "metadata": {}, "outputs": [], "source": [ - "# For Times Square, comment out next line and past next cell with contents of local python file.\n", - "#! from lsst.ts.logging_and_reporting.source_adapters import ExposurelogAdapter, NarrativelogAdapter, keep_fields\n", - "# Once our logrep package has been installed in RSP, we can use the simpler \"import\"" + "# Display overview of Report context \n", + "md(f'''\n", + "Report for **{date}** covering the previous **{days}** observing night(s).\n", + "- Run on logs from **{server}/**\n", + "- Using *Prototype* Logging and Reporting Version: **{lrversion}**\n", + "''')\n", + "\n", + "endpoint_urls_str = '\\n- '.join(['',*sad.all_endpoints(server)])\n", + "md(f'This report will attempt to use the following log sources: {endpoint_urls_str}')" + ] + }, + { + "cell_type": "markdown", + "id": "8", + "metadata": {}, + "source": [ + "# DDV" ] }, { "cell_type": "code", "execution_count": null, - "id": "6", + "id": "9", "metadata": {}, "outputs": [], "source": [ - "# TODO\n", - "# Comment out the import in the cell above this one.\n", - "# Paste contents of source_adapters.py in new cell below this one." + "DDV = f\"{server}/rubintv-dev/ddv/index.html\" if 'summit' in server else f\"{server}/rubintv/ddv/index.html\"\n", + "md(f'Access DDV part of RubinTV: {DDV}')" + ] + }, + { + "cell_type": "markdown", + "id": "10", + "metadata": {}, + "source": [ + "# Almanac" ] }, { "cell_type": "code", "execution_count": null, - "id": "7", + "id": "11", "metadata": {}, "outputs": [], "source": [ - "# This file is part of ts_logging_and_reporting.\n", - "#\n", - "# Developed for Vera C. Rubin Observatory Telescope and Site Systems.\n", - "# This product includes software developed by the LSST Project\n", - "# (https://www.lsst.org).\n", - "# See the COPYRIGHT file at the top-level directory of this distribution\n", - "# for details of code ownership.\n", - "#\n", - "# This program is free software: you can redistribute it and/or modify\n", - "# it under the terms of the GNU General Public License as published by\n", - "# the Free Software Foundation, either version 3 of the License, or\n", - "# (at your option) any later version.\n", - "#\n", - "# This program is distributed in the hope that it will be useful,\n", - "# but WITHOUT ANY WARRANTY; without even the implied warranty of\n", - "# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n", - "# GNU General Public License for more details.\n", - "#\n", - "# You should have received a copy of the GNU General Public License\n", - "# along with this program. If not, see .\n", - "\n", - "\n", - "############################################\n", - "# Python Standard Library\n", - "from urllib.parse import urlencode\n", - "import itertools\n", - "from datetime import datetime\n", - "from warnings import warn\n", - "from collections import defaultdict\n", - "from abc import ABC\n", - "\n", - "############################################\n", - "# External Packages\n", - "import requests\n", - "\n", - "MAX_CONNECT_TIMEOUT = 3.1 # seconds\n", - "MAX_READ_TIMEOUT = 90 * 60 # seconds\n", - "\n", - "def keep_fields(outfields, recs):\n", - " \"\"\"Keep only keys in OUTFIELDS list of RECS (list of dicts)\n", - " SIDE EFFECT: Removes extraneous keys from all dicts in RECS.\n", - " \"\"\"\n", - " if outfields:\n", - " for rec in recs:\n", - " nukefields = set(rec.keys()) - set(outfields)\n", - " print(f'{rec=} {nukefields=}')\n", - " for f in nukefields:\n", - " del rec[f]\n", - "\n", - "class SourceAdapter(ABC):\n", - " \"\"\"Abstract Base Class for all source adapters.\n", - " \"\"\"\n", - " # TODO document class including all class variables.\n", - " def __init__(self, *,\n", - " server_url='https://tucson-teststand.lsst.codes',\n", - " connect_timeout=1.05, # seconds\n", - " read_timeout=2, # seconds\n", - " ):\n", - " self.server = server_url\n", - " self.c_timeout = min(MAX_CONNECT_TIMEOUT,\n", - " float(connect_timeout)) # seconds\n", - " self.r_timeout = min(MAX_READ_TIMEOUT, # seconds\n", - " float(read_timeout))\n", - " self.timeout = (self.c_timeout, self.r_timeout)\n", - "\n", - " # Provide the following in subclass\n", - " output_fields = None\n", - " service = None\n", - " endpoints = None\n", - "\n", - " @property\n", - " def source_url(self):\n", - " return f'{self.server}/{self.service}'\n", - "\n", - "\n", - " def check_endpoints(self, timeout=None):\n", - " to = (timeout or self.timeout)\n", - " print(f'Try connect to each endpoint of {self.server}/{self.service} '\n", - " f'using timeout={to}.')\n", - " url_http_status_code = dict()\n", - " for ep in self.endpoints:\n", - " url = f'{self.server}/{self.service}/{ep}'\n", - " try:\n", - " r = requests.get(url, timeout=(timeout or self.timeout))\n", - " except:\n", - " url_http_status_code[url] = 'timeout'\n", - " else:\n", - " url_http_status_code[url] = r.status_code\n", - " return url_http_status_code\n", - "\n", - "\n", - " def analytics(self, recs, categorical_fields=None):\n", - " if len(recs) == 0:\n", - " return dict(fields=[],\n", - " facet_fields=set(),\n", - " facets=dict())\n", - "\n", - " non_cats = set([\n", - " 'tags', 'urls', 'message_text', 'id', 'date_added',\n", - " 'obs_id', 'day_obs', 'seq_num', 'parent_id', 'user_id',\n", - " 'date_invalidated', 'date_begin', 'date_end',\n", - " 'time_lost', # float\n", - " # 'systems','subsystems','cscs', # values need special handling\n", - " ])\n", - " flds = set(recs[0].keys())\n", - " if not categorical_fields:\n", - " categorical_fields = flds\n", - " ignore_fields = flds - categorical_fields\n", - " facflds = flds - ignore_fields\n", - "\n", - " # facets(field) = set(value-1, value-2, ...)\n", - " facets = {fld: set([str(r[fld])\n", - " for r in recs if not isinstance(r[fld], list)])\n", - " for fld in facflds}\n", - " return dict(fields=flds,\n", - " facet_fields=facflds,\n", - " facets=facets,\n", - " )\n", - "\n", - "\n", - "# Not available on SLAC (usdf) as of 9/9/2024.\n", - "class NightReportAdapter(SourceAdapter):\n", - " service = \"nightreport\"\n", - " endpoints = ['reports']\n", - " primary_endpoint = 'reports'\n", - "\n", - "class NarrativelogAdapter(SourceAdapter):\n", - " \"\"\"TODO full documentation\n", - " \"\"\"\n", - " service = 'narrativelog'\n", - " endpoints = ['messages',]\n", - " primary_endpoint = 'messages'\n", - " fields = {'category',\n", - " 'components',\n", - " 'cscs',\n", - " 'date_added',\n", - " 'date_begin',\n", - " 'date_end',\n", - " 'date_invalidated',\n", - " 'id',\n", - " 'is_human',\n", - " 'is_valid',\n", - " 'level',\n", - " 'message_text',\n", - " 'parent_id',\n", - " 'primary_hardware_components',\n", - " 'primary_software_components',\n", - " 'site_id',\n", - " 'subsystems',\n", - " 'systems',\n", - " 'tags',\n", - " 'time_lost',\n", - " 'time_lost_type',\n", - " 'urls',\n", - " 'user_agent',\n", - " 'user_id'}\n", - " filters = {\n", - " 'site_ids',\n", - " 'message_text', # Message text contain ...\n", - " 'min_level', # inclusive\n", - " 'max_level', # exclusive\n", - " 'user_ids',\n", - " 'user_agents',\n", - " 'categories',\n", - " 'exclude_categories',\n", - " 'time_lost_types',\n", - " 'exclude_time_lost_types',\n", - " 'tags', # at least one must be present.\n", - " 'exclude_tags', # all must be absent\n", - " 'systems',\n", - " 'exclude_systems',\n", - " 'subsystems',\n", - " 'exclude_subsystems',\n", - " 'cscs',\n", - " 'exclude_cscs',\n", - " 'components',\n", - " 'exclude_components',\n", - " 'primary_software_components',\n", - " 'exclude_primary_software_components',\n", - " 'primary_hardware_components',\n", - " 'exclude_primary_hardware_components',\n", - " 'urls',\n", - " 'min_time_lost',\n", - " 'max_time_lost',\n", - " 'has_date_begin',\n", - " 'min_date_begin',\n", - " 'max_date_begin',\n", - " 'has_date_end',\n", - " 'min_date_end',\n", - " 'max_date_end',\n", - " 'is_human', # Allowed: either, true, false; Default=either\n", - " 'is_valid', # Allowed: either, true, false; Default=true\n", - " 'min_date_added', # inclusive, TAI ISO string, no TZ\n", - " 'max_date_added', # exclusive, TAI ISO string, no TZ\n", - " 'has_date_invalidated',\n", - " 'min_date_invalidated',\n", - " 'max_date_invalidated',\n", - " 'has_parent_id',\n", - " 'order_by',\n", - " 'offset',\n", - " 'limit'\n", - " }\n", - "\n", - " def get_messages(self,\n", - " site_ids=None,\n", - " message_text=None,\n", - " min_date_end=None,\n", - " max_date_end=None,\n", - " is_human='either',\n", - " is_valid='either',\n", - " offset=None,\n", - " limit=None,\n", - " outfields=None,\n", - " ):\n", - " qparams = dict(is_human=is_human, is_valid=is_valid)\n", - " if site_ids:\n", - " qparams['site_ids'] = site_ids\n", - " if message_text:\n", - " qparams['message_text'] = message_text\n", - " if min_date_end:\n", - " qparams['min_date_end'] = min_date_end\n", - " if max_date_end:\n", - " qparams['max_date_end'] = max_date_end\n", - " if limit:\n", - " qparams['limit'] = limit\n", - "\n", - " qstr = urlencode(qparams)\n", - " url = f'{self.server}/{self.service}/messages?{qstr}'\n", - " try:\n", - " recs = requests.get(url, timeout=self.timeout).json()\n", - " recs.sort(key=lambda r: r['date_begin'])\n", - " except Exception as err:\n", - " warn(f'No {self.service} records retrieved: {err}')\n", - " recs = []\n", - "\n", - " keep_fields(outfields, recs)\n", - " self.recs = recs\n", - " return self.recs\n", - "\n", - " def get_timelost(self, rollup='day'):\n", - " day_tl = dict() # day_tl[day] = totalDayTimeLost\n", - " for day,dayrecs in itertools.groupby(\n", - " self.recs,\n", - " key=lambda r: datetime.fromisoformat(r['date_begin']).date().isoformat()\n", - " ):\n", - " day_tl[day] = sum([r['time_lost'] for r in dayrecs])\n", - " return day_tl\n", - "\n", - "class ExposurelogAdapter(SourceAdapter):\n", - " \"\"\"TODO full documentation\n", - "\n", - " EXAMPLES:\n", - " gaps,recs = logrep_utils.ExposurelogAdapter(server_url='https://usdf-rsp-dev.slac.stanford.edu').get_observation_gaps('LSSTComCam')\n", - " gaps,recs = logrep_utils.ExposurelogAdapter(server_url='[[https://tucson-teststand.lsst.codes').get_observation_gaps('LSSTComCam')\n", - " \"\"\"\n", - " ignore_fields = ['id']\n", - " service = 'exposurelog'\n", - " endpoints = [\n", - " 'instruments',\n", - " 'exposures',\n", - " 'messages',\n", - " ]\n", - " primary_endpoint = 'messages'\n", - " fields = {'date_added',\n", - " 'date_invalidated',\n", - " 'day_obs',\n", - " 'exposure_flag',\n", - " 'id',\n", - " 'instrument',\n", - " 'is_human',\n", - " 'is_valid',\n", - " 'level',\n", - " 'message_text',\n", - " 'obs_id',\n", - " 'parent_id',\n", - " 'seq_num',\n", - " 'site_id',\n", - " 'tags',\n", - " 'urls',\n", - " 'user_agent',\n", - " 'user_id'}\n", - " filters = {\n", - " 'site_ids',\n", - " 'obs_id',\n", - " 'instruments',\n", - " 'min_day_obs', # inclusive, integer in form YYYMMDD\n", - " 'max_day_obs', # exclusive, integer in form YYYMMDD\n", - " 'min_seq_num',\n", - " 'max_seq_num',\n", - " 'message_text', # Message text contain ...\n", - " 'min_level', # inclusive\n", - " 'max_level', # exclusive\n", - " 'tags', # at least one must be present.\n", - " 'urls',\n", - " 'exclude_tags', # all must be absent\n", - " 'user_ids',\n", - " 'user_agents',\n", - " 'is_human', # Allowed: either, true, false; Default=either\n", - " 'is_valid', # Allowed: either, true, false; Default=true\n", - " 'exposure_flags',\n", - " 'min_date_added', # inclusive, TAI ISO string, no TZ\n", - " 'max_date_added', # exclusive, TAI ISO string, no TZ\n", - " 'has_date_invalidated',\n", - " 'min_date_invalidated',\n", - " 'max_date_invalidated',\n", - " 'has_parent_id',\n", - " 'order_by',\n", - " 'offset',\n", - " 'limit'\n", - " }\n", - "\n", - "\n", - " def check_endpoints(self, timeout=None):\n", - " to = (timeout or self.timeout)\n", - " print(f'Try connect to each endpoint of {self.server}/{self.service} '\n", - " f'using timeout={to}.')\n", - " url_http_status_code = dict()\n", - "\n", - " for ep in self.endpoints:\n", - " qstr = '?instrument=na' if ep == 'exposures' else ''\n", - " url = f'{self.server}/{self.service}/{ep}{qstr}'\n", - " try:\n", - " r = requests.get(url, timeout=to)\n", - " except:\n", - " url_http_status_code[url] = 'timeout'\n", - " else:\n", - " url_http_status_code[url] = r.status_code\n", - " return url_http_status_code\n", - "\n", - "\n", - " def get_instruments(self):\n", - " url = f'{self.server}/{self.service}/instruments'\n", - " try:\n", - " instruments = requests.get(url, timeout=self.timeout).json()\n", - " except Exception as err:\n", - " warn(f'No instruments retrieved: {err}')\n", - " instruments = dict(dummy=[])\n", - " # Flatten the lists\n", - " return list(itertools.chain.from_iterable(instruments.values()))\n", - "\n", - " def get_exposures(self, instrument, registry=1):\n", - " qparams = dict(instrument=instrument, registery=registry)\n", - " url = f'{self.server}/{self.service}/exposures?{urlencode(qparams)}'\n", - " try:\n", - " recs = requests.get(url, timeout=self.timeout).json()\n", - " except Exception as err:\n", - " warn(f'No exposures retrieved: {err}')\n", - " recs = []\n", - " return recs\n", - "\n", - " def get_messages(self,\n", - " site_ids=None,\n", - " obs_ids=None,\n", - " instruments=None,\n", - " message_text=None,\n", - " min_day_obs=None,\n", - " max_day_obs=None,\n", - " is_human='either',\n", - " is_valid='either',\n", - " exposure_flags=None,\n", - " offset=None,\n", - " limit=None,\n", - " outfields=None,\n", - " ):\n", - " qparams = dict(is_human=is_human, is_valid=is_valid)\n", - " if site_ids:\n", - " qparams['site_ids'] = site_ids\n", - " if obs_ids:\n", - " qparams['obs_ids'] = obs_ids\n", - " if instruments:\n", - " qparams['instruments'] = instruments\n", - " if min_day_obs:\n", - " qparams['min_day_obs'] = min_day_obs\n", - " if max_day_obs:\n", - " qparams['max_day_obs'] = max_day_obs\n", - " if exposure_flags:\n", - " qparams['exposure_flags'] = exposure_flags\n", - " if offset:\n", - " qparams['offset'] = offset\n", - " if limit:\n", - " qparams['limit'] = limit\n", - "\n", - " qstr = urlencode(qparams)\n", - " url = f'{self.server}/{self.service}/messages?{qstr}'\n", - " recs = []\n", - " try:\n", - " response = requests.get(url, timeout=self.timeout)\n", - " recs = response.json()\n", - " except Exception as err:\n", - " warnings.warn(f'No {self.service} records retrieved: {err}')\n", - "\n", - " if len(recs) == 0:\n", - " warn(f'No records retrieved from {url}')\n", - "\n", - " if recs:\n", - " recs.sort(key=lambda r: r['day_obs'])\n", - "\n", - " keep_fields(outfields, recs)\n", - " self.recs = recs\n", - " return self.recs\n", - "\n", - " def get_observation_gaps(self, instruments=None,\n", - " min_day_obs=None, # YYYYMMDD\n", - " max_day_obs=None, # YYYYMMDD\n", - " ):\n", - " if not instruments:\n", - " instruments = self.get_instruments()\n", - " assert isinstance(instruments,list), \\\n", - " f'\"instruments\" must be a list. Got {instruments!r}'\n", - " # inst_day_rollupol[instrument] => dict[day] => exposureGapInMinutes\n", - " inst_day_rollup = defaultdict(dict) # Instrument/Day rollup\n", - "\n", - " for instrum in instruments:\n", - " recs = self.get_exposures(instrum)\n", - " instrum_gaps = dict()\n", - " for day,dayrecs in itertools.groupby(recs,\n", - " key=lambda r: r['day_obs']):\n", - " gaps = list()\n", - " begin = end = None\n", - " for rec in dayrecs:\n", - " begin = rec['timespan_begin']\n", - " if end:\n", - " # span in minutes\n", - " diff = (datetime.fromisoformat(begin)\n", - " - datetime.fromisoformat(end)\n", - " ).total_seconds() / 60.0\n", - "\n", - " gaps.append((\n", - " datetime.fromisoformat(end).time().isoformat(),\n", - " datetime.fromisoformat(begin).time().isoformat(),\n", - " diff\n", - " ))\n", - " end = rec['timespan_end']\n", - " instrum_gaps[day] = gaps\n", - "\n", - " #!roll = dict()\n", - " # Rollup gap times by day\n", - " for day,tuples in instrum_gaps.items():\n", - " #!roll[day] = sum([t[2] for t in tuples])\n", - " inst_day_rollup[instrum][day] = sum([t[2] for t in tuples])\n", - "\n", - " return inst_day_rollup\n", - "\n", - "\n", - "\n", - "\n", - "class Dashboard: # TODO Complete and move to its own file.\n", - " \"\"\"Verify that we can get to all the API endpoints and databases we need for\n", - " any of our sources.\n", - " \"\"\"\n", - "\n", - " envs = dict(\n", - " summit = 'https://summit-lsp.lsst.codes',\n", - " usdf_dev = 'https://usdf-rsp-dev.slac.stanford.edu',\n", - " tucson = 'https://tucson-teststand.lsst.codes',\n", - " # Environments not currently used:\n", - " # rubin_usdf_dev = '',\n", - " # data_lsst_cloud = '',\n", - " # usdf = '',\n", - " # base_data_facility = '',\n", - " # rubin_idf_int = '',\n", - " )\n", - " adapters = [ExposurelogAdapter,\n", - " NarrativelogAdapter,\n", - " # NightReportAdapter, # TODO\n", - " ]\n", - "\n", - " def report(self, timeout=None):\n", - " url_status = dict()\n", - " for env,server in self.envs.items():\n", - " for adapter in self.adapters:\n", - " service = adapter(server_url=server)\n", - " # url_status[endpoint_url] = http_status_code\n", - " url_status.update(service.check_endpoints(timeout=timeout))\n", - " total_cnt = 0\n", - " good_cnt = 0\n", - " good = list()\n", - " print('\\nStatus for each endpoint URL:')\n", - " for url,stat in url_status.items():\n", - " print(f'{stat}\\t{url}')\n", - " total_cnt += 1\n", - " if stat == 200:\n", - " good_cnt += 1\n", - " good.append(url)\n", - " print(f'\\nConnected to {good_cnt} out of {total_cnt} endpoints.')\n", - " return good_cnt, good\n" + "# Display various almanac values (for moon, sun)\n", + "rep.AlmanacReport().almanac_as_dataframe()" ] }, { "cell_type": "markdown", - "id": "8", + "id": "12", "metadata": {}, "source": [ - "# Exposure Log" + "# Night Report" ] }, { "cell_type": "code", "execution_count": null, - "id": "9", + "id": "13", "metadata": {}, "outputs": [], "source": [ - "exposure_adapter = ExposurelogAdapter(server_url=server)\n", - "exposure_url = exposure_adapter.source_url\n", - "try:\n", - " exposure_recs = exposure_adapter.get_messages(limit=limit,\n", - " min_day_obs=min_day_obs,\n", - " max_day_obs=max_day_obs,\n", - " )\n", - "except Exception as err:\n", - " exposure_recs = []\n", - " msg = f'ERROR getting records from {exposure_url=}: {err=}'\n", - " raise Exception(msg)" + "# Get data from Night Report log. Display nightly Jira BLOCKS.\n", + "nr_adapter = sad.NightReportAdapter(server_url=server,\n", + " limit=limit,\n", + " min_day_obs=min_day_obs,\n", + " max_day_obs=max_day_obs,)\n", + "\n", + "status = nr_adapter.get_reports()\n", + "nr_url = status['endpoint_url']\n", + "#display(status)\n", + "rep.adapter_overview(nr_adapter, status, limit)" ] }, { "cell_type": "code", "execution_count": null, - "id": "10", + "id": "14", "metadata": {}, "outputs": [], "source": [ - "print(f'Retrieved {len(exposure_recs)} records from {exposure_url}')" + "# Display Jira BLOCKS\n", + "front = 'https://rubinobs.atlassian.net/projects/BLOCK?selectedItem=com.atlassian.plugins.atlassian-connect-plugin:com.kanoah.test-manager__main-project-page#!/'\n", + "tickets = nr_adapter.nightly_tickets(nr_adapter.records)\n", + "\n", + "if tickets:\n", + " mdstr = '## Nightly Jira BLOCKs'\n", + " for day, url_list in tickets.items():\n", + " mdstr += f'\\n- {day}'\n", + " for ticket_url in url_list:\n", + " mdstr += f'\\n - [{ticket_url.replace(front,\"\")}]({ticket_url})'\n", + " md(mdstr)\n", + "else:\n", + " md(f'No jira BLOCK tickets found.', color='lightblue')\n", + " md(f'Used: [API Data]({nr_url})')\n", + "\n", + "# Display time log\n", + "nr_rep = rep.NightlyLogReport(min_day_obs=min_day_obs, max_day_obs=max_day_obs)\n", + "nr_rep.time_log_as_markdown(nr_adapter, nr_url)" + ] + }, + { + "cell_type": "markdown", + "id": "15", + "metadata": {}, + "source": [ + "# Exposure Log" ] }, { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "16", "metadata": {}, "outputs": [], "source": [ - "if exposure_recs:\n", - " new_column_names = dict(message_text='message',\n", - " date_added='date'\n", - " )\n", - " df = pd.DataFrame(exposure_recs).rename(columns=new_column_names)\n", - " user_df = df[['date','message']]\n", - " \n", - " display_markdown(f'### Exposure log for {number_of_days} days {min_day_obs} to {max_day_obs}', raw=True)\n", - " display(user_df)" + "# Get data from Exposure log. Display time log.\n", + "exposure_adapter = sad.ExposurelogAdapter(\n", + " server_url=server,\n", + " limit=limit,\n", + " min_day_obs=min_day_obs,\n", + " max_day_obs=max_day_obs,\n", + ")\n", + "status = exposure_adapter.get_messages()\n", + "exposure_url = status['endpoint_url']\n", + "rep.adapter_overview(exposure_adapter, status, limit)\n", + "\n", + "# Display time log\n", + "exposure_rep = rep.ExposurelogReport(min_day_obs=min_day_obs, max_day_obs=max_day_obs)\n", + "exposure_rep.time_log_as_markdown(exposure_adapter, exposure_url)" ] }, { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "17", "metadata": {}, "outputs": [], "source": [ - "display(Markdown(f\"### {exposure_url}/exposures/ Not yet functional on USDF\"))\n", + "# Display Observation gaps\n", "gaps = exposure_adapter.get_observation_gaps()\n", "if gaps:\n", + " md(f'### Date vs Observation Gap (minutes) for all Instruments')\n", " for instrument, day_gaps in gaps.items():\n", - " display(Markdown(f'### Date vs Observation Gap (minutes) for {instrument=!s}'))\n", - " x,y = zip(*day_gaps.items())\n", - " df = pd.DataFrame(dict(day=x,minutes=y))\n", - " df.plot.bar(x='day', y='minutes', title=f'{instrument=!s}')\n", - "else:\n", - " print(f'No Observation Gaps found in exposures.')" + " if len(day_gaps) == 0:\n", + " md(f'**No day gaps found for *{instrument=!s}* **', color='lightblue')\n", + " else:\n", + " x,y = zip(*day_gaps.items())\n", + " df = pd.DataFrame(dict(day=x,minutes=y))\n", + " df.plot.bar(x='day', y='minutes', title=f'{instrument=!s}')" ] }, { "cell_type": "markdown", - "id": "13", + "id": "18", "metadata": {}, "source": [ "# Narrative Log\n" @@ -693,66 +298,64 @@ { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "19", "metadata": {}, "outputs": [], "source": [ - "narrative_adapter = NarrativelogAdapter(server_url=server)\n", - "narrative_url = narrative_adapter.source_url\n", - "try:\n", - " # date like '2000-01-02 12:00:00'\n", - " # str(datetime(2000, 1, 2, 12, 0, 0))\n", - " min_date = str(datetime.strptime(min_day_obs,'%Y%m%d'))\n", - " max_date = str(datetime.strptime(max_day_obs,'%Y%m%d'))\n", - " print(f'Get data from {narrative_url}: {min_date} to {max_date}')\n", - " narrative_recs = narrative_adapter.get_messages(\n", - " limit=limit,\n", - " min_date_end=min_date,\n", - " max_date_end=max_date\n", - " )\n", - "except Exception as err:\n", - " narrative_recs = []\n", - " msg = f'ERROR getting records from {narrative_url}: {err=}'\n", - " raise Exception(msg)\n", - "\n", - "print(f'Retrieved {len(narrative_recs)} records.')" + "# Get data from Narrative log. Display time log.\n", + "narrative_adapter = sad.NarrativelogAdapter(\n", + " server_url=server,\n", + " limit=limit,\n", + " min_day_obs=min_day_obs,\n", + " max_day_obs=max_day_obs,\n", + ")\n", + "status = narrative_adapter.get_messages()\n", + "narrative_url = status['endpoint_url']\n", + "rep.adapter_overview(narrative_adapter, status, limit)\n", + "\n", + "narrrative_rep = rep.NarrativelogReport(min_day_obs=min_day_obs, max_day_obs=max_day_obs)\n", + "narrrative_rep.time_log_as_markdown(narrative_adapter, narrative_url)" + ] + }, + { + "cell_type": "markdown", + "id": "20", + "metadata": {}, + "source": [ + "# Developer Only Section" ] }, { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "21", "metadata": {}, "outputs": [], "source": [ - "new_column_names = dict(message_text='message',\n", - " date_added='date'\n", - " )\n", - "\n", - "if narrative_recs:\n", - " keep_fields(['message_text','date_added'], narrative_recs)\n", - " df = pd.DataFrame(narrative_recs).rename(columns=new_column_names)\n", - " user_df = df # [['date','message']]\n", - "\n", - " display(Markdown(f'## Narrative log (Style A) for {number_of_days} days {min_day_obs} to {max_day_obs}'))\n", - " display(Markdown(\"### Choose display Style (or offer other suggestion)\"))\n", - " with pd.option_context('display.max_rows', None, 'display.max_columns', None):\n", - " display(user_df)\n", - " \n", - " display(Markdown(f'## Narrative log (Style B)'))\n", - " for index,row in user_df.iterrows():\n", - " print(f\"{datetime.fromisoformat(user_df.iloc[0]['date']).date()}: \"\n", - " f\"{row['message']}\"\n", - " )" + "# Conditionally display our current ability to connect to all needed endpoints.\n", + "if not os.environ.get('EXTERNAL_INSTANCE_URL'):\n", + " md('## Dashboard')\n", + " md('(This is not done when running under Times Square.)')\n", + " %run ./dashboard.ipynb" + ] + }, + { + "cell_type": "markdown", + "id": "22", + "metadata": {}, + "source": [ + "# Finale" ] }, { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "23", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "print(f'Finished {str(datetime.now())}')" + ] } ], "metadata": { @@ -771,7 +374,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.11.9" } }, "nbformat": 4, diff --git a/notebooks_tsqr/NightLog.yaml b/notebooks_tsqr/NightLog.yaml index 79cecb0..ce5fb78 100644 --- a/notebooks_tsqr/NightLog.yaml +++ b/notebooks_tsqr/NightLog.yaml @@ -11,19 +11,19 @@ tags: - prototype - exposure parameters: - record_limit: - type: integer - description: Max number of records to output - default: 99 - minimum: 1 - maximum: 9999 day_obs: type: string - description: The night to report on. (YYYY-MM-DD, TODAY, YESTERDAY) + description: > + The night to report on. (Allowed: YYYY-MM-DD, TODAY, YESTERDAY) + The report will include days upto, but not including, day_obs. + For a value of TODAY, the last observing night that will be + included in the report will be the one that started yesterday. + This is usually what you want. default: "TODAY" number_of_days: type: integer - description: Number of days to show (ending in day_obs) + description: > + Number of days to show in the report. default: 1 minimum: 1 maximum: 9 diff --git a/notebooks_tsqr/README.md b/notebooks_tsqr/README.md index 3efda5a..98fb692 100644 --- a/notebooks_tsqr/README.md +++ b/notebooks_tsqr/README.md @@ -1,5 +1,58 @@ -Times Square notebooks for (potential) use in project-wide Logging & Reporting +# Logging & Reporting Times Square Notebooks -See [offical -documentation](https://rsp.lsst.io/v/usdfdev/guides/times-square/index.html) -on creating notebooks for use by Times Square. +Notebooks found here `./notebooks/` are meant to be run in Times Square towards use in project-wide Nightly Logging & Reporting +Times-Square: + +See [official Times-Square documentation](https://rsp.lsst.io/v/usdfdev/guides/times-square/index.html) on creating notebooks for use by Times Square. + +## Development Guidelines + +Rapid Prototyping is enabled with the branch `prototype` +Times-Square for this repository displays the `prototype` branch. + +- Create a branch for your Jira Ticket in the format `tickets/dm-####` off of `prototype` +- Communicate often with team mate when you want to push changes to `prototype` +- Rebase your branch off `prototype` before merging your branch into `prototype` + +Example of flow: + +1. `git checkout prototype; git pull` +2. `git checkout -b tickets/dm-23456` +3. `git commit -m "work happened"; git push` +4. `git checkout prototype; git pull` +5. `git checkout tickets/dm-23456` +6. `git rebase prototype` +7. `git checkout prototype; git merge tickets/dm-23456; git push` + +  + +Once Per Sprint (2 week), the developers on this repository (Steve Pothier & Valerie Becker) gather to discuss updates made to `prototype`, outstanding pull requests, and tickets that have been completed. + +Once they are in agreement, they merge `prototype` into the `develop` branch and close the related Jira Tickets. Squash commit should be used here with a descriptive title and description in the PR. + + +## NightLog.ipynb + +NightLog.ipynb is our main Logging And Reporting notebook. This notebook is meant to display completed* views of logging information. +Each separate notebook should be used to mature a logging/reporting product, and then expect to be integrated into this 'main' notebook. + +\*_Completed to an alpha\beta level -- quick improvements will continue to happen during Fall-Winter 2024_ + +## Dashboard + +Dashboard.ipynb is intended for local development purposes and debugging. Run this notebook not from RSP to evaluate your connection to an array of data sources. +_RSP is not intended to have access to all of the data sources queried here._ + +## Kernel + +Times Square developers/maintainers have indicated that the LSST Kernel should be used in notebooks displayed there. +[RSP Stack info](https://developer.lsst.io/stack/conda.html#rubin-science-platform-notebooks) + +## Backend Code + +We are working our way into a non-Times-Square dependent project. Towards that effort, we are incrementally abstracting common code out of the notebooks. This code is kept in `./python/lsst/ts/logging_and_reporting/` + +`almanac.py` ... +`reports.py` ... +`source_adapters.py` .... +`utils.py` is used for ... \ No newline at end of file diff --git a/notebooks_tsqr/TEMPLATE_logrep.yaml b/notebooks_tsqr/TEMPLATE_logrep.HIDE_yaml similarity index 100% rename from notebooks_tsqr/TEMPLATE_logrep.yaml rename to notebooks_tsqr/TEMPLATE_logrep.HIDE_yaml diff --git a/notebooks_tsqr/consdb/access_consdb.ipynb b/notebooks_tsqr/consdb/access_consdb.ipynb index 27e58a3..dcc59d1 100644 --- a/notebooks_tsqr/consdb/access_consdb.ipynb +++ b/notebooks_tsqr/consdb/access_consdb.ipynb @@ -30,14 +30,12 @@ "source": [ "import os\n", "from lsst.summit.utils import ConsDbClient\n", - "from lsst.summit.utils.utils import computeCcdExposureId\n", - "import sqlalchemy\n", "import requests\n", "import pandas as pd\n", "from IPython.display import display, Markdown, display_markdown\n", "\n", - "\n", - "URL = \"https://usdf-rsp.slac.stanford.edu/consdb/\" # Need to add that part about the headers to client flow through\n", + "# URL = \"https://usdf-rsp.slac.stanford.edu/consdb/\"\n", + "# Need to add that part about the headers to client flow through\n", "URL = \"http://consdb-pq.consdb:8080/consdb\" # Don't use this one\n", "\n", "os.environ[\"no_proxy\"] += \",.consdb\"\n", @@ -45,8 +43,9 @@ "access_token = os.getenv(\"ACCESS_TOKEN\")\n", "headers = {\"Authorization\": f\"Bearer {access_token}\"}\n", "\n", - "sesh = requests.Session()\n", - "sesh.headers.update(headers)\n", + "# This is how the session object should access the ACCESS Token from the headers\n", + "#sesh = requests.Session()\n", + "#sesh.headers.update(headers)\n", "\n", "%matplotlib inline" ] @@ -58,17 +57,13 @@ "metadata": {}, "outputs": [], "source": [ - "display_markdown('## Attempting to access Consolidated Database', raw=True)\n", - "\n", - "try:\n", - " from lsst.summit.utils import ConsDbClient\n", - " have_consdb = True\n", - "except ImportError:\n", - " have_consdb = False\n", - "\n", - "if have_consdb:\n", - " client = ConsDbClient(URL)\n", - " display_markdown('## Consolidated Database is accessible',raw=True)\n" + "from lsst.summit.utils import ConsDbClient\n", + "client = ConsDbClient(URL)\n", + "print(client)\n", + "#import sqlalchemy\n", + "#connection = sqlalchemy.create_engine('postgresql://usdf@usdf-summitdb.slac.stanford.edu/exposurelog')\n", + "#print(connection)\n", + "display_markdown('### Consolidated Database is accessible',raw=True)" ] }, { @@ -87,7 +82,6 @@ "metadata": {}, "outputs": [], "source": [ - "# Add consdb\n", "day_obs_int = int(day_obs.replace('-', ''))\n", "\n", "visit_query1 = f'''\n", @@ -114,31 +108,33 @@ " WHERE v.day_obs = {day_obs_int} and q.visit_id = v.visit_id\n", "'''\n", "\n", - "if have_consdb:\n", - " # Potentially print some schema information for debugging\n", - " \n", + "# Potentially print some schema information for debugging\n", + "try:\n", " print(client.schema()) # list the instruments\n", " print(client.schema('latiss')) # list tables for an instrument\n", " print(client.schema('latiss', 'cdb_latiss.exposure_flexdata')) # specifically flexdata table\n", " \n", - " try:\n", - " visits_latiss = consdb.query(visit_query1).to_pandas()\n", - " visits_lsstcc = consdb.query(visit_query2).to_pandas()\n", - " visits_lsstccs = consdb.query(visit_query3).to_pandas()\n", + "except requests.HTTPError or requests.JSONDecodeError:\n", + " print(client.schema()) # list the instruments\n", + " print(client.schema('latiss')) # list tables for an instrument\n", + " print(client.schema('latiss', 'cdb_latiss.exposure_flexdata'))\n", + "\n", + "try:\n", + " visits_latiss = client.query(visit_query1).to_pandas()\n", + " visits_lsstcc = client.query(visit_query2).to_pandas()\n", + " visits_lsstccs = client.query(visit_query3).to_pandas()\n", "\n", - " except requests.HTTPError or requests.JSONDecodeError:\n", - " # Try twice\n", - " visits_latiss = consdb.query(visit_query1).to_pandas()\n", - " visits_lsstcc = consdb.query(visit_query2).to_pandas()\n", - " visits_lsstccs = consdb.query(visit_query3).to_pandas()\n", + "except requests.HTTPError or requests.JSONDecodeError:\n", + " # Try twice\n", + " visits_latiss = client.query(visit_query1).to_pandas()\n", + " visits_lsstcc = client.query(visit_query2).to_pandas()\n", + " visits_lsstccs = client.query(visit_query3).to_pandas()\n", "\n", - " quicklook = consdb.query(quicklook_query).to_pandas()\n", + "quicklook = client.query(quicklook_query).to_pandas()\n", "\n", - "else:\n", - " # Assumes at the USDF\n", - " connection = sqlalchemy.create_engine('postgresql://usdf@usdf-summitdb.slac.stanford.edu/exposurelog')\n", - " visits_latiss = pd.read_sql(visit_query1, connection)\n", - " quicklook = pd.read_sql(quicklook_query, connection)\n", + "# Assumes at the USDF\n", + "#visits_latiss_try = pd.read_sql(visit_query1, connection)\n", + "#quicklook_try = pd.read_sql(quicklook_query, connection)\n", "\n", "if len(visits_latiss) > 0:\n", " print(f\"Retrieved {len(visits_latiss)} visits from consdb\")\n", @@ -256,14 +252,17 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "id": "15", "metadata": {}, + "outputs": [], "source": [ - "# Access DDV\n", - "Summit = https://summit-lsp.lsst.codes/rubintv-dev/ddv/index.html \n", - "\n", - "USDF = https://usdf-rsp-dev.slac.stanford.edu/rubintv/ddv/index.html" + "usdf = 'https://usdf-rsp-dev.slac.stanford.edu'\n", + "service_loc = os.environ.get('EXTERNAL_INSTANCE_URL', usdf)\n", + "DDV = f\"{service_loc}/rubintv-dev/ddv/index.html\"\n", + "display_markdown('## Access DDV part of RubinTV', raw=True)\n", + "DDV" ] }, { diff --git a/notebooks_tsqr/consdb/assorted_plots.ipynb b/notebooks_tsqr/consdb/assorted_plots.ipynb new file mode 100644 index 0000000..48343dd --- /dev/null +++ b/notebooks_tsqr/consdb/assorted_plots.ipynb @@ -0,0 +1,158 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "# Parameters\n", + "day_obs = '2024-06-26'\n", + "instruments = 'latiss, lsstcomcamsim, lsstcomcam'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import matplotlib.pyplot as plt\n", + "from IPython.display import display_markdown" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "### 'Get Consdb access'\n", + "from lsst.summit.utils import ConsDbClient\n", + "\n", + "URL = \"http://consdb-pq.consdb:8080/consdb\" # Don't use this one\n", + "os.environ[\"no_proxy\"] += \",.consdb\"\n", + "\n", + "client = ConsDbClient(URL)\n", + "display_markdown('### Consolidated Database is accessible',raw=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline # After all imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "# Put Plot in backend\n", + "plt.style.use('seaborn-v0_8-bright')\n", + "def plot(y, x):\n", + " # plot\n", + " fig = plt.figure(figsize=(6, 6))\n", + " ax = fig.subplots()\n", + " ax.scatter(x, y)\n", + "\n", + " plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "day_obs_int = int(day_obs.replace('-', ''))\n", + "instrument_list = [ins.strip() for ins in instruments.split(',')]\n", + "\n", + "for instrument in instrument_list:\n", + " print(\"------------------------------------------------------------\")\n", + " print()\n", + " display_markdown(f'# Instrument: {instrument}',raw=True)\n", + " #################### Put in Backend\n", + " ccdvisit1_quicklook = f'''\n", + " SELECT * FROM cdb_{instrument}.ccdvisit1_quicklook\n", + " '''\n", + "\n", + " visit1 = f'''\n", + " SELECT * FROM cdb_{instrument}.visit1\n", + " '''\n", + "\n", + " # Join Visit1 and ccdVisit1 to access data and day obs\n", + " visits = client.query(visit1).to_pandas()\n", + " quicklook = client.query(ccdvisit1_quicklook).to_pandas()\n", + "\n", + " visits = visits.join(quicklook, on='visit_id',lsuffix='',rsuffix='_q')\n", + " #################### Put in Backend - end\n", + "\n", + " # If we see data exist in psf, zero, then we should pare down like visits_today below\n", + " try:\n", + " visits_w_psf = visits[visits['psf_area'].notna()]\n", + " time = visits_w_psf['obs_start']\n", + " display_markdown(f'Number of visits with psf_area populated {len(visits_w_psf)}', raw=True)\n", + " display_markdown('## psf_area vs obs_start', raw=True)\n", + " plot(time, visits_w_psf['psf_area'])\n", + " except KeyError as err:\n", + " display_markdown(f\"Psf_area not a column in {instrument} dataframe\",raw=True)\n", + " display_markdown(f\"key error for {err}\", raw=True)\n", + "\n", + " try:\n", + " visits_w_zero = visits[visits['zero_point'].notna()]\n", + " time = visits_w_zero['obs_start']\n", + " display_markdown(f'Number of visits with zero_point populated {len(visits_w_zero)}', raw=True)\n", + " display_markdown('## zero_point vs obs_start', raw=True)\n", + " plot(time, visits_w_zero['zero_point'])\n", + " except KeyError as err:\n", + " display_markdown(f\"Zero_point not a column in {instrument} dataframe\", raw=True)\n", + " display_markdown(f\"key error for {err}\", raw=True)\n", + "\n", + " # Pare down to only day obs\n", + " visits_today = visits[(visits['day_obs'] == day_obs_int)]\n", + " display_markdown(f\"How many visits today? {len(visits_today)}\", raw=True)\n", + "\n", + " ra = visits_today['s_ra']\n", + " dec = visits_today['s_dec']\n", + " display_markdown(f\"How many ra? {len(ra)}\", raw=True)\n", + " display_markdown(f\"How many dec? {len(dec)}\", raw=True)\n", + "\n", + " display_markdown(f'Ra Dec should be populated for {instrument}', raw=True)\n", + " plot(ra, dec)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "LSST", + "language": "python", + "name": "lsst" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks_tsqr/consdb/assorted_plots.yaml b/notebooks_tsqr/consdb/assorted_plots.yaml new file mode 100644 index 0000000..f0c9f5e --- /dev/null +++ b/notebooks_tsqr/consdb/assorted_plots.yaml @@ -0,0 +1,18 @@ +# For use with a Times Square notebook +title: Assorted Plots +description: Trying plots +authors: + - name: Valerie Becker + slack: valerie becker +tags: + - reporting + - prototype +parameters: + day_obs: + type: string + description: Date on which to query + default: '2024-06-26' + instruments: + type: string + description: Which instrument to query + default: 'latiss, lsstcomcamsim, lsstcomcam' \ No newline at end of file diff --git a/notebooks_tsqr/dashboard.ipynb b/notebooks_tsqr/dashboard.ipynb new file mode 100644 index 0000000..8c08d4b --- /dev/null +++ b/notebooks_tsqr/dashboard.ipynb @@ -0,0 +1,135 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Logger Source API Dashboard\n", + "For all of these to work, the following must be enabled:\n", + "- Tucson VPN\n", + "- Summit VPN\n", + "- User has access to USDF-dev (SLAC)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "from pprint import pformat, pp\n", + "from datetime import datetime, date, timedelta\n", + "from lsst.ts.logging_and_reporting.dashboard import Dashboard\n", + "from lsst.ts.logging_and_reporting.reports import md,mdlist" + ] + }, + { + "cell_type": "markdown", + "id": "2", + "metadata": {}, + "source": [ + "## Connects" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "dash = Dashboard()\n", + "score, working = dash.report()\n", + "# On 9/12/2024 this gets two failed connects. \n", + "# The are usdf: exposurelog/instruments,exposurelog/exposures\n", + "# When usdfdev if fully functional, there should be zero failed connects (with both VPNs active)." + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "## Score" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "md(f'{score=:.0%}')\n", + "md('**Servers that are fully functional** for Logging and Reporting:')\n", + "mdlist([f'- {w}' for w in working])" + ] + }, + { + "cell_type": "markdown", + "id": "6", + "metadata": {}, + "source": [ + "## Samples" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "server = 'https://summit-lsp.lsst.codes'\n", + "samples = dash.get_sample_data(server)\n", + "print('One record of data from all endpoints used by LogRep:')\n", + "for endpoint,sample in samples.items():\n", + " base_ep = endpoint.replace(server,'')\n", + " md(f'\\n### Endpoint: {base_ep}')\n", + " print(f'{endpoint}')\n", + " pp(sample)" + ] + }, + { + "cell_type": "markdown", + "id": "8", + "metadata": {}, + "source": [ + "# Finale" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "print(f'Finished {str(datetime.now())}')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks_tsqr/efd.yaml b/notebooks_tsqr/efd.HIDE_yaml similarity index 100% rename from notebooks_tsqr/efd.yaml rename to notebooks_tsqr/efd.HIDE_yaml diff --git a/notebooks_tsqr/exposurelog.ipynb b/notebooks_tsqr/exposurelog.ipynb deleted file mode 100644 index 1581b3d..0000000 --- a/notebooks_tsqr/exposurelog.ipynb +++ /dev/null @@ -1,739 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "0", - "metadata": {}, - "outputs": [], - "source": [ - "# Parameters. Set defaults here.\n", - "# Times Square replaces this cell with the user's parameters.\n", - "record_limit = '99'" - ] - }, - { - "cell_type": "markdown", - "id": "1", - "metadata": {}, - "source": [ - "\n", - "## Imports and General Setup" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2", - "metadata": {}, - "outputs": [], - "source": [ - "# Only use packages available in the Rubin Science Platform\n", - "import requests\n", - "from collections import defaultdict\n", - "import pandas as pd\n", - "from pprint import pp\n", - "from urllib.parse import urlencode\n", - "from IPython.display import FileLink, display_markdown\n", - "from matplotlib import pyplot as plt\n", - "import os\n", - "from datetime import datetime, date" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, - "outputs": [], - "source": [ - "limit = int(record_limit)\n", - "\n", - "response_timeout = 3.05 # seconds, how long to wait for connection\n", - "read_timeout = 20 # seconds\n", - "timeout = (float(response_timeout), float(read_timeout))\n", - "\n", - "summit = 'https://summit-lsp.lsst.codes'\n", - "usdf = 'https://usdf-rsp-dev.slac.stanford.edu'\n", - "tucson = 'https://tucson-teststand.lsst.codes'\n", - "\n", - "# Use server=tucson for dev testing\n", - "server = os.environ.get('EXTERNAL_INSTANCE_URL', summit)\n", - "log = 'exposurelog'\n", - "service = f'{server}/{log}'\n", - "service" - ] - }, - { - "cell_type": "markdown", - "id": "4", - "metadata": {}, - "source": [ - "\n", - "## Setup Source" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5", - "metadata": {}, - "outputs": [], - "source": [ - "# For Times Square, comment out next line and past next cell with contents of local python file.\n", - "#! from lsst.ts.logging_and_reporting.source_adapters import ExposurelogAdapter\n", - "# Once our logrep package has been installed in RSP, we can use the simpler \"import\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "# Paste contents of source_adapters.py here\n", - "# This file is part of ts_logging_and_reporting.\n", - "#\n", - "# Developed for Vera C. Rubin Observatory Telescope and Site Systems.\n", - "# This product includes software developed by the LSST Project\n", - "# (https://www.lsst.org).\n", - "# See the COPYRIGHT file at the top-level directory of this distribution\n", - "# for details of code ownership.\n", - "#\n", - "# This program is free software: you can redistribute it and/or modify\n", - "# it under the terms of the GNU General Public License as published by\n", - "# the Free Software Foundation, either version 3 of the License, or\n", - "# (at your option) any later version.\n", - "#\n", - "# This program is distributed in the hope that it will be useful,\n", - "# but WITHOUT ANY WARRANTY; without even the implied warranty of\n", - "# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n", - "# GNU General Public License for more details.\n", - "#\n", - "# You should have received a copy of the GNU General Public License\n", - "# along with this program. If not, see .\n", - "\n", - "\n", - "############################################\n", - "# Python Standard Library\n", - "from urllib.parse import urlencode\n", - "import itertools\n", - "from datetime import datetime\n", - "import warnings\n", - "from collections import defaultdict\n", - "############################################\n", - "# External Packages\n", - "import requests\n", - "\n", - "\n", - "MAX_CONNECT_TIMEOUT = 3.1 # seconds\n", - "MAX_READ_TIMEOUT = 90 * 60 # seconds\n", - "\n", - "class ApiAdapter:\n", - " def __init__(self, *,\n", - " server_url='https://tucson-teststand.lsst.codes',\n", - " connect_timeout=3.05, # seconds\n", - " read_timeout=10 * 60, # seconds\n", - " ):\n", - " self.server = server_url\n", - " self.c_timeout = min(MAX_CONNECT_TIMEOUT,\n", - " float(connect_timeout)) # seconds\n", - " self.r_timeout = min(MAX_READ_TIMEOUT, # seconds\n", - " float(read_timeout))\n", - " self.timeout = (self.c_timeout, self.r_timeout)\n", - "\n", - " # We may be accessing several endpoints of an API.\n", - " # If so, we will get different types of records for each.\n", - " # The following are for the \"primary_endpoint\".\n", - " self.ignore_fields = list()\n", - " self.categoricals = list()\n", - " self.foreign_keys = list()\n", - "\n", - "\n", - " def analytics(self, recs, categorical_fields=None):\n", - " non_cats = set([\n", - " 'tags', 'urls', 'message_text', 'id', 'date_added',\n", - " 'obs_id', 'day_obs', 'seq_num', 'parent_id', 'user_id',\n", - " 'date_invalidated', 'date_begin', 'date_end',\n", - " 'time_lost', # float\n", - " # 'systems','subsystems','cscs', # values need special handling\n", - " ])\n", - " flds = set(recs[0].keys())\n", - " if not categorical_fields:\n", - " categorical_fields = flds\n", - " ignore_fields = flds - categorical_fields\n", - " facflds = flds - ignore_fields\n", - "\n", - " # facets(field) = set(value-1, value-2, ...)\n", - " facets = {fld: set([str(r[fld])\n", - " for r in recs if not isinstance(r[fld], list)])\n", - " for fld in facflds}\n", - " return dict(fields=flds,\n", - " facet_fields=facflds,\n", - " facets=facets,\n", - " )\n", - "\n", - "\n", - "class NarrativelogAdapter(ApiAdapter):\n", - " service = 'narrativelog'\n", - " primary_endpoint = 'messages'\n", - " fields = {'category',\n", - " 'components',\n", - " 'cscs',\n", - " 'date_added',\n", - " 'date_begin',\n", - " 'date_end',\n", - " 'date_invalidated',\n", - " 'id',\n", - " 'is_human',\n", - " 'is_valid',\n", - " 'level',\n", - " 'message_text',\n", - " 'parent_id',\n", - " 'primary_hardware_components',\n", - " 'primary_software_components',\n", - " 'site_id',\n", - " 'subsystems',\n", - " 'systems',\n", - " 'tags',\n", - " 'time_lost',\n", - " 'time_lost_type',\n", - " 'urls',\n", - " 'user_agent',\n", - " 'user_id'}\n", - " filters = {\n", - " 'site_ids',\n", - " 'message_text', # Message text contain ...\n", - " 'min_level', # inclusive\n", - " 'max_level', # exclusive\n", - " 'user_ids',\n", - " 'user_agents',\n", - " 'categories',\n", - " 'exclude_categories',\n", - " 'time_lost_types',\n", - " 'exclude_time_lost_types',\n", - " 'tags', # at least one must be present.\n", - " 'exclude_tags', # all must be absent\n", - " 'systems',\n", - " 'exclude_systems',\n", - " 'subsystems',\n", - " 'exclude_subsystems',\n", - " 'cscs',\n", - " 'exclude_cscs',\n", - " 'components',\n", - " 'exclude_components',\n", - " 'primary_software_components',\n", - " 'exclude_primary_software_components',\n", - " 'primary_hardware_components',\n", - " 'exclude_primary_hardware_components',\n", - " 'urls',\n", - " 'min_time_lost',\n", - " 'max_time_lost',\n", - " 'has_date_begin',\n", - " 'min_date_begin',\n", - " 'max_date_begin',\n", - " 'has_date_end',\n", - " 'min_date_end',\n", - " 'max_date_end',\n", - " 'is_human', # Allowed: either, true, false; Default=either\n", - " 'is_valid', # Allowed: either, true, false; Default=true\n", - " 'min_date_added', # inclusive, TAI ISO string, no TZ\n", - " 'max_date_added', # exclusive, TAI ISO string, no TZ\n", - " 'has_date_invalidated',\n", - " 'min_date_invalidated',\n", - " 'max_date_invalidated',\n", - " 'has_parent_id',\n", - " 'order_by',\n", - " 'offset',\n", - " 'limit'\n", - " }\n", - "\n", - " def get_messages(self,\n", - " site_ids=None,\n", - " message_text=None,\n", - " min_date_end=None,\n", - " max_date_end=None,\n", - " is_human='either',\n", - " is_valid='either',\n", - " offset=None,\n", - " limit=None\n", - " ):\n", - " qparams = dict(is_human=is_human, is_valid=is_valid)\n", - " if site_ids:\n", - " qparams['site_ids'] = site_ids\n", - " if message_text:\n", - " qparams['message_text'] = message_text\n", - " if min_date_end:\n", - " qparams['min_date_end'] = min_date_end\n", - " if max_date_end:\n", - " qparams['max_date_end'] = max_date_end\n", - " if limit:\n", - " qparams['limit'] = limit\n", - "\n", - " qstr = urlencode(qparams)\n", - " url = f'{self.server}/{self.service}/messages?{qstr}'\n", - " try:\n", - " recs = requests.get(url, timeout=self.timeout).json()\n", - " except Exception as err:\n", - " warnings.warn(f'No {self.service} records retrieved: {err}')\n", - " recs = []\n", - " if len(recs) == 0:\n", - " raise Exception(f'No records retrieved from {url}')\n", - "\n", - " self.recs = recs\n", - " self.recs.sort(key=lambda r: r['date_begin'])\n", - " return recs\n", - "\n", - " def get_timelost(self, rollup='day'):\n", - " day_tl = dict() # day_tl[day] = totalDayTimeLost\n", - " for day,dayrecs in itertools.groupby(\n", - " self.recs,\n", - " key=lambda r: datetime.fromisoformat(r['date_begin']).date().isoformat()\n", - " ):\n", - " day_tl[day] = sum([r['time_lost'] for r in dayrecs])\n", - " return day_tl\n", - "\n", - "class ExposurelogAdapter(ApiAdapter):\n", - " service = 'exposurelog'\n", - " primary_endpoint = 'messages'\n", - " fields = {'date_added',\n", - " 'date_invalidated',\n", - " 'day_obs',\n", - " 'exposure_flag',\n", - " 'id',\n", - " 'instrument',\n", - " 'is_human',\n", - " 'is_valid',\n", - " 'level',\n", - " 'message_text',\n", - " 'obs_id',\n", - " 'parent_id',\n", - " 'seq_num',\n", - " 'site_id',\n", - " 'tags',\n", - " 'urls',\n", - " 'user_agent',\n", - " 'user_id'}\n", - " filters = {\n", - " 'site_ids',\n", - " 'obs_id',\n", - " 'instruments',\n", - " 'min_day_obs', # inclusive, integer in form YYYMMDD\n", - " 'max_day_obs', # exclusive, integer in form YYYMMDD\n", - " 'min_seq_num',\n", - " 'max_seq_num',\n", - " 'message_text', # Message text contain ...\n", - " 'min_level', # inclusive\n", - " 'max_level', # exclusive\n", - " 'tags', # at least one must be present.\n", - " 'urls',\n", - " 'exclude_tags', # all must be absent\n", - " 'user_ids',\n", - " 'user_agents',\n", - " 'is_human', # Allowed: either, true, false; Default=either\n", - " 'is_valid', # Allowed: either, true, false; Default=true\n", - " 'exposure_flags',\n", - " 'min_date_added', # inclusive, TAI ISO string, no TZ\n", - " 'max_date_added', # exclusive, TAI ISO string, no TZ\n", - " 'has_date_invalidated',\n", - " 'min_date_invalidated',\n", - " 'max_date_invalidated',\n", - " 'has_parent_id',\n", - " 'order_by',\n", - " 'offset',\n", - " 'limit'\n", - " }\n", - "\n", - "\n", - "\n", - " def get_instruments(self):\n", - " url = f'{self.server}/{self.service}/instruments'\n", - " try:\n", - " instruments = requests.get(url, timeout=self.timeout).json()\n", - " except Exception as err:\n", - " warnings.warn(f'No instruments retrieved: {err}')\n", - " instruments = dict(dummy=[])\n", - " # Flatten the lists\n", - " return list(itertools.chain.from_iterable(instruments.values()))\n", - "\n", - " def get_exposures(self, instrument, registry=1):\n", - " qparams = dict(instrument=instrument, registery=registry)\n", - " url = f'{self.server}/{self.service}/exposures?{urlencode(qparams)}'\n", - " try:\n", - " recs = requests.get(url, timeout=self.timeout).json()\n", - " except Exception as err:\n", - " warnings.warn(f'No exposures retrieved: {err}')\n", - " recs = []\n", - " return recs\n", - "\n", - " def get_messages(self,\n", - " site_ids=None,\n", - " obs_ids=None,\n", - " instruments=None,\n", - " message_text=None,\n", - " min_day_obs=None,\n", - " max_day_obs=None,\n", - " is_human='either',\n", - " is_valid='either',\n", - " exposure_flags=None,\n", - " offset=None,\n", - " limit=None\n", - " ):\n", - " qparams = dict(is_human=is_human, is_valid=is_valid)\n", - " if site_ids:\n", - " qparams['site_ids'] = site_ids\n", - " if obs_ids:\n", - " qparams['obs_ids'] = obs_ids\n", - " if instruments:\n", - " qparams['instruments'] = instruments\n", - " if min_day_obs:\n", - " qparams['min_day_obs'] = min_day_obs\n", - " if max_day_obs:\n", - " qparams['max_day_obs'] = max_day_obs\n", - " if exposure_flags:\n", - " qparams['exposure_flags'] = exposure_flags\n", - " if offset:\n", - " qparams['offset'] = offset\n", - " if limit:\n", - " qparams['limit'] = limit\n", - "\n", - " qstr = urlencode(qparams)\n", - " url = f'{self.server}/{self.service}/messages?{qstr}'\n", - " try:\n", - " recs = requests.get(url, timeout=self.timeout).json()\n", - " except Exception as err:\n", - " warnings.warn(f'No {self.service} records retrieved: {err}')\n", - " recs = []\n", - " if len(recs) == 0:\n", - " raise Exception(f'No records retrieved from {url}')\n", - "\n", - " self.recs = recs\n", - " self.recs.sort(key=lambda r: r['day_obs'])\n", - " return recs\n", - "\n", - " def get_observation_gaps(self, instruments=None,\n", - " min_day_obs=None, # YYYYMMDD\n", - " max_day_obs=None, # YYYYMMDD\n", - " ):\n", - " if not instruments:\n", - " instruments = self.get_instruments()\n", - " assert isinstance(instruments,list), \\\n", - " f'\"instruments\" must be a list. Got {instruments!r}'\n", - " # inst_day_rollupol[instrument] => dict[day] => exposureGapInMinutes\n", - " inst_day_rollup = defaultdict(dict) # Instrument/Day rollup\n", - "\n", - " for instrum in instruments:\n", - " recs = self.get_exposures(instrum)\n", - " instrum_gaps = dict()\n", - " for day,dayrecs in itertools.groupby(recs,\n", - " key=lambda r: r['day_obs']):\n", - " gaps = list()\n", - " begin = end = None\n", - " for rec in dayrecs:\n", - " begin = rec['timespan_begin']\n", - " if end:\n", - " # span in minutes\n", - " diff = (datetime.fromisoformat(begin)\n", - " - datetime.fromisoformat(end)\n", - " ).total_seconds() / 60.0\n", - "\n", - " gaps.append((\n", - " datetime.fromisoformat(end).time().isoformat(),\n", - " datetime.fromisoformat(begin).time().isoformat(),\n", - " diff\n", - " ))\n", - " end = rec['timespan_end']\n", - " instrum_gaps[day] = gaps\n", - "\n", - " #!roll = dict()\n", - " # Rollup gap times by day\n", - " for day,tuples in instrum_gaps.items():\n", - " #!roll[day] = sum([t[2] for t in tuples])\n", - " inst_day_rollup[instrum][day] = sum([t[2] for t in tuples])\n", - "\n", - " return inst_day_rollup" - ] - }, - { - "cell_type": "markdown", - "id": "7", - "metadata": {}, - "source": [ - "\n", - "## Get Records" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8", - "metadata": {}, - "outputs": [], - "source": [ - "service_adapter = ExposurelogAdapter(server_url=server)\n", - "try:\n", - " recs = service_adapter.get_messages(limit=limit)\n", - "except Exception as err:\n", - " recs = []\n", - " msg = f'ERROR getting records from {server=}: {err=}'\n", - " raise Exception(msg)\n", - "\n", - "metrics = service_adapter.analytics(recs)\n", - "flds = metrics['fields']\n", - "facets = metrics['facets'] # facets(field) = set(value-1, value-2, ...)\n", - "print(f'Retrieved {len(recs)} records, each with {len(flds)} fields.')" - ] - }, - { - "cell_type": "markdown", - "id": "9", - "metadata": {}, - "source": [ - "\n", - "## Tables of (mostly raw) results" - ] - }, - { - "cell_type": "markdown", - "id": "10", - "metadata": {}, - "source": [ - "### Fields names provided in records from log." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "11", - "metadata": {}, - "outputs": [], - "source": [ - "pd.DataFrame(flds, columns=['Field Name'])" - ] - }, - { - "cell_type": "markdown", - "id": "12", - "metadata": {}, - "source": [ - "### Facets from log records.\n", - "A *facet* is the set all of values found for a field in the retrieved records. Facets are only calculated for some fields." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "13", - "metadata": {}, - "outputs": [], - "source": [ - "display(pd.DataFrame.from_dict(facets, orient='index'))\n", - "facets" - ] - }, - { - "cell_type": "markdown", - "id": "14", - "metadata": {}, - "source": [ - "### Show Retrieved Records" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "15", - "metadata": {}, - "outputs": [], - "source": [ - "ignore_fields = ['id']\n", - "new_column_names = dict(message_text='message',\n", - " )\n", - "df = pd.DataFrame(recs).rename(columns=new_column_names)\n", - "with pd.option_context('display.max_rows', None, 'display.max_columns', None):\n", - " display(df.loc[:, ~df.columns.isin(ignore_fields)])" - ] - }, - { - "cell_type": "markdown", - "id": "16", - "metadata": {}, - "source": [ - "### Table of selected log record fields.\n", - "Table can be retrieved as CSV file for local use." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "17", - "metadata": {}, - "outputs": [], - "source": [ - "cols = ['obs_id', 'user_id', 'user_agent','is_human','is_valid','exposure_flag']\n", - "df = pd.DataFrame(recs)[cols]\n", - "\n", - "# Allow download of CSV version of DataFrame\n", - "csvfile = 'tl.csv'\n", - "df.to_csv(csvfile)\n", - "myfile = FileLink(csvfile)\n", - "print('Table available as CSV file: ')\n", - "display(myfile)\n", - "df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "18", - "metadata": {}, - "outputs": [], - "source": [ - "df = pd.DataFrame(recs)\n", - "df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "19", - "metadata": {}, - "outputs": [], - "source": [ - "cols = ['obs_id', 'site_id', 'instrument', 'message_text', 'tags','user_id', 'user_agent','is_human','is_valid','exposure_flag']\n", - "df = pd.DataFrame(recs, columns=None)\n", - "df" - ] - }, - { - "cell_type": "markdown", - "id": "20", - "metadata": {}, - "source": [ - "\n", - "## Plots from log (per day)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "21", - "metadata": {}, - "outputs": [], - "source": [ - "gaps = service_adapter.get_observation_gaps()\n", - "for instrument, day_gaps in gaps.items():\n", - " display_markdown(f'### Date vs Observation Gap (minutes) for {instrument=!s}', raw=True)\n", - " x,y = zip(*day_gaps.items())\n", - " df = pd.DataFrame(dict(day=x,minutes=y))\n", - " df.plot.bar(x='day', y='minutes', title=f'{instrument=!s}')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "22", - "metadata": {}, - "outputs": [], - "source": [ - "dict(gaps)" - ] - }, - { - "cell_type": "markdown", - "id": "23", - "metadata": {}, - "source": [ - "\n", - "## Raw Content Analysis" - ] - }, - { - "cell_type": "markdown", - "id": "24", - "metadata": {}, - "source": [ - "### Example of one record" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "25", - "metadata": {}, - "outputs": [], - "source": [ - "rec = recs[-1]\n", - "\n", - "msg = rec[\"message_text\"]\n", - "md = f'Message text from log:\\n> {msg}'\n", - "display_markdown(md, raw=True)\n", - "\n", - "md = f'One full record (the last one retrieved):\\n> {rec}'\n", - "display_markdown(md, raw=True)\n", - "\n", - "display(rec)" - ] - }, - { - "cell_type": "markdown", - "id": "26", - "metadata": {}, - "source": [ - "\n", - "## Stakeholder Elicitation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "27", - "metadata": {}, - "outputs": [], - "source": [ - "#EXTERNAL_INSTANCE_URL\n", - "dict(os.environ.items())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "28", - "metadata": {}, - "outputs": [], - "source": [ - "print(f'Finished run: {str(datetime.now())}')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "29", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks_tsqr/narrativelog.ipynb b/notebooks_tsqr/narrativelog.ipynb deleted file mode 100644 index b8eb6ae..0000000 --- a/notebooks_tsqr/narrativelog.ipynb +++ /dev/null @@ -1,731 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "0", - "metadata": {}, - "outputs": [], - "source": [ - "# Parameters. Set defaults here.\n", - "# Times Square replaces this cell with the user's parameters.\n", - "record_limit = '99'" - ] - }, - { - "cell_type": "markdown", - "id": "1", - "metadata": {}, - "source": [ - "\n", - "## Imports and General Setup" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2", - "metadata": {}, - "outputs": [], - "source": [ - "# Only use packages available in the Rubin Science Platform\n", - "import requests\n", - "from collections import defaultdict\n", - "import pandas as pd\n", - "from pprint import pp\n", - "from urllib.parse import urlencode\n", - "from IPython.display import FileLink, display_markdown\n", - "from matplotlib import pyplot as plt\n", - "import os\n", - "from datetime import datetime, date" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, - "outputs": [], - "source": [ - "env = 'usdf_dev' # usdf-dev, tucson, slac, summit\n", - "log_name = 'narrativelog'\n", - "log = log_name\n", - "limit = int(record_limit)\n", - "response_timeout = 3.05 # seconds, how long to wait for connection\n", - "read_timeout = 20 # seconds\n", - "\n", - "timeout = (float(response_timeout), float(read_timeout))\n", - "\n", - "summit = 'https://summit-lsp.lsst.codes'\n", - "usdf = 'https://usdf-rsp-dev.slac.stanford.edu'\n", - "tucson = 'https://tucson-teststand.lsst.codes'\n", - "\n", - "# Use server = tucson for dev testing\n", - "server = os.environ.get('EXTERNAL_INSTANCE_URL', summit)\n", - "\n", - "service = f'{server}/{log}'\n", - "service" - ] - }, - { - "cell_type": "markdown", - "id": "4", - "metadata": {}, - "source": [ - "\n", - "## Setup Source" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5", - "metadata": {}, - "outputs": [], - "source": [ - "# For Times Square, comment out next line and past next cell with contents of local python file.\n", - "#! from lsst.ts.logging_and_reporting.source_adapters import NarrativelogAdapter\n", - "# Once our logrep package has been installed in RSP, we can use the simpler \"import\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "# Paste contents of source_adapters.py here\n", - "\n", - "# This file is part of ts_logging_and_reporting.\n", - "#\n", - "# Developed for Vera C. Rubin Observatory Telescope and Site Systems.\n", - "# This product includes software developed by the LSST Project\n", - "# (https://www.lsst.org).\n", - "# See the COPYRIGHT file at the top-level directory of this distribution\n", - "# for details of code ownership.\n", - "#\n", - "# This program is free software: you can redistribute it and/or modify\n", - "# it under the terms of the GNU General Public License as published by\n", - "# the Free Software Foundation, either version 3 of the License, or\n", - "# (at your option) any later version.\n", - "#\n", - "# This program is distributed in the hope that it will be useful,\n", - "# but WITHOUT ANY WARRANTY; without even the implied warranty of\n", - "# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n", - "# GNU General Public License for more details.\n", - "#\n", - "# You should have received a copy of the GNU General Public License\n", - "# along with this program. If not, see .\n", - "\n", - "\n", - "############################################\n", - "# Python Standard Library\n", - "from urllib.parse import urlencode\n", - "import itertools\n", - "from datetime import datetime\n", - "import warnings\n", - "from collections import defaultdict\n", - "############################################\n", - "# External Packages\n", - "import requests\n", - "\n", - "\n", - "MAX_CONNECT_TIMEOUT = 3.1 # seconds\n", - "MAX_READ_TIMEOUT = 90 * 60 # seconds\n", - "\n", - "class ApiAdapter:\n", - " def __init__(self, *,\n", - " server_url='https://tucson-teststand.lsst.codes',\n", - " connect_timeout=3.05, # seconds\n", - " read_timeout=10 * 60, # seconds\n", - " ):\n", - " self.server = server_url\n", - " self.c_timeout = min(MAX_CONNECT_TIMEOUT,\n", - " float(connect_timeout)) # seconds\n", - " self.r_timeout = min(MAX_READ_TIMEOUT, # seconds\n", - " float(read_timeout))\n", - " self.timeout = (self.c_timeout, self.r_timeout)\n", - "\n", - " # We may be accessing several endpoints of an API.\n", - " # If so, we will get different types of records for each.\n", - " # The following are for the \"primary_endpoint\".\n", - " self.ignore_fields = list()\n", - " self.categoricals = list()\n", - " self.foreign_keys = list()\n", - "\n", - "\n", - " def analytics(self, recs, categorical_fields=None):\n", - " non_cats = set([\n", - " 'tags', 'urls', 'message_text', 'id', 'date_added',\n", - " 'obs_id', 'day_obs', 'seq_num', 'parent_id', 'user_id',\n", - " 'date_invalidated', 'date_begin', 'date_end',\n", - " 'time_lost', # float\n", - " # 'systems','subsystems','cscs', # values need special handling\n", - " ])\n", - " flds = set(recs[0].keys())\n", - " if not categorical_fields:\n", - " categorical_fields = flds\n", - " ignore_fields = flds - categorical_fields\n", - " facflds = flds - ignore_fields\n", - "\n", - " # facets(field) = set(value-1, value-2, ...)\n", - " facets = {fld: set([str(r[fld])\n", - " for r in recs if not isinstance(r[fld], list)])\n", - " for fld in facflds}\n", - " return dict(fields=flds,\n", - " facet_fields=facflds,\n", - " facets=facets,\n", - " )\n", - "\n", - "\n", - "class NarrativelogAdapter(ApiAdapter):\n", - " service = 'narrativelog'\n", - " primary_endpoint = 'messages'\n", - " fields = {'category',\n", - " 'components',\n", - " 'cscs',\n", - " 'date_added',\n", - " 'date_begin',\n", - " 'date_end',\n", - " 'date_invalidated',\n", - " 'id',\n", - " 'is_human',\n", - " 'is_valid',\n", - " 'level',\n", - " 'message_text',\n", - " 'parent_id',\n", - " 'primary_hardware_components',\n", - " 'primary_software_components',\n", - " 'site_id',\n", - " 'subsystems',\n", - " 'systems',\n", - " 'tags',\n", - " 'time_lost',\n", - " 'time_lost_type',\n", - " 'urls',\n", - " 'user_agent',\n", - " 'user_id'}\n", - " filters = {\n", - " 'site_ids',\n", - " 'message_text', # Message text contain ...\n", - " 'min_level', # inclusive\n", - " 'max_level', # exclusive\n", - " 'user_ids',\n", - " 'user_agents',\n", - " 'categories',\n", - " 'exclude_categories',\n", - " 'time_lost_types',\n", - " 'exclude_time_lost_types',\n", - " 'tags', # at least one must be present.\n", - " 'exclude_tags', # all must be absent\n", - " 'systems',\n", - " 'exclude_systems',\n", - " 'subsystems',\n", - " 'exclude_subsystems',\n", - " 'cscs',\n", - " 'exclude_cscs',\n", - " 'components',\n", - " 'exclude_components',\n", - " 'primary_software_components',\n", - " 'exclude_primary_software_components',\n", - " 'primary_hardware_components',\n", - " 'exclude_primary_hardware_components',\n", - " 'urls',\n", - " 'min_time_lost',\n", - " 'max_time_lost',\n", - " 'has_date_begin',\n", - " 'min_date_begin',\n", - " 'max_date_begin',\n", - " 'has_date_end',\n", - " 'min_date_end',\n", - " 'max_date_end',\n", - " 'is_human', # Allowed: either, true, false; Default=either\n", - " 'is_valid', # Allowed: either, true, false; Default=true\n", - " 'min_date_added', # inclusive, TAI ISO string, no TZ\n", - " 'max_date_added', # exclusive, TAI ISO string, no TZ\n", - " 'has_date_invalidated',\n", - " 'min_date_invalidated',\n", - " 'max_date_invalidated',\n", - " 'has_parent_id',\n", - " 'order_by',\n", - " 'offset',\n", - " 'limit'\n", - " }\n", - "\n", - " def get_messages(self,\n", - " site_ids=None,\n", - " message_text=None,\n", - " min_date_end=None,\n", - " max_date_end=None,\n", - " is_human='either',\n", - " is_valid='either',\n", - " offset=None,\n", - " limit=None\n", - " ):\n", - " qparams = dict(is_human=is_human, is_valid=is_valid)\n", - " if site_ids:\n", - " qparams['site_ids'] = site_ids\n", - " if message_text:\n", - " qparams['message_text'] = message_text\n", - " if min_date_end:\n", - " qparams['min_date_end'] = min_date_end\n", - " if max_date_end:\n", - " qparams['max_date_end'] = max_date_end\n", - " if limit:\n", - " qparams['limit'] = limit\n", - "\n", - " qstr = urlencode(qparams)\n", - " url = f'{self.server}/{self.service}/messages?{qstr}'\n", - " try:\n", - " recs = requests.get(url, timeout=self.timeout).json()\n", - " except Exception as err:\n", - " warnings.warn(f'No {self.service} records retrieved: {err}')\n", - " recs = []\n", - " if len(recs) == 0:\n", - " raise Exception(f'No records retrieved from {url}')\n", - "\n", - " self.recs = recs\n", - " self.recs.sort(key=lambda r: r['date_begin'])\n", - " return recs\n", - "\n", - " def get_timelost(self, rollup='day'):\n", - " day_tl = dict() # day_tl[day] = totalDayTimeLost\n", - " for day,dayrecs in itertools.groupby(\n", - " self.recs,\n", - " key=lambda r: datetime.fromisoformat(r['date_begin']).date().isoformat()\n", - " ):\n", - " day_tl[day] = sum([r['time_lost'] for r in dayrecs])\n", - " return day_tl\n", - "\n", - "class ExposurelogAdapter(ApiAdapter):\n", - " service = 'exposurelog'\n", - " primary_endpoint = 'messages'\n", - " fields = {'date_added',\n", - " 'date_invalidated',\n", - " 'day_obs',\n", - " 'exposure_flag',\n", - " 'id',\n", - " 'instrument',\n", - " 'is_human',\n", - " 'is_valid',\n", - " 'level',\n", - " 'message_text',\n", - " 'obs_id',\n", - " 'parent_id',\n", - " 'seq_num',\n", - " 'site_id',\n", - " 'tags',\n", - " 'urls',\n", - " 'user_agent',\n", - " 'user_id'}\n", - " filters = {\n", - " 'site_ids',\n", - " 'obs_id',\n", - " 'instruments',\n", - " 'min_day_obs', # inclusive, integer in form YYYMMDD\n", - " 'max_day_obs', # exclusive, integer in form YYYMMDD\n", - " 'min_seq_num',\n", - " 'max_seq_num',\n", - " 'message_text', # Message text contain ...\n", - " 'min_level', # inclusive\n", - " 'max_level', # exclusive\n", - " 'tags', # at least one must be present.\n", - " 'urls',\n", - " 'exclude_tags', # all must be absent\n", - " 'user_ids',\n", - " 'user_agents',\n", - " 'is_human', # Allowed: either, true, false; Default=either\n", - " 'is_valid', # Allowed: either, true, false; Default=true\n", - " 'exposure_flags',\n", - " 'min_date_added', # inclusive, TAI ISO string, no TZ\n", - " 'max_date_added', # exclusive, TAI ISO string, no TZ\n", - " 'has_date_invalidated',\n", - " 'min_date_invalidated',\n", - " 'max_date_invalidated',\n", - " 'has_parent_id',\n", - " 'order_by',\n", - " 'offset',\n", - " 'limit'\n", - " }\n", - "\n", - "\n", - "\n", - " def get_instruments(self):\n", - " url = f'{self.server}/{self.service}/instruments'\n", - " try:\n", - " instruments = requests.get(url, timeout=self.timeout).json()\n", - " except Exception as err:\n", - " warnings.warn(f'No instruments retrieved: {err}')\n", - " instruments = dict(dummy=[])\n", - " # Flatten the lists\n", - " return list(itertools.chain.from_iterable(instruments.values()))\n", - "\n", - " def get_exposures(self, instrument, registry=1):\n", - " qparams = dict(instrument=instrument, registery=registry)\n", - " url = f'{self.server}/{self.service}/exposures?{urlencode(qparams)}'\n", - " try:\n", - " recs = requests.get(url, timeout=self.timeout).json()\n", - " except Exception as err:\n", - " warnings.warn(f'No exposures retrieved: {err}')\n", - " recs = []\n", - " return recs\n", - "\n", - " def get_messages(self,\n", - " site_ids=None,\n", - " obs_ids=None,\n", - " instruments=None,\n", - " message_text=None,\n", - " min_day_obs=None,\n", - " max_day_obs=None,\n", - " is_human='either',\n", - " is_valid='either',\n", - " exposure_flags=None,\n", - " offset=None,\n", - " limit=None\n", - " ):\n", - " qparams = dict(is_human=is_human, is_valid=is_valid)\n", - " if site_ids:\n", - " qparams['site_ids'] = site_ids\n", - " if obs_ids:\n", - " qparams['obs_ids'] = obs_ids\n", - " if instruments:\n", - " qparams['instruments'] = instruments\n", - " if min_day_obs:\n", - " qparams['min_day_obs'] = min_day_obs\n", - " if max_day_obs:\n", - " qparams['max_day_obs'] = max_day_obs\n", - " if exposure_flags:\n", - " qparams['exposure_flags'] = exposure_flags\n", - " if offset:\n", - " qparams['offset'] = offset\n", - " if limit:\n", - " qparams['limit'] = limit\n", - "\n", - " qstr = urlencode(qparams)\n", - " url = f'{self.server}/{self.service}/messages?{qstr}'\n", - " try:\n", - " recs = requests.get(url, timeout=self.timeout).json()\n", - " except Exception as err:\n", - " warnings.warn(f'No {self.service} records retrieved: {err}')\n", - " recs = []\n", - " if len(recs) == 0:\n", - " raise Exception(f'No records retrieved from {url}')\n", - "\n", - " self.recs = recs\n", - " self.recs.sort(key=lambda r: r['day_obs'])\n", - " return recs\n", - "\n", - " def get_observation_gaps(self, instruments=None,\n", - " min_day_obs=None, # YYYYMMDD\n", - " max_day_obs=None, # YYYYMMDD\n", - " ):\n", - " if not instruments:\n", - " instruments = self.get_instruments()\n", - " assert isinstance(instruments,list), \\\n", - " f'\"instruments\" must be a list. Got {instruments!r}'\n", - " # inst_day_rollupol[instrument] => dict[day] => exposureGapInMinutes\n", - " inst_day_rollup = defaultdict(dict) # Instrument/Day rollup\n", - "\n", - " for instrum in instruments:\n", - " recs = self.get_exposures(instrum)\n", - " instrum_gaps = dict()\n", - " for day,dayrecs in itertools.groupby(recs,\n", - " key=lambda r: r['day_obs']):\n", - " gaps = list()\n", - " begin = end = None\n", - " for rec in dayrecs:\n", - " begin = rec['timespan_begin']\n", - " if end:\n", - " # span in minutes\n", - " diff = (datetime.fromisoformat(begin)\n", - " - datetime.fromisoformat(end)\n", - " ).total_seconds() / 60.0\n", - "\n", - " gaps.append((\n", - " datetime.fromisoformat(end).time().isoformat(),\n", - " datetime.fromisoformat(begin).time().isoformat(),\n", - " diff\n", - " ))\n", - " end = rec['timespan_end']\n", - " instrum_gaps[day] = gaps\n", - "\n", - " #!roll = dict()\n", - " # Rollup gap times by day\n", - " for day,tuples in instrum_gaps.items():\n", - " #!roll[day] = sum([t[2] for t in tuples])\n", - " inst_day_rollup[instrum][day] = sum([t[2] for t in tuples])\n", - "\n", - " return inst_day_rollup\n", - "\n", - "\n", - "\n", - "# gaps,recs = logrep_utils.ExposurelogAdapter(server_url='https://usdf-rsp-dev.slac.stanford.edu').get_observation_gaps('LSSTComCam')\n", - "\n", - "# gaps,recs = logrep_utils.ExposurelogAdapter(server_url='[[https://tucson-teststand.lsst.codes').get_observation_gaps('LSSTComCam')\n" - ] - }, - { - "cell_type": "markdown", - "id": "7", - "metadata": {}, - "source": [ - "\n", - "## Get Records" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8", - "metadata": {}, - "outputs": [], - "source": [ - "service_adapter = NarrativelogAdapter(server_url=server)\n", - "try:\n", - " recs = service_adapter.get_messages()\n", - "except Exception as err:\n", - " recs = []\n", - " msg = f'ERROR getting records from {server=}: {err=}'\n", - " raise Exception(msg)\n", - "\n", - "metrics = service_adapter.analytics(recs)\n", - "flds = metrics['fields']\n", - "facets = metrics['facets'] # facets(field) = set(value-1, value-2, ...)\n", - "print(f'Retrieved {len(recs)} records, each with {len(flds)} fields.')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9", - "metadata": {}, - "outputs": [], - "source": [ - "date_tl = service_adapter.get_timelost()" - ] - }, - { - "cell_type": "markdown", - "id": "10", - "metadata": {}, - "source": [ - "\n", - "## Tables of (mostly raw) results" - ] - }, - { - "cell_type": "markdown", - "id": "11", - "metadata": {}, - "source": [ - "### Fields names provided in records from log." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "12", - "metadata": {}, - "outputs": [], - "source": [ - "pd.DataFrame(flds, columns=['Field Name'])" - ] - }, - { - "cell_type": "markdown", - "id": "13", - "metadata": {}, - "source": [ - "### Facets from log records.\n", - "A *facet* is the set all of values found for a field in the retrieved records. Facets are only calculated for some fields." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "14", - "metadata": {}, - "outputs": [], - "source": [ - "display(pd.DataFrame.from_dict(facets, orient='index'))\n", - "facets" - ] - }, - { - "cell_type": "markdown", - "id": "15", - "metadata": {}, - "source": [ - "### Show Retrieved Records" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "16", - "metadata": {}, - "outputs": [], - "source": [ - "ignore_fields = ['id']\n", - "new_column_names = dict(message_text='message',\n", - " primary_software_components='PSC',\n", - " primary_hardware_components='PHC',\n", - " )\n", - "\n", - "df = pd.DataFrame(recs).rename(columns=new_column_names)\n", - "with pd.option_context('display.max_rows', None, 'display.max_columns', None):\n", - " display(df.loc[:, ~df.columns.isin(ignore_fields)])" - ] - }, - { - "cell_type": "markdown", - "id": "17", - "metadata": {}, - "source": [ - "### Table of selected log record fields.\n", - "Table can be retrieved as CSV file for local use." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "18", - "metadata": {}, - "outputs": [], - "source": [ - "cols = ['date_added', 'time_lost', 'time_lost_type']\n", - "df = pd.DataFrame(recs)[cols]\n", - "\n", - "# Allow download of CSV version of DataFrame\n", - "csvfile = 'tl.csv'\n", - "df.to_csv(csvfile)\n", - "myfile = FileLink(csvfile)\n", - "print('Table available as CSV file: ')\n", - "display(myfile)\n", - "df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "19", - "metadata": {}, - "outputs": [], - "source": [ - "cols = ['message_text','tags','user_id', 'components','date_end']\n", - "df = pd.DataFrame(recs, columns=cols)\n", - "df" - ] - }, - { - "cell_type": "markdown", - "id": "20", - "metadata": {}, - "source": [ - "\n", - "## Plots from log (per day)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "21", - "metadata": {}, - "outputs": [], - "source": [ - "display_markdown(f'### Date vs Time Lost', raw=True)\n", - "x = date_tl.keys()\n", - "y = date_tl.values()\n", - "\n", - "df = pd.DataFrame(dict(day=x,time_lost=y))\n", - "df.plot.bar(x='day', y='time_lost', title=f'Time Lost')\n", - "display(df)" - ] - }, - { - "cell_type": "markdown", - "id": "22", - "metadata": {}, - "source": [ - "\n", - "## Raw Content Analysis" - ] - }, - { - "cell_type": "markdown", - "id": "23", - "metadata": {}, - "source": [ - "### Example of one record" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "24", - "metadata": {}, - "outputs": [], - "source": [ - "rec = recs[-1]\n", - "\n", - "msg = rec[\"message_text\"]\n", - "md = f'Message text from log:\\n> {msg}'\n", - "display_markdown(md, raw=True)\n", - "\n", - "md = f'One full record (the last one retrieved):\\n> {rec}'\n", - "display_markdown(md, raw=True)\n", - "\n", - "display(rec)" - ] - }, - { - "cell_type": "markdown", - "id": "25", - "metadata": {}, - "source": [ - "\n", - "## Stakeholder Elicitation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "26", - "metadata": {}, - "outputs": [], - "source": [ - "print(f'Finished run: {str(datetime.now())}')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "27", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks_tsqr/sources_dashboard.ipynb b/notebooks_tsqr/sources_dashboard.ipynb deleted file mode 100644 index eb05fca..0000000 --- a/notebooks_tsqr/sources_dashboard.ipynb +++ /dev/null @@ -1,74 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "0", - "metadata": {}, - "outputs": [], - "source": [ - "from lsst.ts.logging_and_reporting.source_adapters import Dashboard" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1", - "metadata": {}, - "outputs": [], - "source": [ - "status = Dashboard().report(timeout=0.5)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2", - "metadata": {}, - "outputs": [], - "source": [ - "status" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, - "outputs": [], - "source": [ - "successes = [k for k,v in status.items() if v == 200]\n", - "successes" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/python/lsst/ts/logging_and_reporting/almanac.py b/python/lsst/ts/logging_and_reporting/almanac.py new file mode 100644 index 0000000..a527629 --- /dev/null +++ b/python/lsst/ts/logging_and_reporting/almanac.py @@ -0,0 +1,81 @@ +from datetime import datetime, date, time, timedelta +import math +from astroplan import Observer +from astropy.time import Time + +class Almanac: + + def __init__(self, *, day_obs=None, site='Rubin'): + if day_obs is None: + astro_day = date.today() - timedelta(days=1) + else: + astro_day = datetime.strptime(str(day_obs), '%Y%m%d').date() + + self.observer = Observer.at_site(site, timezone='Chile/Continental') + self.astro_day = astro_day + self.astro_noon = datetime.combine(self.astro_day,time(12)) + + self.get_moon() + self.get_sun() + + def get_moon(self): + self.moon_rise_time = self.observer.moon_rise_time(self.astro_noon) + self.moon_set_time = self.observer.moon_set_time(self.astro_noon) + + # Percent of moon lit + self.moon_illum = self.observer.moon_illumination(self.astro_noon) + + def get_sun(self): + time = self.observer.datetime_to_astropy_time(self.astro_noon) + + # ast(ronoimical) twilight: -18 degrees) + self.ast_twilight_morning = self.observer.twilight_morning_astronomical( + time) + self.ast_twilight_evening = self.observer.twilight_evening_astronomical( + time) + + # nau(tical) twilight: -12 degrees) + self.nau_twilight_morning = self.observer.twilight_morning_nautical( + time) + self.nau_twilight_evening = self.observer.twilight_evening_nautical( + time) + + # civ(il) twilight: -6 degrees) + self.civ_twilight_morning = self.observer.twilight_morning_civil( + time) + self.civ_twilight_evening = self.observer.twilight_evening_civil( + time) + + self.sun_rise_time = self.observer.sun_rise_time(time) + self.sun_set_time = self.observer.sun_set_time(time) + + @property + def as_dict(self): + data_dict = { + 'Moon Rise': self.moon_rise_time.iso, + 'Moon Set': self.moon_set_time.iso, + 'Moon Illumination': f'{self.moon_illum:.0%}', + + 'Astronomical Twilight (morning)': self.ast_twilight_morning.iso, + 'Astronomical Twilight (evening)': self.ast_twilight_evening.iso, + 'Nautical Twilight (morning)': self.nau_twilight_morning.iso, + 'Nautical Twilight (evening)': self.nau_twilight_evening.iso, + 'Civil Twilight (morning)': self.civ_twilight_morning.iso, + 'Civil Twilight (evening)': self.civ_twilight_evening.iso, + 'Sun Rise': self.sun_rise_time.iso, + 'Sun Set': self.sun_set_time.iso, + } + help_dict = { + 'Moon Rise': '', + 'Moon Set': '', + 'Moon Illumination': '(% lit)', + 'Astronomical Twilight (morning)': '(-18 degrees)', + 'Astronomical Twilight (evening)': '(-18 degrees)', + 'Nautical Twilight (morning)': '(-12 degrees)', + 'Nautical Twilight (evening)': '(-12 degrees)', + 'Civil Twilight (morning)': '(-6 degrees)', + 'Civil Twilight (evening)': '(-6 degrees)', + 'Sun Rise': '', + 'Sun Set': '', + } + return data_dict, help_dict diff --git a/python/lsst/ts/logging_and_reporting/dashboard.py b/python/lsst/ts/logging_and_reporting/dashboard.py new file mode 100644 index 0000000..dddae68 --- /dev/null +++ b/python/lsst/ts/logging_and_reporting/dashboard.py @@ -0,0 +1,100 @@ +# Python Standard Library +from collections import defaultdict +from warnings import warn +import lsst.ts.logging_and_reporting.source_adapters as sad +# External Packages +import requests + + +class Dashboard: # TODO Move to its own file (utils.py). + """Verify that we can get to all the API endpoints and databases we need for + any of our sources. + """ + timeout = 0.8 + + envs = dict( # key, server + summit = 'https://summit-lsp.lsst.codes', + usdf_dev = 'https://usdf-rsp-dev.slac.stanford.edu', + tucson = 'https://tucson-teststand.lsst.codes', + # Environments not currently used: + # rubin_usdf_dev = '', + # data_lsst_cloud = '', + # usdf = '', + # base_data_facility = '', + # rubin_idf_int = '', + ) + adapters = [sad.ExposurelogAdapter, + sad.NarrativelogAdapter, + sad.NightReportAdapter, + ] + + def get_sample_data(self, server): + + samples = defaultdict(dict) # samples[endpoint_url] -> one_record_dict + for adapter in self.adapters: + sa = adapter(server_url=server, limit=1) + for ep in sa.endpoints: + qstr = '?instrument=LSSTComCamSim' if ep == 'exposures' else '' + url = f'{server}/{sa.service}/{ep}{qstr}' + try: + res = requests.get(url, timeout=self.timeout) + recs = res.json() + if isinstance(recs, dict): + samples[url] = recs + else: + samples[url] = recs[0] + except Exception as err: + warn(f'Could not get data from {url}: {res.content=} {err=}') + samples[url] = None + return dict(samples) + + + def report(self, timeout=None): + """Check our ability to connect to every Source on every Environment. + Report a summary. + + RETURN: percentage of good connectons. + """ + url_status = dict() # url_status[endpoint_url] = http _status_code + working = set() # Set of servers that work for all our required endpoints. + + for env,server in self.envs.items(): + server_all_good = True + for adapter in self.adapters: + service = adapter(server_url=server) + stats, adapter_all_good = service.check_endpoints(timeout=timeout) + url_status.update(stats) + server_all_good &= adapter_all_good + if server_all_good: + working.add(server) + + total_cnt = good_cnt = 0 + good = list() + bad = list() + for url,stat in url_status.items(): + total_cnt += 1 + if stat == 200: + good_cnt += 1 + good.append(url) + else: + bad.append((url,stat)) + + print(f'\nConnected to {good_cnt} out of {total_cnt} endpoints.' + f'({good_cnt/total_cnt:.0%})' + ) + goodstr = "\n\t".join(good) + print(f'Successful connects ({good_cnt}): ') + for gurl in good: + print(f'\t{gurl}') + + print(f'Failed connects ({total_cnt - good_cnt}): ') + for burl,stat in bad: + print(f'\t{stat}: {burl}') + + status = dict(num_good=good_cnt, + num_total=total_cnt, + good_urls=good, + bad_ursl=bad, + ) + return good_cnt/total_cnt, working +# END: class Dashboard diff --git a/python/lsst/ts/logging_and_reporting/exceptions.py b/python/lsst/ts/logging_and_reporting/exceptions.py new file mode 100644 index 0000000..07fe44f --- /dev/null +++ b/python/lsst/ts/logging_and_reporting/exceptions.py @@ -0,0 +1,78 @@ +# This file is part of ts_logging_and_reporting. +# +# Developed for Vera C. Rubin Observatory Telescope and Site Systems. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# error_code values should be no bigger than 8 characters 12345678 +import traceback +from warnings import warn + +class BaseLogrepException(Exception): + is_an_error_response = True + status_code = 400 + error_message = '' + saved_tb = None + + def get_subclass_name(self): + return self.__class__.__name__ + + def __init__(self, error_message, error_code=None, status_code=None): + Exception.__init__(self) + self.error_message = error_message + if error_code: + assert len(error_code) <= 8, f'error_code "{error_code}" too big' + self.error_code = error_code + + if error_code is not None: + self.error_code = error_code + if status_code is not None: + self.status_code = status_code or self.status_code + + self.saved_tb = traceback.format_exc() + + def __str__(self): + return (f'[{self.error_code}] {self.error_message}' + f' {self.saved_tb=}') + + def to_dict(self): + dd = dict(errorMessage=self.error_message, + errorCode=self.error_code, + #! trace=self.saved_tb, + statusCode=self.status_code) + return dd + + +example_error_from_exposurelog = { + 'detail': [ + {'type': 'int_parsing', + 'loc': ['query', 'min_day_obs'], + 'msg': 'Input should be a valid integer, unable to parse string as an integer', + 'input': '2024-08-19'}, + {'type': 'int_parsing', + 'loc': ['query', 'max_day_obs'], + 'msg': 'Input should be a valid integer, unable to parse string as an integer', + 'input': '2024-09-21'}]} + + +class BadStatus(BaseLogrepException): + """Non-200 HTTP status from API endpoint. Typically + this will occur when a URL query string parameter is passed a value with + a bad format. It may also be that the Service is broken. + """ + error_code = 'BADQSTR' diff --git a/python/lsst/ts/logging_and_reporting/reports.py b/python/lsst/ts/logging_and_reporting/reports.py new file mode 100644 index 0000000..ef63418 --- /dev/null +++ b/python/lsst/ts/logging_and_reporting/reports.py @@ -0,0 +1,135 @@ +# This file is part of ts_logging_and_reporting. +# +# Developed for Vera C. Rubin Observatory Telescope and Site Systems. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Python Standard Library +from urllib.parse import urlencode +import itertools +from datetime import datetime, date, time, timedelta +from warnings import warn +from collections import defaultdict +from abc import ABC +# External Packages +import requests +from IPython.display import display, Markdown +import pandas as pd +# Local Packages +import lsst.ts.logging_and_reporting.almanac as alm + +def md(markdown_str, color=None): + # see https://www.w3schools.com/colors/colors_names.asp + if color: + display(Markdown(f"{markdown_str}")) + else: + display(Markdown(markdown_str)) + +def mdlist(markdown_list, color=None): + if markdown_list is None: + return + + for markdown_str in markdown_list: + md(markdown_str, color=color) + + +def dict_to_md(in_dict): + md_list = list() + for key, content_list in in_dict.items(): + md_list.append(f'- {key}') + for elem in content_list: + md_list.append(f' - {elem}') + return md_list + +def adapter_overview(adapter, status, limit): + count = status["number_of_records"] + error = status["error"] + more = '(There may be more.)' if count >= limit else '' + result = error if error else f'Got {count} records. ' + mdlist([f'## Overview for Service: `{adapter.service}` [{count}]', + f'- Endpoint: {status["endpoint_url"]}', + ]) + print(f'- {result} {more}') + + +# TODO move all instances of "row_header", "row_str_func" from source_adapters to here. +class Report(ABC): + def __init__(self, *, + min_day_obs=None, # INCLUSIVE: default=Yesterday + max_day_obs=None, # EXCLUSIVE: default=Today + ): + self.min_day_obs = min_day_obs + self.max_day_obs = max_day_obs + + def time_log_as_markdown(self, source_adapter, url, + log_title=None, + zero_message=False, + ): + records = source_adapter.records + service = source_adapter.service + title = log_title if log_title else '' + if records: + md(f'### {title}') + table = source_adapter.day_table('date_added') + mdlist(table) + else: + if zero_message: + md(f'No {service} records found.', color='lightblue') + md(f'Used [API Data]({url})') + +class AlmanacReport(Report): + # moon rise,set,illumination % + # (astronomical,nautical,civil) twilight (morning,evening) + # sun rise,set + def almanac_as_dataframe(self): + # This display superfluous header: "0, 1" + return pd.DataFrame(alm.Almanac().as_dict).T + + +class NightlyLogReport(Report): + + def block_tickets_as_markdown(self, tickets, + title='## Nightly Jira BLOCKs'): + # tickets[day_obs] = {ticket_url, ...} + mdstr = '' + if title: + mdstr += title + + for day, url_list in tickets.items(): + mdstr += f'\n- {day}' + for ticket_url in url_list: + mdstr += f'\n - [{ticket_url.replace(front,"")}]({ticket_url})' + return mdstr + + +class ExposurelogReport(Report): + + # date, time, obs_id, message_text + def time_log_as_markown(self, records, + title='# Exposure Log'): + pass # TODO use "day_table" + + def daily_observation_gap(self, min_day_obs, max_day_obs): + pass + +class NarrativelogReport(Report): + + # date, time, message_text + def time_log_as_markown(self, records, + title='# Exposure Log'): + pass # TODO use "day_table" diff --git a/python/lsst/ts/logging_and_reporting/source_adapters.py b/python/lsst/ts/logging_and_reporting/source_adapters.py index 8679dbc..cc32d3f 100644 --- a/python/lsst/ts/logging_and_reporting/source_adapters.py +++ b/python/lsst/ts/logging_and_reporting/source_adapters.py @@ -20,92 +20,171 @@ # along with this program. If not, see . -############################################ +'''\ +TODO: This is considered Proof of Concept code. +Tests and documentation exist minimally or not at all since until the +concept is Proven, it all might be thrown away or rewritten. +''' + # Python Standard Library from urllib.parse import urlencode import itertools -from datetime import datetime +from datetime import datetime, date, time, timedelta from warnings import warn from collections import defaultdict from abc import ABC - -############################################ # External Packages import requests +# Local Packages +import lsst.ts.logging_and_reporting.utils as ut +import lsst.ts.logging_and_reporting.exceptions as ex -MAX_CONNECT_TIMEOUT = 3.1 # seconds -MAX_READ_TIMEOUT = 90 * 60 # seconds -def keep_fields(outfields, recs): - """Keep only keys in OUTFIELDS list of RECS (list of dicts) - SIDE EFFECT: Removes extraneous keys from all dicts in RECS. - """ - if outfields: - for rec in recs: - nukefields = set(rec.keys()) - set(outfields) - print(f'{rec=} {nukefields=}') - for f in nukefields: - del rec[f] +MAX_CONNECT_TIMEOUT = 3.1 # seconds +MAX_READ_TIMEOUT = 180 # seconds + + +def all_endpoints(server): + endpoints = itertools.chain.from_iterable( + [sa(server_url=server).used_endpoints() for sa in adapters] + ) + return list(endpoints) + + +def validate_response(response, endpoint_url): + if response.status_code == 200: + return True + else: + msg = f'Error: {response.json()} {endpoint_url=} {response.reason}' + raise ex.BadStatus(msg) + + class SourceAdapter(ABC): """Abstract Base Class for all source adapters. """ # TODO document class including all class variables. def __init__(self, *, server_url='https://tucson-teststand.lsst.codes', + min_day_obs=None, # INCLUSIVE: default=Yesterday + max_day_obs=None, # EXCLUSIVE: default=Today other=YYYY-MM-DD + limit=99, + offset=0, connect_timeout=1.05, # seconds read_timeout=2, # seconds ): + if min_day_obs is None: # Inclusive + min_day_obs = ut.datetime_to_day_obs( + datetime.today() - timedelta(days=1)) + if max_day_obs is None: # Exclusive + max_day_obs = ut.datetime_to_day_obs( + datetime.today() + timedelta(days=1)) self.server = server_url + self.min_day_obs = min_day_obs + self.max_day_obs = max_day_obs + self.min_date = ut.get_datetime_from_day_obs_str(min_day_obs) + self.max_date = ut.get_datetime_from_day_obs_str(max_day_obs) + self.limit = limit + self.offset = offset self.c_timeout = min(MAX_CONNECT_TIMEOUT, float(connect_timeout)) # seconds self.r_timeout = min(MAX_READ_TIMEOUT, # seconds float(read_timeout)) self.timeout = (self.c_timeout, self.r_timeout) + self.records = None # else: list of dict # Provide the following in subclass output_fields = None service = None endpoints = None + + + def keep_fields(self, recs, outfields): + """Keep only keys in OUTFIELDS list of RECS (list of dicts) + SIDE EFFECT: Removes extraneous keys from all dicts in RECS. + """ + if outfields: + for rec in recs: + nukefields = set(rec.keys()) - set(outfields) + for f in nukefields: + del rec[f] + + @property + def row_header(self): + return '| Time | Message |\n|--------|------|' + + def row_str_func(self, datetime_str, rec): + msg = rec['message_text'] + return f'> {datetime_str} | 
{msg}
' + + + # Break on DAY_OBS. Within that, break on DATE, within that only show time. + def day_table(self, datetime_field, + dayobs_field=None, + row_str_func=None, + zero_message=False, + ): + def obs_night(rec): + if 'day_obs' in rec: + return ut.day_obs_str(rec['day_obs']) # -> # "YYYY-MM-DD" + else: + dt = datetime.fromisoformat(rec[datetime_field]) + return ut.datetime_to_day_obs(dt) + + def obs_date(rec): + dt = datetime.fromisoformat(rec[datetime_field]) + return dt.replace(microsecond=0) + + recs = self.records + if len(recs) == 0: + if zero_message: + print('Nothing to display.') + return + dates = set([obs_date(r).date() for r in recs]) + table = list() + # Group by night. + recs = sorted(recs,key=lambda r: obs_night(r)) + for night,g0 in itertools.groupby(recs, key=lambda r: obs_night(r)): + # Group by date + table.append(f'## NIGHT: {night}: ') + for date,g1 in itertools.groupby(g0, key=lambda r: obs_date(r)): + table.append(f'### DATE: {date.date()}: ') + for rec in g0: + dt = obs_date(rec) + dtstr = str(dt.time()) + table.append(f'{self.row_str_func(dtstr, rec)}') + table.append(':EOT') + return table + + @property def source_url(self): return f'{self.server}/{self.service}' - - def check_endpoints(self, timeout=None): - to = (timeout or self.timeout) - print(f'Try connect to each endpoint of {self.server}/{self.service} ' - f'using timeout={to}.') - url_http_status_code = dict() + def used_endpoints(self): + used = list() for ep in self.endpoints: - url = f'{self.server}/{self.service}/{ep}' - try: - r = requests.get(url, timeout=(timeout or self.timeout)) - except: - url_http_status_code[url] = 'timeout' - else: - url_http_status_code[url] = r.status_code - return url_http_status_code - - service = None - endpoints = None + used.append(f'{self.server}/{self.service}/{ep}') + return used - def check_endpoints(self, timeout=None): + def check_endpoints(self, timeout=None, verbose=True): to = (timeout or self.timeout) - print(f'Try connect to each endpoint of {self.server}/{self.service} ' - f'using timeout={to}.') + if verbose: + print(f'Try connect to each endpoint of' + f' {self.server}/{self.service} ') url_http_status_code = dict() for ep in self.endpoints: url = f'{self.server}/{self.service}/{ep}' try: r = requests.get(url, timeout=(timeout or self.timeout)) - except: - url_http_status_code[url] = 'timeout' + validate_response(r, url) + except Exception as err: + url_http_status_code[url] = 'GET error' else: url_http_status_code[url] = r.status_code - return url_http_status_code + return url_http_status_code, all([v==200 for v in url_http_status_code.values()]) def analytics(self, recs, categorical_fields=None): @@ -135,6 +214,7 @@ def analytics(self, recs, categorical_fields=None): facet_fields=facflds, facets=facets, ) +# END: class SourceAdapter # Not available on SLAC (usdf) as of 9/9/2024. @@ -142,136 +222,166 @@ class NightReportAdapter(SourceAdapter): service = "nightreport" endpoints = ['reports'] primary_endpoint = 'reports' + outfields = { + 'confluence_url', + 'date_added', + 'date_invalidated', + 'date_sent', + 'day_obs', + 'id', + 'is_valid', + 'observers_crew', + 'parent_id', + 'site_id', + 'summary', + 'telescope', + 'telescope_status', + 'user_agent', + 'user_id', + } + + def row_str_func(self, datetime_str, rec): + return f"> {datetime_str} | 
{rec['summary']}
" + + def get_reports(self, + site_ids=None, + summary=None, + is_human='either', + is_valid='either', + ): + qparams = dict(is_human=is_human, is_valid=is_valid) + if site_ids: + qparams['site_ids'] = site_ids + if summary: + qparams['summary'] = summary + if self.min_day_obs: + qparams['min_day_obs'] = ut.day_obs_int(self.min_day_obs) + if self.max_day_obs: + qparams['max_day_obs'] = ut.day_obs_int(self.max_day_obs) + if self.limit: + qparams['limit'] = self.limit + + qstr = urlencode(qparams) + url = f'{self.server}/{self.service}/reports?{qstr}' + error = None + try: + response = requests.get(url, timeout=self.timeout) + validate_response(response, url) + recs = response.json() + recs.sort(key=lambda r: r['day_obs']) + except Exception as err: + recs = [] + error = f'{response.text=} Exception={err}' + + self.keep_fields(recs, self.outfields) + self.records = recs + status = dict( + endpoint_url=url, + number_of_records=len(recs), + error=error, + ) + return status + + def nightly_tickets(self, recs): + tickets = defaultdict(set) # tickets[day_obs] = {ticket_url, ...} + for r in recs: + ticket_url = r['confluence_url'] + if ticket_url: + tickets[r['day_obs']].add(ticket_url) + return {dayobs:list(urls) for dayobs,urls in tickets.items()} + class NarrativelogAdapter(SourceAdapter): - """TODO full documentation - """ service = 'narrativelog' endpoints = ['messages',] primary_endpoint = 'messages' - fields = {'category', - 'components', - 'cscs', - 'date_added', - 'date_begin', - 'date_end', - 'date_invalidated', - 'id', - 'is_human', - 'is_valid', - 'level', - 'message_text', - 'parent_id', - 'primary_hardware_components', - 'primary_software_components', - 'site_id', - 'subsystems', - 'systems', - 'tags', - 'time_lost', - 'time_lost_type', - 'urls', - 'user_agent', - 'user_id'} - filters = { - 'site_ids', - 'message_text', # Message text contain ... - 'min_level', # inclusive - 'max_level', # exclusive - 'user_ids', - 'user_agents', - 'categories', - 'exclude_categories', - 'time_lost_types', - 'exclude_time_lost_types', - 'tags', # at least one must be present. - 'exclude_tags', # all must be absent - 'systems', - 'exclude_systems', - 'subsystems', - 'exclude_subsystems', - 'cscs', - 'exclude_cscs', - 'components', - 'exclude_components', - 'primary_software_components', - 'exclude_primary_software_components', - 'primary_hardware_components', - 'exclude_primary_hardware_components', - 'urls', - 'min_time_lost', - 'max_time_lost', - 'has_date_begin', - 'min_date_begin', - 'max_date_begin', - 'has_date_end', - 'min_date_end', - 'max_date_end', - 'is_human', # Allowed: either, true, false; Default=either - 'is_valid', # Allowed: either, true, false; Default=true - 'min_date_added', # inclusive, TAI ISO string, no TZ - 'max_date_added', # exclusive, TAI ISO string, no TZ - 'has_date_invalidated', - 'min_date_invalidated', - 'max_date_invalidated', - 'has_parent_id', - 'order_by', - 'offset', - 'limit' + outfields = { + # 'category', + # 'components', + # 'cscs', + 'date_added', + # 'date_begin', + # 'date_end', + # 'date_invalidated', + # 'id', + # 'is_human', + # 'is_valid', + # 'level', + 'message_text', + # 'parent_id', + # 'primary_hardware_components', + # 'primary_software_components', + # 'site_id', + # 'subsystems', + # 'systems', + # 'tags', + 'time_lost', + 'time_lost_type', + # 'urls', + # 'user_agent', + # 'user_id', } def get_messages(self, site_ids=None, message_text=None, - min_date_end=None, - max_date_end=None, is_human='either', is_valid='either', offset=None, - limit=None, - outfields=None, ): - qparams = dict(is_human=is_human, is_valid=is_valid) + qparams = dict( + is_human=is_human, + is_valid=is_valid, + order_by='-date_begin', + ) if site_ids: qparams['site_ids'] = site_ids if message_text: qparams['message_text'] = message_text - if min_date_end: - qparams['min_date_end'] = min_date_end - if max_date_end: - qparams['max_date_end'] = max_date_end - if limit: - qparams['limit'] = limit + if self.min_day_obs: + qparams['min_date_added'] = datetime.combine( + self.min_date, time() + ).isoformat() + if self.max_day_obs: + qparams['max_date_added'] = datetime.combine( + self.max_date, time() + ).isoformat() + if self.limit: + qparams['limit'] = self.limit qstr = urlencode(qparams) url = f'{self.server}/{self.service}/messages?{qstr}' + error = None try: - recs = requests.get(url, timeout=self.timeout).json() + r = requests.get(url, timeout=self.timeout) + validate_response(r, url) + recs = r.json() recs.sort(key=lambda r: r['date_begin']) except Exception as err: - warn(f'No {self.service} records retrieved: {err}') recs = [] + error = str(err) - keep_fields(outfields, recs) - self.recs = recs - return self.recs + self.keep_fields(recs, self.outfields) + self.records = recs + status = dict( + endpoint_url=url, + number_of_records=len(recs), + error=error, + ) + return status + + def get_timelost(self, recs, rollup='day'): + def iso_date_begin(rec): + return datetime.fromisoformat(rec['date_begin']).date().isoformat() - def get_timelost(self, rollup='day'): day_tl = dict() # day_tl[day] = totalDayTimeLost - for day,dayrecs in itertools.groupby( - self.recs, - key=lambda r: datetime.fromisoformat(r['date_begin']).date().isoformat() - ): + for day,dayrecs in itertools.groupby(recs, key=iso_date_begin): day_tl[day] = sum([r['time_lost'] for r in dayrecs]) return day_tl +# END: class NarrativelogAdapter -class ExposurelogAdapter(SourceAdapter): - """TODO full documentation - EXAMPLES: - gaps,recs = logrep_utils.ExposurelogAdapter(server_url='https://usdf-rsp-dev.slac.stanford.edu').get_observation_gaps('LSSTComCam') - gaps,recs = logrep_utils.ExposurelogAdapter(server_url='[[https://tucson-teststand.lsst.codes').get_observation_gaps('LSSTComCam') - """ +class ExposurelogAdapter(SourceAdapter): ignore_fields = ['id'] service = 'exposurelog' endpoints = [ @@ -280,59 +390,46 @@ class ExposurelogAdapter(SourceAdapter): 'messages', ] primary_endpoint = 'messages' - fields = {'date_added', - 'date_invalidated', - 'day_obs', - 'exposure_flag', - 'id', - 'instrument', - 'is_human', - 'is_valid', - 'level', - 'message_text', - 'obs_id', - 'parent_id', - 'seq_num', - 'site_id', - 'tags', - 'urls', - 'user_agent', - 'user_id'} - filters = { - 'site_ids', + outfields = { + 'date_added', + # 'date_invalidated', + 'day_obs', + # 'exposure_flag', + # 'id', + 'instrument', + # 'is_human', + # 'is_valid', + # 'level', + 'message_text', 'obs_id', - 'instruments', - 'min_day_obs', # inclusive, integer in form YYYMMDD - 'max_day_obs', # exclusive, integer in form YYYMMDD - 'min_seq_num', - 'max_seq_num', - 'message_text', # Message text contain ... - 'min_level', # inclusive - 'max_level', # exclusive - 'tags', # at least one must be present. - 'urls', - 'exclude_tags', # all must be absent - 'user_ids', - 'user_agents', - 'is_human', # Allowed: either, true, false; Default=either - 'is_valid', # Allowed: either, true, false; Default=true - 'exposure_flags', - 'min_date_added', # inclusive, TAI ISO string, no TZ - 'max_date_added', # exclusive, TAI ISO string, no TZ - 'has_date_invalidated', - 'min_date_invalidated', - 'max_date_invalidated', - 'has_parent_id', - 'order_by', - 'offset', - 'limit' - } + # 'parent_id', + # 'seq_num', + # 'site_id', + # 'tags', + # 'urls', + # 'user_agent', + # 'user_id', + } + + @property + def row_header(self): + return('| Time | OBS ID | Telescope | Message |\n' + '|------|--------|-----------|---------|' + ) + def row_str_func(self, datetime_str, rec): + return(f"> {datetime_str} " + f"| {rec['obs_id']} " + f"| {rec['instrument']} " + f"| 
{rec['message_text']}
" + ) - def check_endpoints(self, timeout=None): + + def check_endpoints(self, timeout=None, verbose=True): to = (timeout or self.timeout) - print(f'Try connect to each endpoint of {self.server}/{self.service} ' - f'using timeout={to}.') + if verbose: + print(f'Try connect to each endpoint of ' + f'{self.server}/{self.service} ') url_http_status_code = dict() for ep in self.endpoints: @@ -340,30 +437,34 @@ def check_endpoints(self, timeout=None): url = f'{self.server}/{self.service}/{ep}{qstr}' try: r = requests.get(url, timeout=to) - except: - url_http_status_code[url] = 'timeout' + validate_response(r, url) + except Exception as err: + url_http_status_code[url] = 'GET error' else: url_http_status_code[url] = r.status_code - return url_http_status_code - + return url_http_status_code, all([v==200 for v in url_http_status_code.values()]) def get_instruments(self): url = f'{self.server}/{self.service}/instruments' try: - instruments = requests.get(url, timeout=self.timeout).json() + r = requests.get(url, timeout=self.timeout).json() + validate_response(r, url) + instruments = r.json() except Exception as err: - warn(f'No instruments retrieved: {err}') instruments = dict(dummy=[]) # Flatten the lists return list(itertools.chain.from_iterable(instruments.values())) def get_exposures(self, instrument, registry=1): qparams = dict(instrument=instrument, registery=registry) + if self.min_day_obs: + qparams['min_day_obs'] = ut.day_obs_int(self.min_day_obs) + if self.max_day_obs: + qparams['max_day_obs'] = ut.day_obs_int(self.max_day_obs) url = f'{self.server}/{self.service}/exposures?{urlencode(qparams)}' try: recs = requests.get(url, timeout=self.timeout).json() except Exception as err: - warn(f'No exposures retrieved: {err}') recs = [] return recs @@ -372,63 +473,61 @@ def get_messages(self, obs_ids=None, instruments=None, message_text=None, - min_day_obs=None, - max_day_obs=None, is_human='either', is_valid='either', exposure_flags=None, - offset=None, - limit=None, - outfields=None, ): - qparams = dict(is_human=is_human, is_valid=is_valid) + qparams = dict(is_human=is_human, + is_valid=is_valid, + order_by='-date_added', + ) if site_ids: qparams['site_ids'] = site_ids if obs_ids: qparams['obs_ids'] = obs_ids if instruments: qparams['instruments'] = instruments - if min_day_obs: - qparams['min_day_obs'] = min_day_obs - if max_day_obs: - qparams['max_day_obs'] = max_day_obs + if self.min_day_obs: + qparams['min_day_obs'] = ut.day_obs_int(self.min_day_obs) + if self.max_day_obs: + qparams['max_day_obs'] = ut.day_obs_int(self.max_day_obs) if exposure_flags: qparams['exposure_flags'] = exposure_flags - if offset: - qparams['offset'] = offset - if limit: - qparams['limit'] = limit + if self.limit: + qparams['limit'] = self.limit qstr = urlencode(qparams) url = f'{self.server}/{self.service}/messages?{qstr}' recs = [] + error = None try: response = requests.get(url, timeout=self.timeout) + validate_response(response, url) recs = response.json() except Exception as err: - warnings.warn(f'No {self.service} records retrieved: {err}') - - if len(recs) == 0: - warn(f'No records retrieved from {url}') + recs = [] + error = str(err) if recs: recs.sort(key=lambda r: r['day_obs']) - keep_fields(outfields, recs) - self.recs = recs - return self.recs + self.keep_fields(recs, self.outfields) + self.records = recs + status = dict( + endpoint_url=url, + number_of_records=len(recs), + error=error, + ) + return status + - def get_observation_gaps(self, instruments=None, - min_day_obs=None, # YYYYMMDD - max_day_obs=None, # YYYYMMDD - ): + def get_observation_gaps(self, instruments=None): if not instruments: instruments = self.get_instruments() assert isinstance(instruments,list), \ f'"instruments" must be a list. Got {instruments!r}' - # inst_day_rollupol[instrument] => dict[day] => exposureGapInMinutes + # inst_day_rollup[instrument] => dict[day] => exposureGapInMinutes inst_day_rollup = defaultdict(dict) # Instrument/Day rollup - for instrum in instruments: recs = self.get_exposures(instrum) instrum_gaps = dict() @@ -452,54 +551,14 @@ def get_observation_gaps(self, instruments=None, end = rec['timespan_end'] instrum_gaps[day] = gaps - #!roll = dict() # Rollup gap times by day for day,tuples in instrum_gaps.items(): - #!roll[day] = sum([t[2] for t in tuples]) inst_day_rollup[instrum][day] = sum([t[2] for t in tuples]) return inst_day_rollup +# END: class ExposurelogAdapter - - - -class Dashboard: # TODO Complete and move to its own file. - """Verify that we can get to all the API endpoints and databases we need for - any of our sources. - """ - - envs = dict( - summit = 'https://summit-lsp.lsst.codes', - usdf_dev = 'https://usdf-rsp-dev.slac.stanford.edu', - tucson = 'https://tucson-teststand.lsst.codes', - # Environments not currently used: - # rubin_usdf_dev = '', - # data_lsst_cloud = '', - # usdf = '', - # base_data_facility = '', - # rubin_idf_int = '', - ) - adapters = [ExposurelogAdapter, - NarrativelogAdapter, - # NightReportAdapter, # TODO - ] - - def report(self, timeout=None): - url_status = dict() - for env,server in self.envs.items(): - for adapter in self.adapters: - service = adapter(server_url=server) - # url_status[endpoint_url] = http_status_code - url_status.update(service.check_endpoints(timeout=timeout)) - total_cnt = 0 - good_cnt = 0 - good = list() - print('\nStatus for each endpoint URL:') - for url,stat in url_status.items(): - print(f'{stat}\t{url}') - total_cnt += 1 - if stat == 200: - good_cnt += 1 - good.append(url) - print(f'\nConnected to {good_cnt} out of {total_cnt} endpoints.') - return good_cnt, good +adapters = [ExposurelogAdapter, + NarrativelogAdapter, + NightReportAdapter, + ] diff --git a/python/lsst/ts/logging_and_reporting/utils.py b/python/lsst/ts/logging_and_reporting/utils.py index c6801c8..54534a4 100644 --- a/python/lsst/ts/logging_and_reporting/utils.py +++ b/python/lsst/ts/logging_and_reporting/utils.py @@ -19,6 +19,60 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . + +import time +import datetime # datetime, date, time, timedelta + + +# See https://github.com/lsst-sitcom/summit_utils/blob/0b3fd8795c9cca32f30cef0c37625c5d96804b74/python/lsst/summit/utils/efdUtils.py#L633 +# was: datetime_to_dayobs # TODO remove +def datetime_to_day_obs(dt) -> str: + """Convert a datetime object to day_obs. + Round to the date of the start of the observing night. + Both the input datetime and output dayobs are in the same timezone. + Format of dayobs is + + Parameters + ---------- + dt : `datetime.datetime` + The date-time. + + Returns + ------- + day_obs : `str` + The day_obs, as a strung, e.g. 2023-12-25 (YYYY-MM-DD) + """ + dodate = (dt - datetime.timedelta(hours=12)).date() + return dodate.strftime('%Y-%m-%d') + +# day_obs int to day_obs string (YYYY-MM-DD) +def day_obs_str(day_obs: int) -> str: + dos = str(day_obs) + return f'{dos[0:4]}-{dos[4:6]}-{dos[6:8]}' # "YYYY-MM-DD" + +# day_obs str (YYYY-MM-DD) to day_obs int +def day_obs_int(day_obs: str) -> int: + return int(day_obs.replace('-','')) + +# day_obs (str:YYYY-MM-DD or YYYYMMDD) to datetime. +# Allow TODAY, YESTERDAY, TOMORROW +# was: dos2dt +def get_datetime_from_day_obs_str(day_obs): + match day_obs.lower(): + case 'today': + date = datetime.datetime.now().date() + case 'yesterday': + date = datetime.datetime.now().date() - datetime.timedelta(days=1) + case 'tomorrow': + date = datetime.datetime.now().date() + datetime.timedelta(days=1) + case _: + no_dash = day_obs.replace('-','') + date = datetime.datetime.strptime(no_dash, '%Y%m%d').date() + return date + + + + def tic(): """Start timer. """ diff --git a/python/lsst/ts/logging_and_reporting/version.py b/python/lsst/ts/logging_and_reporting/version.py index ee1c147..07fc7b6 100644 --- a/python/lsst/ts/logging_and_reporting/version.py +++ b/python/lsst/ts/logging_and_reporting/version.py @@ -1,3 +1,3 @@ # Generated by setuptools_scm __all__ = ["__version__"] -__version__ = "0.1.dev28+g968997e.d20240904" +__version__ = "0.1.dev63+g83a61e0.d20240918" diff --git a/requirements.txt b/requirements.txt index c65b6c6..6b7e061 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,10 @@ requests # jupyter-lab --ip=0.0.0.0 jupyterlab # =3.1.17 pandas -#matplotlib -#numpy +matplotlib +numpy pre-commit +astroplan +astropy +lsst_efd_client +#lsst.summit.utils \ No newline at end of file diff --git a/times-square.yaml b/times-square.yaml index 0a976ce..6f8ca8e 100644 --- a/times-square.yaml +++ b/times-square.yaml @@ -1,8 +1,6 @@ enabled: true root: notebooks_tsqr ignore: - - efd_* - logrep_proto_* - - TEMPLATE_* description: > Times Square for project-wide Logging and Reporting.