diff --git a/notebooks_tsqr/NightLog.ipynb b/notebooks_tsqr/NightLog.ipynb index f0dd6d7..1708471 100644 --- a/notebooks_tsqr/NightLog.ipynb +++ b/notebooks_tsqr/NightLog.ipynb @@ -3,9 +3,7 @@ { "cell_type": "markdown", "id": "0", - "metadata": { - "jp-MarkdownHeadingCollapsed": true - }, + "metadata": {}, "source": [ "# Initialization" ] @@ -114,7 +112,7 @@ "import lsst.ts.logging_and_reporting.source_adapters as sad\n", "import lsst.ts.logging_and_reporting.almanac as alm\n", "import lsst.ts.logging_and_reporting.reports as rep \n", - "from lsst.ts.logging_and_reporting.reports import md,mdlist" + "from lsst.ts.logging_and_reporting.reports import md,mdlist, NightlyLogReport" ] }, { @@ -160,11 +158,7 @@ "cell_type": "code", "execution_count": null, "id": "9", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "metadata": {}, "outputs": [], "source": [ "# Display overview of Report context \n", @@ -172,7 +166,6 @@ "Report for **{date}** covering the previous **{days}** observing night(s).\n", "- Run on logs from **{server}/**\n", "- Using *Prototype* Logging and Reporting Version: **{lrversion}**\n", - "- {enable_efd=}\n", "''')" ] }, @@ -204,29 +197,24 @@ "id": "12", "metadata": {}, "source": [ - "# Nightly Report" + "# Night Report" ] }, { "cell_type": "code", "execution_count": null, "id": "13", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "metadata": {}, "outputs": [], "source": [ "# Get data from Night Report log. Display nightly Jira BLOCKS.\n", - "nr_adapter = sad.NightReportAdapter(server_url=server)\n", + "nr_adapter = sad.NightReportAdapter(server_url=server,\n", + " limit=limit,\n", + " min_day_obs=min_day_obs,\n", + " max_day_obs=max_day_obs,)\n", "nr_url = nr_adapter.source_url\n", "try:\n", - " nr_recs,nr_url = nr_adapter.get_reports(\n", - " limit=limit,\n", - " min_day_obs=min_day_obs,\n", - " max_day_obs=max_day_obs,\n", - " )\n", + " nr_recs,nr_url = nr_adapter.get_reports()\n", "except Exception as err:\n", " nr_recs = []\n", " msg = f'ERROR getting records from {nr_url=}: {err=}'\n", @@ -248,6 +236,10 @@ "else:\n", " md(f'No jira BLOCK tickets found.', color='lightblue')\n", " md(f'Used: [API Data]({nr_url})')\n", + "\n", + "# Display time log\n", + "nr_rep = NightlyLogReport(min_day_obs=min_day_obs, max_day_obs=max_day_obs)\n", + "nr_rep.time_log_as_markdown(nr_recs, nr_adapter, nr_url)\n", " " ] }, @@ -263,22 +255,19 @@ "cell_type": "code", "execution_count": null, "id": "15", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "metadata": {}, "outputs": [], "source": [ "# Get data from Exposure log. Display time log.\n", - "exposure_adapter = sad.ExposurelogAdapter(server_url=server)\n", + "exposure_adapter = sad.ExposurelogAdapter(\n", + " server_url=server,\n", + " limit=limit,\n", + " min_day_obs=min_day_obs,\n", + " max_day_obs=max_day_obs,\n", + ")\n", "exposure_url = exposure_adapter.source_url\n", "try:\n", - " exposure_recs,url = exposure_adapter.get_messages(\n", - " limit=limit,\n", - " min_day_obs=min_day_obs,\n", - " max_day_obs=max_day_obs,\n", - " )\n", + " exposure_recs,url = exposure_adapter.get_messages()\n", "except Exception as err:\n", " exposure_recs = []\n", " msg = f'ERROR getting records from {url=}: {err=}'\n", @@ -297,19 +286,23 @@ "cell_type": "code", "execution_count": null, "id": "16", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "metadata": {}, + "outputs": [], + "source": [ + "len(exposure_recs)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, "outputs": [], "source": [ "# Display Observation gaps\n", "if usdf == os.environ.get('EXTERNAL_INSTANCE_URL'):\n", " md(f\"**Warning:** The `/exposures/` endpoint is not yet functional on SERVER=usdf.\", color='red')\n", - "gaps = exposure_adapter.get_observation_gaps(min_day_obs=min_day_obs,\n", - " max_day_obs=max_day_obs\n", - " )\n", + "gaps = exposure_adapter.get_observation_gaps()\n", "if gaps:\n", " md(f'### Date vs Observation Gap (minutes) for all Instruments')\n", " for instrument, day_gaps in gaps.items():\n", @@ -325,7 +318,7 @@ }, { "cell_type": "markdown", - "id": "17", + "id": "18", "metadata": {}, "source": [ "# Narrative Log\n" @@ -334,16 +327,17 @@ { "cell_type": "code", "execution_count": null, - "id": "18", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "id": "19", + "metadata": {}, "outputs": [], "source": [ "# Get data from Narrative log. Display time log.\n", - "narrative_adapter = sad.NarrativelogAdapter(server_url=server)\n", + "narrative_adapter = sad.NarrativelogAdapter(\n", + " server_url=server,\n", + " limit=limit,\n", + " min_day_obs=min_day_obs,\n", + " max_day_obs=max_day_obs,\n", + ")\n", "narrative_url = narrative_adapter.source_url\n", "try:\n", " # date like '2000-01-02 12:00:00'\n", @@ -351,11 +345,7 @@ " min_date = str(datetime.strptime(min_day_obs,'%Y%m%d'))\n", " max_date = str(datetime.strptime(max_day_obs,'%Y%m%d'))\n", " #!print(f'Get data from {narrative_url}: {min_date} to {max_date}')\n", - " narrative_recs,url = narrative_adapter.get_messages(\n", - " limit=limit,\n", - " min_date_end=min_date,\n", - " max_date_end=max_date\n", - " )\n", + " narrative_recs,url = narrative_adapter.get_messages()\n", "except Exception as err:\n", " narrative_recs = []\n", " msg = f'ERROR getting records from {url}: {err=}'\n", @@ -378,16 +368,12 @@ { "cell_type": "code", "execution_count": null, - "id": "19", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "id": "20", + "metadata": {}, "outputs": [], "source": [ "# Conditionally display our current ability to connect to all needed endpoints.\n", - "if not os.environ.get('EXTERNAL_INSTANCE_URL'):\n", + "if False and not os.environ.get('EXTERNAL_INSTANCE_URL'):\n", " md('# Dashboard')\n", " md('(This is not done when running under Times Square.)')\n", " %run ./dashboard.ipynb" @@ -396,7 +382,17 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "print(f'Finished {str(datetime.now())}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22", "metadata": {}, "outputs": [], "source": [] diff --git a/notebooks_tsqr/dashboard.ipynb b/notebooks_tsqr/dashboard.ipynb index 8058cbf..e788866 100644 --- a/notebooks_tsqr/dashboard.ipynb +++ b/notebooks_tsqr/dashboard.ipynb @@ -1,9 +1,21 @@ { "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Logger Source API Dashboard\n", + "For all of these to work, the following must be enabled:\n", + "- Tucson VPN\n", + "- Summit VPN\n", + "- User has access to USDF-dev (SLAC)" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "0", + "id": "1", "metadata": {}, "outputs": [], "source": [ @@ -13,7 +25,7 @@ { "cell_type": "code", "execution_count": null, - "id": "1", + "id": "2", "metadata": {}, "outputs": [], "source": [ @@ -26,7 +38,7 @@ { "cell_type": "code", "execution_count": null, - "id": "2", + "id": "3", "metadata": {}, "outputs": [], "source": [ @@ -38,7 +50,7 @@ { "cell_type": "code", "execution_count": null, - "id": "3", + "id": "4", "metadata": {}, "outputs": [], "source": [] diff --git a/python/lsst/ts/logging_and_reporting/reports.py b/python/lsst/ts/logging_and_reporting/reports.py index 7c148a0..f18c8e1 100644 --- a/python/lsst/ts/logging_and_reporting/reports.py +++ b/python/lsst/ts/logging_and_reporting/reports.py @@ -62,7 +62,21 @@ def __init__(self, *, min_day_obs=None, # INCLUSIVE: default=Yesterday max_day_obs=None, # EXCLUSIVE: default=Today ): - pass # TODO + self.min_day_obs = min_day_obs + self.max_day_obs = max_day_obs + + def time_log_as_markdown(self, records, source_adapter, url, + log_title=None, + ): + service = source_adapter.service + title = log_title if log_title else '' + if records: + md(f'### {title}') + table = source_adapter.day_table(records, 'date_added') + mdlist(table) + else: + md(f'No {service} records found.', color='lightblue') + md(f'Used [API Data]({url})') class AlmanacReport(Report): # moon rise,set,illumination % diff --git a/python/lsst/ts/logging_and_reporting/source_adapters.py b/python/lsst/ts/logging_and_reporting/source_adapters.py index 206b2e5..7134c78 100644 --- a/python/lsst/ts/logging_and_reporting/source_adapters.py +++ b/python/lsst/ts/logging_and_reporting/source_adapters.py @@ -26,7 +26,6 @@ concept is Proven, it all might be thrown away or rewritten. ''' - # Python Standard Library from urllib.parse import urlencode import itertools @@ -49,10 +48,18 @@ class SourceAdapter(ABC): # TODO document class including all class variables. def __init__(self, *, server_url='https://tucson-teststand.lsst.codes', + min_day_obs=None, # INCLUSIVE: default=Yesterday + max_day_obs=None, # EXCLUSIVE: default=Today + limit=99, + offset=0, connect_timeout=1.05, # seconds read_timeout=2, # seconds ): self.server = server_url + self.min_day_obs = min_day_obs + self.max_day_obs = max_day_obs + self.limit = limit + self.offset = offset self.c_timeout = min(MAX_CONNECT_TIMEOUT, float(connect_timeout)) # seconds self.r_timeout = min(MAX_READ_TIMEOUT, # seconds @@ -200,27 +207,26 @@ class NightReportAdapter(SourceAdapter): 'user_id', } + def row_str_func(self, datetime_str, rec): + return f"> {datetime_str} | 
{rec['summary']}
" def get_reports(self, site_ids=None, summary=None, - min_day_obs=None, - max_day_obs=None, is_human='either', is_valid='either', - limit=None, ): qparams = dict(is_human=is_human, is_valid=is_valid) if site_ids: qparams['site_ids'] = site_ids if summary: qparams['summary'] = summary - if min_day_obs: - qparams['min_day_obs'] = min_day_obs - if max_day_obs: - qparams['max_day_obs'] = max_day_obs - if limit: - qparams['limit'] = limit + if self.min_day_obs: + qparams['min_day_obs'] = self.min_day_obs + if self.max_day_obs: + qparams['max_day_obs'] = self.max_day_obs + if self.limit: + qparams['limit'] = self.limit qstr = urlencode(qparams) url = f'{self.server}/{self.service}/reports?{qstr}' @@ -282,7 +288,6 @@ def get_messages(self, is_human='either', is_valid='either', offset=None, - limit=None, ): qparams = dict( is_human=is_human, @@ -297,8 +302,8 @@ def get_messages(self, qparams['min_date_end'] = min_date_end if max_date_end: qparams['max_date_end'] = max_date_end - if limit: - qparams['limit'] = limit + if self.limit: + qparams['limit'] = self.limit qstr = urlencode(qparams) url = f'{self.server}/{self.service}/messages?{qstr}' @@ -336,7 +341,7 @@ class ExposurelogAdapter(SourceAdapter): 'day_obs', # 'exposure_flag', # 'id', - # 'instrument', + 'instrument', # 'is_human', # 'is_valid', # 'level', @@ -353,10 +358,16 @@ class ExposurelogAdapter(SourceAdapter): @property def row_header(self): - return '| Time | OBS ID | Message |\n|--------|-------|------|' + return('| Time | OBS ID | Telescope | Message |\n' + '|------|--------|-----------|---------|' + ) def row_str_func(self, datetime_str, rec): - return f"> {datetime_str} | {rec['obs_id']} | 
{rec['message_text']}
" + return(f"> {datetime_str} " + f"| {rec['obs_id']} " + f"| {rec['instrument']} " + f"| 
{rec['message_text']}
" + ) def check_endpoints(self, timeout=None, verbose=True): to = (timeout or self.timeout) @@ -385,16 +396,12 @@ def get_instruments(self): # Flatten the lists return list(itertools.chain.from_iterable(instruments.values())) - def get_exposures(self, instrument, - registry=1, - min_day_obs=None, - max_day_obs=None, - ): + def get_exposures(self, instrument, registry=1): qparams = dict(instrument=instrument, registery=registry) - if min_day_obs: - qparams['min_day_obs'] = min_day_obs - if max_day_obs: - qparams['max_day_obs'] = max_day_obs + if self.min_day_obs: + qparams['min_day_obs'] = self.min_day_obs + if self.max_day_obs: + qparams['max_day_obs'] = self.max_day_obs url = f'{self.server}/{self.service}/exposures?{urlencode(qparams)}' try: recs = requests.get(url, timeout=self.timeout).json() @@ -407,13 +414,9 @@ def get_messages(self, obs_ids=None, instruments=None, message_text=None, - min_day_obs=None, - max_day_obs=None, is_human='either', is_valid='either', exposure_flags=None, - offset=None, - limit=None, ): qparams = dict(is_human=is_human, is_valid=is_valid, @@ -425,16 +428,14 @@ def get_messages(self, qparams['obs_ids'] = obs_ids if instruments: qparams['instruments'] = instruments - if min_day_obs: - qparams['min_day_obs'] = min_day_obs - if max_day_obs: - qparams['max_day_obs'] = max_day_obs + if self.min_day_obs: + qparams['min_day_obs'] = self.min_day_obs + if self.max_day_obs: + qparams['max_day_obs'] = self.max_day_obs if exposure_flags: qparams['exposure_flags'] = exposure_flags - if offset: - qparams['offset'] = offset - if limit: - qparams['limit'] = limit + if self.limit: + qparams['limit'] = self.limit qstr = urlencode(qparams) url = f'{self.server}/{self.service}/messages?{qstr}' @@ -451,11 +452,7 @@ def get_messages(self, self.keep_fields(recs, self.outfields) return recs,url - def get_observation_gaps(self, - instruments=None, - min_day_obs=None, # YYYYMMDD - max_day_obs=None, # YYYYMMDD - ): + def get_observation_gaps(self, instruments=None): if not instruments: instruments = self.get_instruments() assert isinstance(instruments,list), \ @@ -463,10 +460,7 @@ def get_observation_gaps(self, # inst_day_rollup[instrument] => dict[day] => exposureGapInMinutes inst_day_rollup = defaultdict(dict) # Instrument/Day rollup for instrum in instruments: - recs = self.get_exposures(instrum, - min_day_obs=min_day_obs, - max_day_obs=max_day_obs, - ) + recs = self.get_exposures(instrum) instrum_gaps = dict() for day,dayrecs in itertools.groupby(recs, key=lambda r: r['day_obs']):