From 725fb0da8456883ed03f85e9838d3d3788b40505 Mon Sep 17 00:00:00 2001 From: Steve Pothier Date: Tue, 29 Oct 2024 07:18:57 -0700 Subject: [PATCH] tally_exposure_flags, drill-down links --- notebooks_tsqr/ExposureDetail.ipynb | 145 ++++++++++++++++ notebooks_tsqr/ExposureDetail.yaml | 45 +++++ notebooks_tsqr/NightLog.ipynb | 160 ++++++------------ .../ts/logging_and_reporting/all_sources.py | 28 ++- python/lsst/ts/logging_and_reporting/efd.py | 15 +- .../logging_and_reporting/source_adapters.py | 58 +++++++ python/lsst/ts/logging_and_reporting/utils.py | 15 ++ 7 files changed, 342 insertions(+), 124 deletions(-) create mode 100644 notebooks_tsqr/ExposureDetail.ipynb create mode 100644 notebooks_tsqr/ExposureDetail.yaml diff --git a/notebooks_tsqr/ExposureDetail.ipynb b/notebooks_tsqr/ExposureDetail.ipynb new file mode 100644 index 0000000..7d026fa --- /dev/null +++ b/notebooks_tsqr/ExposureDetail.ipynb @@ -0,0 +1,145 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "# Parameters.\n", + "# Times Square replaces this cell with the user's parameters.\n", + "# So, don't put anything else here!\n", + "\n", + "# day_obs values: TODAY, YESTERDAY, YYYY-MM-DD\n", + "# Report on observing nights that start upto but not included this day.\n", + "#!day_obs = '2024-09-25' # Value to use for local testing (Summit)\n", + "day_obs = \"2024-09-25\" # TODO Change to 'YESTERDAY' to test with default before push\n", + "\n", + "# Total number of days of data to display (ending on day_obs)\n", + "number_of_days = \"1\" # TODO Change to '1' to test with default before push\n", + "\n", + "instrument = \"LSSTComCam\"\n", + "obs_reason = \"ALL\"\n", + "obs_type = \"ALL\"\n", + "science_program = \"ALL\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "\n", + "# When running under Times Square, install pkg from github.\n", + "# Otherwise use what is installed locally (intended to be dev editiable pkg)\n", + "if os.environ.get(\"EXTERNAL_INSTANCE_URL\"):\n", + " print(\n", + " 'Installing \"lsst.ts.logging_and_reporting\" from github using \"prototype\" branch....'\n", + " )\n", + " print(\n", + " \"TODO: Make the need for this go away by getting Logging_and_Reporting installed in RSP.\"\n", + " )\n", + " !pip install --upgrade git+https://github.com/lsst-ts/ts_logging_and_reporting.git@prototype > /dev/null 2>&1\n", + "import lsst.ts.logging_and_reporting.source_adapters as sad\n", + "import lsst.ts.logging_and_reporting.utils as ut\n", + "\n", + "# Set default env to \"usdf\" and try before PUSH to repo.\n", + "\n", + "# The default provided here is for local testing.\n", + "# Under Times Square it is ignored.\n", + "server = os.environ.get(\n", + " \"EXTERNAL_INSTANCE_URL\", ut.Server.summit\n", + ") # TODO try with \"usdf\" before push (else \"summit\")\n", + "\n", + "# Normalize Parameters (both explicit Times Squares params and implicit ones)\n", + "obs_reason = None if obs_reason == \"ALL\" else obs_reason\n", + "obs_type = None if obs_type == \"ALL\" else obs_type\n", + "science_program = None if science_program == \"ALL\" else science_program" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "src_exp = sad.ExposurelogAdapter(\n", + " server_url=server,\n", + " min_dayobs=None, # INCLUSIVE: default=Yesterday\n", + " max_dayobs=None, # EXCLUSIVE: default=Today other=YYYY-MM-DD\n", + " limit=5000,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "instrument, science_program, obs_reason, obs_type" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "src_exp.exposure_detail(\n", + " instrument=instrument,\n", + " science_program=science_program,\n", + " observation_reason=obs_reason,\n", + " observation_type=obs_type,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "# src_exp.exposures[instrument]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks_tsqr/ExposureDetail.yaml b/notebooks_tsqr/ExposureDetail.yaml new file mode 100644 index 0000000..b03be86 --- /dev/null +++ b/notebooks_tsqr/ExposureDetail.yaml @@ -0,0 +1,45 @@ +# For use with a Times Square notebook +title: Exposure Detail +description: Details of Nightly Exposures +authors: + - name: Steve Pothier + slack: Steve Pothier +tags: + - reporting + - prototype + - exposure +parameters: + day_obs: + type: string + description: > + The last observing night to report on. (Allowed: YYYY-MM-DD, + TODAY, YESTERDAY) You probably want YESTERDAY. A value of TODAY + means the last night shown will be the one that starts tonight.` + default: "YESTERDAY" + number_of_days: + type: integer + description: > + Number of days (nights) to show in the report. + default: 1 + minimum: 1 + maximum: 9 + instrument: + type: string + description: > + Instrument used for exposures. + default: "LSSTComCam" + obs_reason: + type: string + description: > + Observation reason. + default: "ALL" + obs_type: + type: string + description: > + Observation reason. + default: "ALL" + science_program: + type: string + description: > + Science Program + default: "ALL" diff --git a/notebooks_tsqr/NightLog.ipynb b/notebooks_tsqr/NightLog.ipynb index 039d786..e740214 100644 --- a/notebooks_tsqr/NightLog.ipynb +++ b/notebooks_tsqr/NightLog.ipynb @@ -24,7 +24,7 @@ "# day_obs values: TODAY, YESTERDAY, YYYY-MM-DD\n", "# Report on observing nights that start upto but not included this day.\n", "#!day_obs = '2024-09-25' # Value to use for local testing (Summit)\n", - "day_obs = \"YESTERDAY\" # TODO Change to 'YESTERDAY' to test with default before push\n", + "day_obs = \"2024-09-25\" # TODO Change to 'YESTERDAY' to test with default before push\n", "\n", "# Total number of days of data to display (ending on day_obs)\n", "number_of_days = \"1\" # TODO Change to '1' to test with default before push" @@ -43,6 +43,7 @@ "from pprint import pformat, pp\n", "from urllib.parse import urlencode, urlparse\n", "import pandas as pd\n", + "from IPython.display import HTML\n", "\n", "# When running under Times Square, install pkg from github.\n", "# Otherwise use what is installed locally (intended to be dev editiable pkg)\n", @@ -54,8 +55,6 @@ " \"TODO: Make the need for this go away by getting Logging_and_Reporting installed in RSP.\"\n", " )\n", " !pip install --upgrade git+https://github.com/lsst-ts/ts_logging_and_reporting.git@prototype > /dev/null 2>&1\n", - "# import lsst.ts.logging_and_reporting.almanac as alm\n", - "# import lsst.ts.logging_and_reporting.reports as rep\n", "from lsst.ts.logging_and_reporting.all_sources import AllSources\n", "from lsst.ts.logging_and_reporting.all_sources import uniform_field_counts\n", "from lsst.ts.logging_and_reporting.all_reports import AllReports\n", @@ -103,14 +102,11 @@ "outputs": [], "source": [ "# Set default env to \"usdf\" and try before PUSH to repo.\n", - "summit = \"https://summit-lsp.lsst.codes\"\n", - "usdf = \"https://usdf-rsp-dev.slac.stanford.edu\"\n", - "tucson = \"https://tucson-teststand.lsst.codes\"\n", "\n", "# The default provided here is for local testing.\n", "# Under Times Square it is ignored.\n", "server = os.environ.get(\n", - " \"EXTERNAL_INSTANCE_URL\", summit\n", + " \"EXTERNAL_INSTANCE_URL\", ut.Server.summit\n", ") # TODO try with \"usdf\" before push (else \"summit\")" ] }, @@ -179,7 +175,7 @@ " - Simonyi\n", "* [Almanac](#almanac)\n", "* [Summary plots of whole night](#Summary-Plots)\n", - "* [Summary Scalars for the night](#Summary-Scalars)\n", + "* [Time Accounting](#Time Accounting)\n", "* [Jira Tickets](#Jira-Tickets)\n", " - AuxTel\n", " - Simonyi\n", @@ -280,8 +276,8 @@ "id": "16", "metadata": {}, "source": [ - "\n", - "## Summary Scalars for the night \n", + "\n", + "## Time Accounting\n", "**NEW**: Added \"Total (fault, etc) loss\". Source=Narrativelog" ] }, @@ -363,7 +359,7 @@ "id": "22", "metadata": {}, "source": [ - "## Exposure Log" + "## Data Log" ] }, { @@ -374,22 +370,25 @@ "outputs": [], "source": [ "# Exposure Report\n", - "if allsrc.exp_src.exposures[\"LATISS\"]:\n", + "if experimental.get(\"exposure_type_tally\"):\n", + " md(\"### Experimental Exposure field Tally ENABLED\")\n", + " md(\"Tally exposures with specific values of selected fields\")\n", + "else:\n", + " md(\"### Experimental Exposure field Tally DISASBLED\")\n", + "\n", + "for instrum in allsrc.exp_src.exposures.keys():\n", + " md(f\"**{instrum}: {len(allsrc.exp_src.exposures[instrum])} total exposures**\")\n", + " if 0 == len(allsrc.exp_src.exposures[instrum]):\n", + " continue\n", + "\n", " # Facet counts\n", " if experimental.get(\"exposure_type_tally\"):\n", - " md(\"### Experimental Exposure field Tally ENABLED\")\n", - " counts, totals = uniform_field_counts(allsrc.exp_src.exposures[\"LATISS\"])\n", - " tkeys = sum([list(tally.keys()) for tally in counts.values()], [])\n", - " column_map = {tkey: str(tkey).replace(\"_\", \" \") for tkey in tkeys}\n", + " counts = uniform_field_counts(allsrc.exp_src.exposures[instrum])\n", " for fname, tally in counts.items():\n", - " md(\n", - " f\"#### Tally of values for Field: {fname} (Total exposures: {totals[fname]})\"\n", - " )\n", + " md(f\"{fname}\")\n", " df = pd.DataFrame.from_dict(tally, orient=\"index\").T\n", - " df = df.rename(columns=column_map) # allow wrap of column header\n", - " display(df)\n", - " else:\n", - " md(\"### Experimental Exposure field Tally DISASBLED\")" + " df = ut.wrap_dataframe_columns(df)\n", + " display(HTML(df.to_html(render_links=True, escape=False)))" ] }, { @@ -416,20 +415,29 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "id": "26", "metadata": {}, + "outputs": [], "source": [ - "## Narrative Log\n", - "(Now \"linked things\" from /narrativelog/messages.urls)\n", - "\n", - "Date Time | Time Lost | Time Lost Type " + "for instrum, recs in allsrc.exp_src.exposures.items():\n", + " md(f\"**{instrum} {len(recs)} Exposures with messages**\")\n", + " allsrc.tally_exposure_flags(instrum)" + ] + }, + { + "cell_type": "markdown", + "id": "27", + "metadata": {}, + "source": [ + "## Narrative Log" ] }, { "cell_type": "code", "execution_count": null, - "id": "27", + "id": "28", "metadata": {}, "outputs": [], "source": [ @@ -440,7 +448,7 @@ }, { "cell_type": "markdown", - "id": "28", + "id": "29", "metadata": {}, "source": [ "\n", @@ -452,68 +460,10 @@ "May also contain sections that have moved out of he user section because they are not defined in the Storyboard." ] }, - { - "cell_type": "markdown", - "id": "29", - "metadata": {}, - "source": [ - "## Experimental " - ] - }, { "cell_type": "markdown", "id": "30", "metadata": {}, - "source": [ - "### Bullets\n", - "- Level1\n", - " - Level 2\n", - " - again lev 2\n", - "### Indent, no bullets\n", - "Level 1\n", - " Level 2" - ] - }, - { - "cell_type": "markdown", - "id": "31", - "metadata": {}, - "source": [ - "### Widgets" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "32", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, - "outputs": [], - "source": [ - "from ipywidgets import interact, interactive, fixed, interact_manual\n", - "import ipywidgets as widgets\n", - "import pandas as pd\n", - "\n", - "records = [dict(a=i, b=i, c=i, d=i, e=i) for i in range(100)]\n", - "\n", - "\n", - "def records_toggle(show_records):\n", - " if show_records:\n", - " return pd.DataFrame(records)\n", - " else:\n", - " return print(f\"Number of records: {len(records)}\")\n", - "\n", - "\n", - "interact(records_toggle, show_records=False);" - ] - }, - { - "cell_type": "markdown", - "id": "33", - "metadata": {}, "source": [ "## Overview \n" ] @@ -521,7 +471,7 @@ { "cell_type": "code", "execution_count": null, - "id": "34", + "id": "31", "metadata": { "jupyter": { "source_hidden": true @@ -547,7 +497,7 @@ { "cell_type": "code", "execution_count": null, - "id": "35", + "id": "32", "metadata": { "jupyter": { "source_hidden": true @@ -569,7 +519,7 @@ }, { "cell_type": "markdown", - "id": "36", + "id": "33", "metadata": {}, "source": [ "## This report uses the following data sources\n", @@ -584,7 +534,7 @@ }, { "cell_type": "markdown", - "id": "37", + "id": "34", "metadata": {}, "source": [ "## DDV " @@ -593,7 +543,7 @@ { "cell_type": "code", "execution_count": null, - "id": "38", + "id": "35", "metadata": { "jupyter": { "source_hidden": true @@ -611,7 +561,7 @@ }, { "cell_type": "markdown", - "id": "39", + "id": "36", "metadata": {}, "source": [ "## Where was this run?\n", @@ -626,7 +576,7 @@ }, { "cell_type": "markdown", - "id": "40", + "id": "37", "metadata": {}, "source": [ "## Section overviews moved here" @@ -635,12 +585,8 @@ { "cell_type": "code", "execution_count": null, - "id": "41", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "id": "38", + "metadata": {}, "outputs": [], "source": [ "# Night Report Overview\n", @@ -654,12 +600,8 @@ { "cell_type": "code", "execution_count": null, - "id": "42", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "id": "39", + "metadata": {}, "outputs": [], "source": [ "# Conditionally display our current ability to connect to all needed endpoints.\n", @@ -671,7 +613,7 @@ }, { "cell_type": "markdown", - "id": "43", + "id": "40", "metadata": {}, "source": [ "## Finale" @@ -680,7 +622,7 @@ { "cell_type": "code", "execution_count": null, - "id": "44", + "id": "41", "metadata": { "jupyter": { "source_hidden": true diff --git a/python/lsst/ts/logging_and_reporting/all_sources.py b/python/lsst/ts/logging_and_reporting/all_sources.py index 08b04a4..eef9fc8 100644 --- a/python/lsst/ts/logging_and_reporting/all_sources.py +++ b/python/lsst/ts/logging_and_reporting/all_sources.py @@ -273,6 +273,10 @@ def get_slews(self): pass + def tally_exposure_flags(self, instrument): + fc = facet_counts(self.exp_src.records, fieldnames=["exposure_flag"]) + return fc + @property def urls(self): return self.nar_src.urls | self.exp_src.urls @@ -320,13 +324,16 @@ def facet_counts(records, fieldnames=None, ignore_fields=None): # (This is not the count of football jerseys in play. ) -def uniform_field_counts(records): +def uniform_field_counts( + records, + server="https://usdf-rsp-dev.slac.stanford.edu", +): """Count number of records of each value in a Uniform Field. A Uniform Field is one that only has a small number of values. RETURN: dict[fieldname] -> dict[value] -> count """ if len(records) == 0: - return None + return None, None facets, ignored = get_facets(records) # Explicitly remove tables that we expect to be useless. facets.pop("day_obs", None) @@ -334,6 +341,19 @@ def uniform_field_counts(records): facets.pop("target_name", None) facets.pop("group_name", None) facets.pop("seq_num", None) + facets.pop("exposure_time", None) counts = {k: dict(Counter([r[k] for r in records])) for k in facets.keys()} - totals = {k: sum(v.values()) for k, v in counts.items()} - return counts, totals + root = ( + f"{server}/times-square/github/lsst-ts/ts_logging_and_reporting" + + "/ExposureDetail" + ) + link_to_detail = { + fname: { + fvalue: f'{num}' + for fvalue, num in tally.items() + } + for fname, tally in counts.items() + } + return link_to_detail + # #!totals = {k: sum(v.values()) for k,v in counts.items()} + # #! return counts,totals diff --git a/python/lsst/ts/logging_and_reporting/efd.py b/python/lsst/ts/logging_and_reporting/efd.py index 0c8892b..8a29078 100644 --- a/python/lsst/ts/logging_and_reporting/efd.py +++ b/python/lsst/ts/logging_and_reporting/efd.py @@ -21,13 +21,6 @@ from lsst_efd_client import EfdClient -# Servers we might use -class Server: - summit = "https://summit-lsp.lsst.codes" - usdf = "https://usdf-rsp-dev.slac.stanford.edu" - tucson = "https://tucson-teststand.lsst.codes" - - class EfdAdapter(SourceAdapter): salindex = 2 service = "efd" @@ -50,21 +43,21 @@ def __init__( instance_url = os.getenv("EXTERNAL_INSTANCE_URL", self.server) self.server_url = server_url or instance_url match self.server_url: - case Server.summit: + case ut.Server.summit: self.client = EfdClient("summit_efd") - case Server.usdf: + case ut.Server.usdf: self.client = EfdClient("usdf_efd") os.environ["RUBIN_SIM_DATA_DIR"] = ( "/sdf/data/rubin/shared/rubin_sim_data" ) - case Server.tucson: + case ut.Server.tucson: pass case _: msg = ( f"Unknown server from EXTERNAL_INSTANCE (env var). " f"Got {self.server_url=} " f"Expected one of: " - f"{Server.summit}, {Server.usdf}, {Server.tucson}" + f"{ut.Server.summit}, {ut.Server.usdf}, {ut.Server.tucson}" ) raise Exception(msg) diff --git a/python/lsst/ts/logging_and_reporting/source_adapters.py b/python/lsst/ts/logging_and_reporting/source_adapters.py index a3db2fd..e6bd55a 100644 --- a/python/lsst/ts/logging_and_reporting/source_adapters.py +++ b/python/lsst/ts/logging_and_reporting/source_adapters.py @@ -45,6 +45,7 @@ import lsst.ts.logging_and_reporting.parse_message as pam import lsst.ts.logging_and_reporting.reports as rep import lsst.ts.logging_and_reporting.utils as ut +import pandas as pd import requests MAX_CONNECT_TIMEOUT = 3.1 # seconds @@ -705,6 +706,50 @@ def obs_id(rec): table.append(str) return table + # /exposurelog/exposures?instrument=LSSTComCamSim + def exposure_detail( + self, + instrument, + science_program=None, + observation_type=None, + observation_reason=None, + ): + fields = [ + "obs_id", + "timespan_begin", # 'time', + "seq_num", + "observation_type", + "observation_reason", + "science_program", + "exposure_time", + # 'physical_filter', + # 'nimage', + # 'hasPD', + # 'metadata', + ] + program = science_program + recs = [ + r + for r in self.exposures[instrument] + if ( + ((program is None) or (r["science_program"] == program)) + and ( + (observation_type is None) + or (r["observation_type"] == observation_type) + ) + and ( + (observation_reason is None) + or (r["observation_reason"] == observation_reason) + ) + ) + ] + if len(recs) == 0: + return "No matching records" + + # #!df = pd.DataFrame(self.exposures[instrument])[fields] + df = pd.DataFrame(recs)[fields] + return ut.wrap_dataframe_columns(df) + def check_endpoints(self, timeout=None, verbose=True): to = timeout or self.timeout if verbose: @@ -776,6 +821,16 @@ def get_exposures(self, instrument, verbose=False): # #! progstr = sprogram if sprogram else ' ' self.exposures[instrument] = recs + for r in recs: + exp_secs = ( + dt.datetime.fromisoformat(r["timespan_end"]) + - dt.datetime.fromisoformat(r["timespan_end"]) + ).total_seconds() + r["exposure_time"] = exp_secs + + # exposure_lut[instrument] => dict[obsid] => rec + self.exposure_lut = {instrument: {r["obs_id"]: r} for r in recs} + status = dict( endpoint_url=url, number_of_records=len(recs), @@ -828,6 +883,9 @@ def get_records( self.keep_fields(recs, self.outfields) self.records = recs + # messages[instrument] => dict[obsid] => rec + self.messages = {r["instrument"]: {r["obs_id"]: r} for r in recs} + status = dict( endpoint_url=url, number_of_records=len(recs), diff --git a/python/lsst/ts/logging_and_reporting/utils.py b/python/lsst/ts/logging_and_reporting/utils.py index 1c205de..4f6de3f 100644 --- a/python/lsst/ts/logging_and_reporting/utils.py +++ b/python/lsst/ts/logging_and_reporting/utils.py @@ -171,3 +171,18 @@ def tic(self): def toc(self): elapsed_seconds = time.perf_counter() - self.start return elapsed_seconds # fractional + + +# Servers we might use +class Server: + summit = "https://summit-lsp.lsst.codes" + usdf = "https://usdf-rsp-dev.slac.stanford.edu" + tucson = "https://tucson-teststand.lsst.codes" + + +def wrap_dataframe_columns(df): + def spacify(name): + str(name).replace("_", " ") + + column_map = {colname: spacify(colname) for colname in df.columns} + return df.rename(columns=column_map)