time_logs aligned with storyboard

notebooks_tsqr/NightLog.ipynb

@@ -25,7 +25,7 @@
     "# day_obs values: TODAY, YESTERDAY, YYYY-MM-DD\n",
     "# Report on observing nights that start upto but not included this day.\n",
     "#!day_obs = '2024-09-25' # Value to use for local testing (Summit)\n",
-    "day_obs = \"2024-09-25\"  # TODO Change to 'YESTERDAY' to test with default before push\n",
+    "day_obs = \"2024-09-24\"  # TODO Change to 'YESTERDAY' to test with default before push\n",
     "\n",
     "# Total number of days of data to display (ending on day_obs)\n",
     "number_of_days = \"1\"  # TODO Change to '1' to test with default before push"
@@ -35,7 +35,11 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "2",
-   "metadata": {},
+   "metadata": {
+    "jupyter": {
+     "source_hidden": true
+    }
+   },
    "outputs": [],
    "source": [
     "import datetime as dt\n",
@@ -113,11 +117,7 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "4",
-   "metadata": {
-    "jupyter": {
-     "source_hidden": true
-    }
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "# Set default env to \"usdf\" and try before PUSH to repo.\n",
@@ -129,14 +129,23 @@
     "# Under Times Square it is ignored.\n",
     "server = os.environ.get(\n",
     "    \"EXTERNAL_INSTANCE_URL\", summit\n",
-    ")  # TODO try with \"usdf\" before push (else \"summit\")\n",
-    "\n",
+    ")  # TODO try with \"usdf\" before push (else \"summit\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Read records from (almost) all sources\n",
     "allsrc = AllSources(server_url=server)"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "5",
+   "id": "6",
    "metadata": {},
    "source": [
     "# Table of Contents\n",
@@ -150,7 +159,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "6",
+   "id": "7",
    "metadata": {},
    "source": [
     "# Overview \n",
@@ -160,7 +169,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "7",
+   "id": "8",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -172,20 +181,27 @@
     "- This report will include available data from noon **{min_date}** to noon **{max_date}**.\n",
     "- Using ***Prototype* Logging and Reporting** Version: **{lrversion}**\n",
     "\"\"\"\n",
-    ")\n",
-    "\n",
-    "sources_md_str = \"\"\n",
-    "#! sources_md_str += '\\n- '.join(['',*sad.all_endpoints(server)])  # TODO move\n",
-    "sources_md_str += \"\\n- ConsDB\"\n",
-    "sources_md_str += \"\\n- EFD\"\n",
-    "sources_md_str += \"\\n- (DDV)\"\n",
-    "sources_md_str += \"\\n- (Astroplan for Almanac)\"\n",
-    "md(f\"### This report uses the following data sources: {sources_md_str}\")"
+    ")"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "8",
+   "id": "9",
+   "metadata": {},
+   "source": [
+    "### This report uses the following data sources\n",
+    "- NightReport\n",
+    "- Exposurelog\n",
+    "- Narrativelog\n",
+    "- EFD\n",
+    "- ConsDB\n",
+    "- (DDV)\n",
+    "- (Almanac from Astroplan)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "10",
    "metadata": {},
    "source": [
     "<a id=\"ddv\"></a>\n",
@@ -195,7 +211,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "9",
+   "id": "11",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -209,7 +225,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "10",
+   "id": "12",
    "metadata": {},
    "source": [
     "<a id=\"almanac\"></a>\n",
@@ -219,7 +235,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "11",
+   "id": "13",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -230,7 +246,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "12",
+   "id": "14",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -244,7 +260,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "13",
+   "id": "15",
    "metadata": {},
    "source": [
     "# Night Report "
@@ -253,7 +269,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "14",
+   "id": "16",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -278,12 +294,13 @@
     "    md(f\"Used: [API Data]({allsrc.nig_src.source_url})\")\n",
     "\n",
     "# Display time log\n",
-    "nr_rep.time_log_as_markdown()"
+    "nr_rep.time_log_as_markdown()\n",
+    "md(\"-------------\")"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "15",
+   "id": "17",
    "metadata": {},
    "source": [
     "# Exposure Log"
@@ -292,7 +309,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "16",
+   "id": "18",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -324,12 +341,13 @@
     "    md(f\"Used: [API Data]({allsrc.exp_src.source_url})\")\n",
     "\n",
     "# Time Log\n",
-    "exposure_rep.time_log_as_markdown()"
+    "exposure_rep.time_log_as_markdown()\n",
+    "md(\"-------------\")"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "17",
+   "id": "19",
    "metadata": {},
    "source": [
     "# Narrative Log\n"
@@ -338,7 +356,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "18",
+   "id": "20",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -349,20 +367,21 @@
     "narrative_rep.overview\n",
     "\n",
     "# Time Log\n",
-    "narrative_rep.time_log_as_markdown()"
+    "narrative_rep.time_log_as_markdown()\n",
+    "md(\"-------------\")"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "19",
+   "id": "21",
    "metadata": {},
    "source": [
     "# Developer Only Section"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "20",
+   "id": "22",
    "metadata": {},
    "source": [
     "## Where was this run?\n",
@@ -378,7 +397,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "21",
+   "id": "23",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -391,7 +410,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "22",
+   "id": "24",
    "metadata": {},
    "source": [
     "# Finale"
@@ -400,7 +419,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "23",
+   "id": "25",
    "metadata": {},
    "outputs": [],
    "source": [

python/lsst/ts/logging_and_reporting/all_sources.py

@@ -119,3 +119,19 @@ async def night_tally_observation_gaps(self, verbose=True):
         # https://ts-xml.lsst.io/sal_interfaces/Scheduler.html#slewtime
         # edf.get_targets() => "slewTime"                             # (d,g,h)
         return instrument_tally
+
+    def records_per_source(self):
+        sources = [
+            self.nig_src,
+            self.exp_src,
+            self.nar_src,
+            self.efd_src,
+        ]
+
+        # Until efd.get_targets is run, it will report 0 records.
+        # That method is run in: await allsrc.night_tally_observation_gaps()
+        res = {
+            src.service: src.status[src.primary_endpoint]["number_of_records"]
+            for src in sources
+        }
+        return res

python/lsst/ts/logging_and_reporting/efd.py

@@ -29,6 +29,11 @@ class Server:
 
 class EfdAdapter(SourceAdapter):
     salindex = 2
+    service = "efd"
+    endpoints = [
+        "targets",
+    ]
+    primary_endpoint = "targets"
 
     def __init__(
         self,
@@ -62,6 +67,13 @@ def __init__(
                 )
                 raise Exception(msg)
 
+        self.targets = None
+        self.status["targets"] = dict(
+            endpoint_url="NA",
+            number_of_records=0,
+            error=None,
+        )
+
     async def get_topics(self):
         self.topics = await self.client.get_topics()
         return self.topics
@@ -129,6 +141,9 @@ async def query_nights(self, topic, fields, index=301):
 
     # slewTime (and probably others) are EXPECTED times, not ACTUAL.
     async def get_targets(self):
+        if self.targets is not None:  # is cached
+            return self.targets
+
         topic = "lsst.sal.Scheduler.logevent_target"
         fields_wanted = [
             "blockId",
@@ -143,6 +158,12 @@ async def get_targets(self):
             fields_wanted,
             index=self.salindex,
         )
+        self.targets = targets
+        self.status["targets"] = dict(
+            endpoint_url="NA",
+            number_of_records=len(targets),
+            error=None,
+        )
         return targets
 
     async def get_weather(self, days=1):

python/lsst/ts/logging_and_reporting/reports.py

@@ -95,13 +95,8 @@ def time_log_as_markdown(
         records = adapter.records
         service = adapter.service
         url = adapter.get_status().get("endpoint_url")
-        title = log_title if log_title else ""
         if records:
-            md(
-                f"## Time Log for {self.source_adapter.min_date} "
-                f"to {self.source_adapter.max_date}"
-            )
-            md(f"### {title}")
+            md("## Time Log")
             table = self.source_adapter.day_table("date_added")
             mdlist(table)
         else: