diff --git a/notebooks_tsqr/NightLog.ipynb b/notebooks_tsqr/NightLog.ipynb
index 9b3872c..ab7a42d 100644
--- a/notebooks_tsqr/NightLog.ipynb
+++ b/notebooks_tsqr/NightLog.ipynb
@@ -9,6 +9,7 @@
"***What are we missing?*** See the slack *#ts-logging* channel for discussion about this page. Use it to report problems, ask questions, and make requests for changes. \n",
"\n",
"## What is new in this application?(newest change at top of list)\n",
+ "- Added initial Consolidated Database section\n",
"- Add section for \"Links to related resources\". Let us know is other links should be added.\n",
"- Exposure quality flag added to Exposure Detail (accessed by following row links in Data Log)\n",
"- \n",
@@ -30,10 +31,10 @@
"# Times Square replaces this cell with the user's parameters.\n",
"# So, don't put anything else here!\n",
"\n",
- "# day_obs values: TODAY, YESTERDAY, YYYY-MM-DD\n",
+ "# day_obs values: TODAY, v, YYYY-MM-DD\n",
"# Report on observing nights that start upto but not included this day.\n",
"#!day_obs = '2024-09-25' # Value to use for local testing (Summit)\n",
- "day_obs = \"2024-11-25\" # TODO Change to 'YESTERDAY' and 'TODAY' to test with default before push\n",
+ "day_obs = \"YESTERDAY\" # TODO Change to 'YESTERDAY' and 'TODAY' to test with default before push\n",
"\n",
"# Total number of days of data to display (ending on day_obs)\n",
"number_of_days = \"1\" # TODO Change to '1' to test with default before push\n",
@@ -149,9 +150,10 @@
"outputs": [],
"source": [
"instrum_str = \", \".join(list(allsrc.exp_src.instruments.keys()))\n",
- "md(f\"# Showing data for {min_date.date()} to {max_date.date()} for {instrum_str}\")\n",
+ "md(f\"# Showing data for {min_date.date()} to {max_date.date()}\")\n",
+ "md(f\"**Instruments:** {instrum_str}\")\n",
"md(\n",
- " f\"Most recently run on: **{dt.datetime.now().replace(microsecond=0).isoformat(sep=' ')}**\"\n",
+ " f'Most recently run on: **{dt.datetime.now().replace(microsecond=0).isoformat(sep=\" \")}**'\n",
")"
]
},
@@ -176,6 +178,7 @@
" - Simonyi\n",
"* [Data Log](#Data-Log-BETA)\n",
"* [Narrative Log](#Narrative-Log-BETA)\n",
+ "* [Consolidated Database](#Consolidated-Database-DRAFT)\n",
"* [Developer Only Section](#Developer-Only-Section-REMOVE)"
]
},
@@ -183,29 +186,6 @@
"cell_type": "markdown",
"id": "9",
"metadata": {},
- "source": [
- "## Night Report BETA \n",
- "#### Possible Changes\n",
- "- Word wrap instead of chopping words at end of line?\n",
- "- (Add summary from AI)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "10",
- "metadata": {},
- "outputs": [],
- "source": [
- "# Night Report\n",
- "# Display time log\n",
- "allrep.nig_rep.time_log_as_markdown()"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "11",
- "metadata": {},
"source": [
"## Almanac BETA \n",
"\n",
@@ -216,8 +196,12 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "12",
- "metadata": {},
+ "id": "10",
+ "metadata": {
+ "jupyter": {
+ "source_hidden": true
+ }
+ },
"outputs": [],
"source": [
"# Display various almanac values (for moon, sun)\n",
@@ -227,29 +211,30 @@
},
{
"cell_type": "markdown",
- "id": "13",
+ "id": "11",
"metadata": {},
"source": [
- "## Consolidated Database DRAFT\n",
- "Retrieve exposure records from ConsDB\n",
- "\n",
- "TODO: Integrate this and other queries into columns of other sections.\n",
- "TODO: remove hardcode day, change backend to allow range of days"
+ "## Night Report BETA \n",
+ "#### Possible Changes\n",
+ "- Word wrap instead of chopping words at end of line?\n",
+ "- (Add summary from AI)"
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "14",
+ "id": "12",
"metadata": {},
"outputs": [],
"source": [
- "allsrc.cdb_src.get_exposures(instrument=\"lsstcomcam\")"
+ "# Night Report\n",
+ "# Display time log\n",
+ "allrep.nig_rep.time_log_as_markdown()"
]
},
{
"cell_type": "markdown",
- "id": "15",
+ "id": "13",
"metadata": {},
"source": [
"## Summary plots of whole night DRAFT"
@@ -257,7 +242,7 @@
},
{
"cell_type": "markdown",
- "id": "16",
+ "id": "14",
"metadata": {},
"source": [
"(content not yet defined in storyboard)\n",
@@ -268,7 +253,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "17",
+ "id": "15",
"metadata": {},
"outputs": [],
"source": [
@@ -279,7 +264,7 @@
},
{
"cell_type": "markdown",
- "id": "18",
+ "id": "16",
"metadata": {},
"source": [
"## Links to related resources BETA\n",
@@ -289,7 +274,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "19",
+ "id": "17",
"metadata": {},
"outputs": [],
"source": [
@@ -308,7 +293,7 @@
},
{
"cell_type": "markdown",
- "id": "20",
+ "id": "18",
"metadata": {},
"source": [
"## Time Accounting BETA\n",
@@ -325,7 +310,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "21",
+ "id": "19",
"metadata": {},
"outputs": [],
"source": [
@@ -337,7 +322,7 @@
},
{
"cell_type": "markdown",
- "id": "22",
+ "id": "20",
"metadata": {},
"source": [
"- (1) *Expected* slew time as per Schedular\n",
@@ -346,7 +331,7 @@
},
{
"cell_type": "markdown",
- "id": "23",
+ "id": "21",
"metadata": {},
"source": [
"## Jira Tickets BETA \n",
@@ -358,7 +343,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "24",
+ "id": "22",
"metadata": {},
"outputs": [],
"source": [
@@ -372,7 +357,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "25",
+ "id": "23",
"metadata": {},
"outputs": [],
"source": [
@@ -393,7 +378,7 @@
},
{
"cell_type": "markdown",
- "id": "26",
+ "id": "24",
"metadata": {},
"source": [
"## BLOCKS Observed DRAFT\n",
@@ -402,7 +387,7 @@
},
{
"cell_type": "markdown",
- "id": "27",
+ "id": "25",
"metadata": {},
"source": [
"## Data Log BETA\n",
@@ -412,7 +397,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "28",
+ "id": "26",
"metadata": {},
"outputs": [],
"source": [
@@ -429,7 +414,7 @@
},
{
"cell_type": "markdown",
- "id": "29",
+ "id": "27",
"metadata": {},
"source": [
"## Narrative Log BETA \n",
@@ -442,7 +427,17 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "30",
+ "id": "28",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "allsrc.nar_src"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "29",
"metadata": {
"editable": true,
"slideshow": {
@@ -459,8 +454,32 @@
},
{
"cell_type": "markdown",
+ "id": "30",
+ "metadata": {},
+ "source": [
+ "## Consolidated Database DRAFT\n",
+ "Retrieve exposure records from ConsDB. The Developer Only Section contains a list of ALL fields currently available in ConsDB per [ConsDB API](https://usdf-rsp.slac.stanford.edu/consdb/docs).\n",
+ "\n",
+ "- TODO: Integrate this and other queries into columns of other sections.\n",
+ "- "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
"id": "31",
"metadata": {},
+ "outputs": [],
+ "source": [
+ "allsrc.cdb_src.get_exposures(instrument=\"lsstcomcam\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "32",
+ "metadata": {
+ "jp-MarkdownHeadingCollapsed": true
+ },
"source": [
"-----------\n",
"------------"
@@ -468,7 +487,7 @@
},
{
"cell_type": "markdown",
- "id": "32",
+ "id": "33",
"metadata": {},
"source": [
"# Developer Only Section REMOVE\n",
@@ -479,7 +498,7 @@
},
{
"cell_type": "markdown",
- "id": "33",
+ "id": "34",
"metadata": {},
"source": [
"## Overview \n"
@@ -488,7 +507,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "34",
+ "id": "35",
"metadata": {},
"outputs": [],
"source": [
@@ -521,7 +540,7 @@
},
{
"cell_type": "markdown",
- "id": "35",
+ "id": "36",
"metadata": {},
"source": [
"## Data Status\n",
@@ -531,7 +550,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "36",
+ "id": "37",
"metadata": {},
"outputs": [],
"source": [
@@ -542,7 +561,7 @@
},
{
"cell_type": "markdown",
- "id": "37",
+ "id": "38",
"metadata": {},
"source": [
"## This report uses the following data sources\n",
@@ -557,7 +576,7 @@
},
{
"cell_type": "markdown",
- "id": "38",
+ "id": "39",
"metadata": {},
"source": [
"## Where was this run?\n",
@@ -572,26 +591,35 @@
},
{
"cell_type": "markdown",
- "id": "39",
+ "id": "40",
"metadata": {},
"source": [
"## Available Consolidated Database fields\n",
- "instrument/tablename/fieldname"
+ "It should be easy to add data to this report when it corresponds to any of the fields listed below. \n",
+ "\n",
+ "**Format:** *instrument/tablename/fieldname*"
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "40",
+ "id": "41",
"metadata": {},
"outputs": [],
"source": [
- "con_src.all_available_fields"
+ "#! txt = '\\n- '.join(allsrc.cdb_src.all_available_fields)\n",
+ "#! md(f'- {txt}')\n",
+ "table = \"\"\n",
+ "table += \"\\n| instrument | tablename | fieldname |\"\n",
+ "table += \"\\n| :--------: | :-------: | :-------: |\"\n",
+ "for field in allsrc.cdb_src.all_available_fields:\n",
+ " table += f'\\n| {field.replace(\"/\",\" | \")} |'\n",
+ "md(table)"
]
},
{
"cell_type": "markdown",
- "id": "41",
+ "id": "42",
"metadata": {},
"source": [
"## Section overviews moved here"
@@ -600,7 +628,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "42",
+ "id": "43",
"metadata": {},
"outputs": [],
"source": [
@@ -615,7 +643,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "43",
+ "id": "44",
"metadata": {},
"outputs": [],
"source": [
@@ -629,7 +657,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "44",
+ "id": "45",
"metadata": {},
"outputs": [],
"source": [
@@ -639,7 +667,7 @@
},
{
"cell_type": "markdown",
- "id": "45",
+ "id": "46",
"metadata": {},
"source": [
"## Finale"
@@ -648,7 +676,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "46",
+ "id": "47",
"metadata": {},
"outputs": [],
"source": [
diff --git a/notebooks_tsqr/dashboard.ipynb b/notebooks_tsqr/dashboard.ipynb
index 9c42ac3..e52e431 100644
--- a/notebooks_tsqr/dashboard.ipynb
+++ b/notebooks_tsqr/dashboard.ipynb
@@ -41,7 +41,7 @@
" \"primary_software_components\",\n",
" \"primary_hardware_components\",\n",
"]\n",
- "samples = dash.get_big_sample(endpoint, 10000, fields)\n",
+ "samples = dash.get_big_sample(endpoint, 100, fields)\n",
"len(samples)"
]
},
@@ -70,7 +70,7 @@
"id": "5",
"metadata": {},
"source": [
- "## Connects"
+ "## Samples"
]
},
{
@@ -80,11 +80,14 @@
"metadata": {},
"outputs": [],
"source": [
- "dash = Dashboard()\n",
- "score, working = dash.report()\n",
- "# On 9/12/2024 this gets two failed connects.\n",
- "# The are usdf: exposurelog/instruments,exposurelog/exposures\n",
- "# When usdfdev if fully functional, there should be zero failed connects (with both VPNs active)."
+ "server = \"https://usdf-rsp-dev.slac.stanford.edu\"\n",
+ "samples = dash.get_sample_data(server, count=3)\n",
+ "print(\"N Records from each endpoint used by LogRep:\")\n",
+ "for endpoint, sample in samples.items():\n",
+ " base_ep = endpoint.replace(server, \"\")\n",
+ " md(f\"\\n### Endpoint: {base_ep}\")\n",
+ " print(f\"{endpoint}\")\n",
+ " pp(sample)"
]
},
{
@@ -92,7 +95,7 @@
"id": "7",
"metadata": {},
"source": [
- "## Score"
+ "## Connects"
]
},
{
@@ -102,9 +105,11 @@
"metadata": {},
"outputs": [],
"source": [
- "md(f\"{score=:.0%}\")\n",
- "md(\"**Servers that are fully functional** for Logging and Reporting:\")\n",
- "mdlist([f\"- {w}\" for w in working])"
+ "dash = Dashboard()\n",
+ "score, working = dash.report()\n",
+ "# On 9/12/2024 this gets two failed connects.\n",
+ "# The are usdf: exposurelog/instruments,exposurelog/exposures\n",
+ "# When usdfdev if fully functional, there should be zero failed connects (with both VPNs active)."
]
},
{
@@ -112,7 +117,7 @@
"id": "9",
"metadata": {},
"source": [
- "## Samples"
+ "## Score"
]
},
{
@@ -122,14 +127,9 @@
"metadata": {},
"outputs": [],
"source": [
- "server = \"https://usdf-rsp-dev.slac.stanford.edu\"\n",
- "samples = dash.get_sample_data(server)\n",
- "print(\"One record of data from all endpoints used by LogRep:\")\n",
- "for endpoint, sample in samples.items():\n",
- " base_ep = endpoint.replace(server, \"\")\n",
- " md(f\"\\n### Endpoint: {base_ep}\")\n",
- " print(f\"{endpoint}\")\n",
- " pp(sample)"
+ "md(f\"{score=:.0%}\")\n",
+ "md(\"**Servers that are fully functional** for Logging and Reporting:\")\n",
+ "mdlist([f\"- {w}\" for w in working])"
]
},
{
diff --git a/python/lsst/ts/logging_and_reporting/dashboard.py b/python/lsst/ts/logging_and_reporting/dashboard.py
index 70a4f8a..cf19949 100644
--- a/python/lsst/ts/logging_and_reporting/dashboard.py
+++ b/python/lsst/ts/logging_and_reporting/dashboard.py
@@ -77,11 +77,10 @@ def nr_values(self, samples):
return (cset | sset | hset), histo
- def get_sample_data(self, server):
-
+ def get_sample_data(self, server, count=1):
samples = defaultdict(dict) # samples[endpoint_url] -> one_record_dict
for adapter in self.adapters:
- sa = adapter(server_url=server, limit=1)
+ sa = adapter(server_url=server, limit=count)
for ep in sa.endpoints:
qstr = "?instrument=LSSTComCamSim" if ep == "exposures" else ""
url = f"{server}/{sa.service}/{ep}{qstr}"
@@ -91,7 +90,7 @@ def get_sample_data(self, server):
if isinstance(recs, dict):
samples[url] = recs
else:
- samples[url] = recs[0]
+ samples[url] = recs[0:count]
except Exception as err:
# Made following more complicated to get around
# comboniation of BLACK re-write and FLAKE8
diff --git a/python/lsst/ts/logging_and_reporting/source_adapters.py b/python/lsst/ts/logging_and_reporting/source_adapters.py
index b300af2..5f4403a 100644
--- a/python/lsst/ts/logging_and_reporting/source_adapters.py
+++ b/python/lsst/ts/logging_and_reporting/source_adapters.py
@@ -647,7 +647,8 @@ def obs_date(rec):
table = list()
- # Sort by OBS_NIGHT
+ # Sort by OBS_NIGHT within that by date_added (datetime)
+ recs = sorted(recs, key=obs_date)
recs = sorted(recs, key=obs_night)
# time_lost_type=weather is RARE
for tele, g0 in itertools.groupby(recs, key=obs_night):
@@ -657,7 +658,8 @@ def obs_date(rec):
attrstr = ""
attrstr += f"**{rec_dt}**"
if rec.get("components"):
- attrstr += " **" + ", ".join(rec.get("components", [])) + "** "
+ complist = ", ".join(rec.get("components", []))
+ attrstr += f" **{complist}**"
if rec.get("time_lost", 0) > 0:
attrstr += f" Time Lost: {rec.get('time_lost')};"