Capture MVP progress in Prototype (#29)

lsst-ts · Jan 9, 2025 · 21c2b9a · 21c2b9a
1 parent 809427b
commit 21c2b9a
Show file tree

Hide file tree

Showing 40 changed files with 5,784 additions and 1,186 deletions.
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
@@ -1,34 +1,7 @@
-name: Python CI
+name: lint
 
-env:
-  PYTHON_VERSION: "3.12"  # Default Python version
-
-"on":
-  merge_group: {}
-  pull_request: {}
-  push:
-    branches-ignore:
-      # These should always correspond to pull requests, so ignore them for
-      # the push trigger and let them be triggered by the pull_request
-      # trigger, avoiding running the workflow twice.  This is a minor
-      # optimization so there's no need to ensure this is comprehensive.
-      - "dependabot/**"
-      - "renovate/**"
-      - "tickets/**"
-      - "u/**"
+on: push
 
 jobs:
-
-  lint:
-
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: Run pre-commit
-        uses: pre-commit/[email protected]
+  call-workflow:
+    uses: lsst-ts/tssw_workflows/.github/workflows/lint.yaml@main
diff --git a/.gitignore b/.gitignore
@@ -77,6 +77,8 @@ target/
 
 # Jupyter Notebook
 .ipynb_checkpoints
+Untitled.ipynb
+
 
 # IPython
 profile_default/
@@ -160,3 +162,8 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+.pre-commit-config.yaml
+.flake8
+.isort.cfg
+.mypy.ini
+.ruff.toml
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
diff --git a/.ts_pre_commit_config.yaml b/.ts_pre_commit_config.yaml
@@ -0,0 +1,10 @@
+black: true
+clang-format: false
+flake8: true
+format-xmllint: false
+isort: true
+mypy: false
+check-yaml: true
+check-xml: true
+ruff: true
+towncrier: false
diff --git a/README.md b/README.md
@@ -2,6 +2,20 @@
 
 Project-wide Logging and Reporting
 
+## About this software
+This is a **Prototype** application.  It is intended to help figure
+out what is needed for a *Project-wide Logging and Reporting* system
+for Rubin. It helps with this by presenting content in the
+[NightLog](https://usdf-rsp-dev.slac.stanford.edu/times-square/github/lsst-ts/ts_logging_and_reporting/NightLog)
+jupyter notebook under Times Square. The intent is to eventually put
+the content into a solid User Interface environment (such as **LOVE:**
+*LSST Operator's Visualization Environment*). The Times Square
+environment is used because it allows rapid turn-around from code
+change to an easily accessible report on USDF.
+
+While current *rendering* is in the Notebook, most of the rest of the code
+(back-end) is in the python package included in this repository.  The
+intent is for that package to be modified and used with the final UI.
 
 ## Set up
 

diff --git a/notebooks_tsqr/ExposureDetail.ipynb b/notebooks_tsqr/ExposureDetail.ipynb
@@ -0,0 +1,282 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Parameters.\n",
+    "# Times Square replaces this cell with the user's parameters.\n",
+    "# So, don't put anything else here!\n",
+    "\n",
+    "# day_obs values: TODAY, YESTERDAY, YYYY-MM-DD\n",
+    "# Report on observing nights that start upto but not included this day.\n",
+    "#!day_obs = '2024-09-25' # Value to use for local testing (usdf)\n",
+    "day_obs = \"2024-11-20\"  # TODO Change to 'YESTERDAY' to test with default before push\n",
+    "instrument = \"LSSTComCam\"  # LSSTComCam, LATISS, LSSTCam\n",
+    "observation_reason = \"ALL\"\n",
+    "observation_type = \"science\"  # TODO: \"science\", \"acq\", default=\"ALL\"\n",
+    "science_program = \"ALL\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1",
+   "metadata": {
+    "jupyter": {
+     "source_hidden": true
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# IMPORT everything\n",
+    "import os\n",
+    "import datetime as dt\n",
+    "import pandas as pd\n",
+    "from IPython.display import HTML, display\n",
+    "\n",
+    "# When running under Times Square, install pkg from github.\n",
+    "# Otherwise use what is installed locally (intended to be dev editiable pkg)\n",
+    "if os.environ.get(\"EXTERNAL_INSTANCE_URL\"):\n",
+    "    dev_msg = (\n",
+    "        'Installing \"lsst.ts.logging_and_reporting\" from github using \"prototype\" branch. \\n'\n",
+    "        \"TODO: Make the need for this go away by getting Logging_and_Reporting installed in RSP.\"\n",
+    "    )\n",
+    "    !pip install --upgrade git+https://github.com/lsst-ts/ts_logging_and_reporting.git@prototype > /dev/null 2>&1\n",
+    "else:\n",
+    "    dev_msg = \"Imported lsst.ts.logging_and_reporting from local packages.\"\n",
+    "#! import lsst.ts.logging_and_reporting.source_adapters as sad\n",
+    "import lsst.ts.logging_and_reporting.utils as ut\n",
+    "from lsst.ts.logging_and_reporting.reports import md\n",
+    "\n",
+    "from lsst.ts.logging_and_reporting.all_sources import AllSources\n",
+    "from lsst.ts.logging_and_reporting.all_reports import AllReports"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2",
+   "metadata": {
+    "jupyter": {
+     "source_hidden": true
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Set default env to \"usdf\" and try before PUSH to repo.\n",
+    "\n",
+    "# The default provided here is for local testing.\n",
+    "# Under Times Square it is ignored.\n",
+    "server = os.environ.get(\n",
+    "    \"EXTERNAL_INSTANCE_URL\", ut.Server.usdf\n",
+    ")  # TODO try with \"usdf\" before push (else \"summit\")\n",
+    "\n",
+    "# Normalize Parameters (both explicit Times Squares params and implicit ones)\n",
+    "observation_reason = None if observation_reason == \"ALL\" else observation_reason\n",
+    "observation_type = None if observation_type == \"ALL\" else observation_type\n",
+    "science_program = None if science_program == \"ALL\" else science_program\n",
+    "params = dict(\n",
+    "    day_obs=day_obs,\n",
+    "    instrument=instrument,\n",
+    "    observation_reason=observation_reason,\n",
+    "    observation_type=observation_type,\n",
+    "    science_program=science_program,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3",
+   "metadata": {
+    "jupyter": {
+     "source_hidden": true
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Normalize Parameters (both explicit Times Squares params, in implicit ones)\n",
+    "date = ut.get_datetime_from_dayobs_str(day_obs)\n",
+    "# date:  is EXCLUSIVE (upto, but not including)\n",
+    "days = 1\n",
+    "\n",
+    "# Thus: [min_day_obs,max_day_obs)\n",
+    "# Format: string, YYYY-MM-DD\n",
+    "min_date = date\n",
+    "max_date = date + dt.timedelta(days=1)\n",
+    "min_day_obs = min_date.strftime(\"%Y-%m-%d\")  # Inclusive\n",
+    "max_day_obs = max_date.strftime(\"%Y-%m-%d\")  # prep for Exclusive"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Read records from (almost) all sources\n",
+    "timer = ut.Timer()\n",
+    "allsrc = AllSources(\n",
+    "    server_url=server,\n",
+    "    min_dayobs=min_day_obs,\n",
+    "    max_dayobs=max_day_obs,\n",
+    "    verbose=False,  # TODO change to False before push\n",
+    "    warning=True,  # TODO change to True before push\n",
+    "    limit=5000,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# https://usdf-rsp-dev.slac.stanford.edu/consdb/query"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if observation_reason:\n",
+    "    md(f\"# Observation Reason: {observation_reason}\")\n",
+    "if observation_type:\n",
+    "    md(f\"# Observation Type: {observation_type}\")\n",
+    "if science_program:\n",
+    "    md(f\"# Science Program: {science_program}\")\n",
+    "df = allsrc.exposure_detail(\n",
+    "    instrument=instrument,\n",
+    "    science_program=science_program,\n",
+    "    observation_reason=observation_reason,\n",
+    "    observation_type=observation_type,\n",
+    ")\n",
+    "\n",
+    "md(f\"The number of exposures in this filtered result is {len(df.index)}.\")\n",
+    "md(\n",
+    "    f\"Exposures cover the day_obs range starting {allsrc.min_dayobs} and ending before {allsrc.max_dayobs}.\"\n",
+    ")\n",
+    "display(HTML(df.to_html(index=False)))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7",
+   "metadata": {},
+   "source": [
+    "-----------------"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8",
+   "metadata": {
+    "jp-MarkdownHeadingCollapsed": true
+   },
+   "source": [
+    "# Developer Only Section"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "9",
+   "metadata": {},
+   "source": [
+    "| Symbol | Meaning |\n",
+    "|:---|:---|\n",
+    "| <font style=\"background-color:green; color:white; font-size:20px\">&nbsp;G&nbsp;</font>| Good |\n",
+    "| <font style=\"background-color:yellow; color:black; font-size:20px\">&nbsp;?&nbsp;</font> | Questionable |\n",
+    "| <font style=\"background-color:red; color:black; font-size:20px\">&nbsp;R&nbsp;</font> | Junk |"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "10",
+   "metadata": {
+    "jupyter": {
+     "source_hidden": true
+    }
+   },
+   "outputs": [],
+   "source": [
+    "display(params)\n",
+    "print(allsrc.exp_src)\n",
+    "print(f\"{instrument=}, {science_program=}, {observation_reason=}, {observation_type=}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "11",
+   "metadata": {
+    "jupyter": {
+     "source_hidden": true
+    }
+   },
+   "outputs": [],
+   "source": [
+    "print({k: len(v) for k, v in allsrc.exp_src.exposures.items()})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "12",
+   "metadata": {
+    "jupyter": {
+     "source_hidden": true
+    }
+   },
+   "outputs": [],
+   "source": [
+    "allsrc"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "13",
+   "metadata": {
+    "jupyter": {
+     "source_hidden": true
+    }
+   },
+   "outputs": [],
+   "source": [
+    "print(f\"Elapsed time (excluding code import): {timer.toc:.1f} seconds\")\n",
+    "print(f\"Finished {str(dt.datetime.now().replace(microsecond=0))} UTC\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}