From 3a0d0fd6cbc7df0fa55f24a7b38c79cfae3bfd3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= <boschi1997@gmail.com>
Date: Mon, 29 Jan 2024 11:38:13 +0100
Subject: [PATCH] fix

---
 .github/workflows/run-benchmarks.yml       |  5 +-
 ragstack-e2e-tests/benchmarks/visualize.py | 58 ++++++++++++++++++++--
 2 files changed, 55 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/run-benchmarks.yml b/.github/workflows/run-benchmarks.yml
index d61dd2597..9c87ae26d 100644
--- a/.github/workflows/run-benchmarks.yml
+++ b/.github/workflows/run-benchmarks.yml
@@ -56,7 +56,7 @@ jobs:
           git config user.name "$GITHUB_ACTOR"
           git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
 
-          cp ragstack-e2e-tests/benchmarks/reports/report.html /tmp/report.html
+          mv ragstack-e2e-tests/benchmarks/reports /tmp/reports
           git reset --hard
           git clean -df
           git status
@@ -66,8 +66,7 @@ jobs:
           git reset --hard origin/gh-pages
           mkdir -p benchmarks/reports
           dir=benchmarks/reports/$(date +"%Y-%m-%d_%H-%M-%S")
-          mkdir -p $dir
-          cp /tmp/report.html $dir/report.html
+          mv /tmp/reports $dir
           git add -A benchmarks
           git commit -am "Update benchmarks ${{ github.run_id }} - $dir"
           git push origin gh-pages
diff --git a/ragstack-e2e-tests/benchmarks/visualize.py b/ragstack-e2e-tests/benchmarks/visualize.py
index 51aee24a5..e0e28ff24 100644
--- a/ragstack-e2e-tests/benchmarks/visualize.py
+++ b/ragstack-e2e-tests/benchmarks/visualize.py
@@ -137,24 +137,70 @@ def render_html(values, export_to: str):
                   </div>
                 </div>
               </div>
-            <table class="table-fixed">
         """
 
     with open(export_to, "w") as f:
         f.write(html)
 
 
+def render_markdown(values, export_to: str, plot_src: str = "plot.png"):
+    title = "RAGStack - Benchmarks Report"
+    import datetime
+
+    title += " - " + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    table_rows = []
+    for i, value in enumerate(values):
+        ps = []
+        for p in PERCENTILES:
+            k = f"p{p}"
+            ps.append(value["values"][k])
+        table_rows.append([value["name"]] + ps)
+
+    headers = ["Test Case"] + [f"p{p}" for p in PERCENTILES]
+
+    def draw_row(list):
+        return f"| {' | '.join(str(col) for col in list)} |"
+
+    table = draw_row(headers)
+    table += "\n|" + "|".join(["---" for _ in headers]) + "|"
+    for row in table_rows:
+        table += "\n" + draw_row(row)
+
+    plot = f"<img src='{plot_src}' />"
+
+    md = f"""# {title}
+
+{table}
+
+{plot}"""
+
+    with open(export_to, "w") as f:
+        f.write(md)
+
+
 def draw_report(directory_path: str, format: str, filter_by: str):
     values = scan_result_directory(directory_path, filter_by)
-    if format == "plot" or format == "plot_svg":
+    is_all = format == "all"
+    if is_all or format == "plot" or format == "plot_svg":
         render_plot(
             values,
             export_to=os.path.join(directory_path, "plot.svg"),
             show=format == "plot",
         )
-    elif format == "html":
+    if is_all or format == "html":
         render_html(values, export_to=os.path.join(directory_path, "report.html"))
-    else:
+    if is_all or format == "markdown":
+        render_plot(
+            values,
+            export_to=os.path.join(directory_path, "plot.svg"),
+            show=False,
+        )
+        render_markdown(
+            values,
+            export_to=os.path.join(directory_path, "report.md"),
+            plot_src="plot.svg",
+        )
+    if is_all or format == "table":
         render_table(values)
 
 
@@ -172,7 +218,9 @@ def draw_report(directory_path: str, format: str, filter_by: str):
         help="Reports dir",
     )
     parser.add_argument(
-        "--format", choices=["table", "plot", "plot_svg", "html"], default="table"
+        "--format",
+        choices=["table", "plot", "plot_svg", "html", "markdown", "all"],
+        default="table",
     )
     parser.add_argument(
         "-f",