Adding interactive data grid

This needs to be updated once we have data for multiple days.
fixie-ai · Apr 13, 2024 · 4ae2551 · 4ae2551
1 parent 0b40532
commit 4ae2551
Show file tree

Hide file tree

Showing 10 changed files with 271 additions and 22 deletions.
diff --git a/README.md b/README.md
@@ -4,4 +4,8 @@ Website with current metrics on the fastest AI models.
 ## thefastest.ai website
 Everything is in the `\website` folder. Pushing to main branch will trigger GitHub action to build and publish the site.
 
+## Data
+The data displayed in the table is from a custom benchmarking tool (full [source is on GitHub](https://github.com/fixie-ai/ai-benchmarks)). We run this tool daily in multiple [regions](https://fly.io/docs/reference/regions/) so that we have benchmarks for multiple geos.
+
+## Other
 Built using Astro and the https://github.com/nicdun/astro-tech-blog
diff --git a/website/package.json b/website/package.json
@@ -15,6 +15,8 @@
 		"@astrojs/tailwind": "^5.0.4",
 		"@types/react": "^18.2.45",
 		"@types/react-dom": "^18.2.18",
+		"ag-grid-community": "^31.2.1",
+		"ag-grid-react": "^31.2.1",
 		"astro": "^4.0.7",
 		"canvas-confetti": "^1.9.2",
 		"mdast-util-to-string": "^4.0.0",

diff --git a/website/src/components/BenchmarkDefinitions.astro b/website/src/components/BenchmarkDefinitions.astro
@@ -0,0 +1,20 @@
+---
+import Section from './Section.astro';
+import { ModelDefinition, RegionDefinition, TTRDefinition, TTFTDefinition, TPSDefinition, TokensDefinition, TotalTimeDefinition } from '@/utils/DataGridDefinitions';
+---
+
+<Section>
+	<div class="text-sm">
+		<p class="text-xl font-bold">Definitions</p>
+		<br />
+		<ul class="list-none">
+			<li class="pb-2"><b>{ ModelDefinition.title } → </b>{ ModelDefinition.definition }</li>
+			<li class="pb-2"><b>{ RegionDefinition.title } → </b>{ RegionDefinition.definition }</li>
+			<li class="pb-2"><b>{ TTRDefinition.title } → </b>{ TTRDefinition.definition }</li>
+			<li class="pb-2"><b>{ TTFTDefinition.title } → </b>{ TTFTDefinition.definition }</li>
+			<li class="pb-2"><b>{ TPSDefinition.title } → </b>{ TPSDefinition.definition }</li>
+			<li class="pb-2"><b>{ TokensDefinition.title } → </b>{ TokensDefinition.definition }</li>
+			<li class="pb-2"><b>{ TotalTimeDefinition.title } → </b>{ TotalTimeDefinition.definition }</li>
+		</ul>
+	</div>
+</Section>
diff --git a/website/src/components/DataGrid.astro b/website/src/components/DataGrid.astro
@@ -0,0 +1,64 @@
+---
+// Import the necessary CSS for AG Grid
+import 'ag-grid-community/styles/ag-grid.css';
+import 'ag-grid-community/styles/ag-theme-quartz.css';
+---
+<style>
+  .ag-theme-quartz {
+    font-family: monospace;
+  }
+</style>
+<div id="myGrid" class="ag-theme-quartz text-sm bg-stone-100 font-mono text-gray-950 dark:bg-stone-900 dark:text-white " style="height: 800px; width: 1000px;"></div>
+
+<script>
+  import { createGrid } from 'ag-grid-community';
+  import { gridOptions } from '@/utils/DataGridDefinitions.ts';
+
+  let gridApi;
+
+  // URLs of our JSON data
+  const urls = [
+    'https://storage.googleapis.com/thefastest-data/cdg/text/2024-04-13.json',
+    'https://storage.googleapis.com/thefastest-data/iad/text/2024-04-13.json',
+    'https://storage.googleapis.com/thefastest-data/sea/text/2024-04-13.json',
+  ];
+
+  async function fetchBenchmarkData(urls) {
+    try {
+      // Fetch all the JSON files concurrently
+      const fetchPromises = urls.map(url => fetch(url).then(response => response.json()));
+      const jsonFiles = await Promise.all(fetchPromises);
+
+      // Combine the data from all files
+      const combinedData = jsonFiles.map(file => {
+        // Extract date from the time string and ignore the timestamp
+        const date = file.time.split('T')[0];
+
+        // Map each result object to a new object that includes the date and region
+        return file.results.map(result => ({
+          date,
+          region: file.region,
+          ...result
+        }));
+      }).flat(); // Flatten the array of arrays into a single array
+
+      return combinedData;
+    } catch (error) {
+      console.error('Error combining JSON files:', error);
+    }
+  }
+
+  fetchBenchmarkData(urls)
+    .then(combinedResults => {
+      console.log('Combined results:', combinedResults);
+      gridOptions.rowData = combinedResults;
+      gridApi = createGrid(document.querySelector("#myGrid"), gridOptions);
+    })
+    .catch(error => {
+      console.error('Error in combining JSON files:', error);
+    });
+
+</script>
+
+
+
diff --git a/website/src/components/Hero.astro b/website/src/components/Hero.astro
@@ -1,6 +1,5 @@
 ---
 import Section from './Section.astro';
-
 ---
 
 <Section>
@@ -9,16 +8,6 @@ import Section from './Section.astro';
 		<p class="py-2">Apps are only as fast as their slowest component. LLMs vary tremendously on quality and speed. The table below shows the lastest performance metrics for many models and environments. We hope this data helps you choose the most performant model. </p>
 		<br />
 		<p>Have another model you want us to benchmark? Hit us up with an <a class="underline hover:text-orange-600" href="https://github.com/fixie-ai/fastest.ai/issues" target="_blank">issue on GitHub.</a></p>
-		<br />
-		<p>Definitions:</p>
-		<br />
-		<ul class="list-none">
-			<li class="pb-2"><b>TTR</b> → Time to (HTTP) Response. Indicates the overall speed of the serving infrastructure of the provider/model.</li>
-			<li class="pb-2"><b>TTFT</b> → Time to First Token. Time to get first text from the model. This translates directly into how quickly the UI starts to update when displaying the response and indicates the overall speed with which the model begins working on the request and processing the input tokens.</li>
-			<li class="pb-2"><b>TPS</b> → Tokens per Second. This is how quickly text is emitted from the model and translates directly into how quickly the UI finishes displaying the response. It also indicates how quickly the model can produce each output token.</li>
-			<li class="pb-2"><b>Tokens</b> → The total number of output tokens. Longer responses take longer to produce.</li>
-			<li class="pb-2"><b>Total Time</b> → The total time from the start of the request until the response is complete, i.e., the last token has been generated. Total time = TTFT + TPS * Tokens.</li>
-		</ul>
 	</div>
 </Section>
 

diff --git a/website/src/components/HowToUse.astro b/website/src/components/HowToUse.astro
@@ -0,0 +1,12 @@
+---
+import Section from './Section.astro';
+---
+
+<Section>
+	<div class="text-sm">
+		<p class="text-xl font-bold">How to Use</p>
+		<br />
+		<p>The above table is interactive. You can sort on any column. Further, the hamburger menu (appears when you hover a column header) allows for custom filtering. Have fun exploring!</p>
+
+	</div>
+</Section>
diff --git a/website/src/pages/index.astro b/website/src/pages/index.astro
@@ -2,19 +2,21 @@
 import { AppConfig } from '@/utils/AppConfig';
 import Base from '@/layouts/Base.astro';
 import Hero from '@/components/Hero.astro';
-import MarkdownTable from '@/components/MarkdownTable.astro';
-import { Content as CurrentMetrics } from '@/data/currentMetrics.md';
-import * as metrics from '@/data/currentMetrics.md'; 
+import Section from '@/components/Section.astro';
+import DataGrid from '@/components/DataGrid.astro';
+import BenchmarkDefinitions from '@/components/BenchmarkDefinitions.astro';
+import HowToUse from '@/components/HowToUse.astro';
 
 const { title } = AppConfig;
 const { description } = AppConfig;
 
 ---
 <Base head={{ title, description }}>
 	<Hero />
-
-	<MarkdownTable>
-		<CurrentMetrics />
-		<p class="italic text-right">Last updated: {metrics.frontmatter.date}</p>
-	</MarkdownTable>
+	<Section>
+		<DataGrid />
+		<p class="italic text-right">Last updated: April 13, 2024</p>
+	</Section>
+	<HowToUse />
+	<BenchmarkDefinitions />
 </Base>
diff --git a/website/src/utils/DataGridDefinitions.ts b/website/src/utils/DataGridDefinitions.ts
@@ -0,0 +1,128 @@
+interface ValueFormatterParam {
+  value: number;
+}
+
+export const ModelDefinition = {
+	title: "Provider/Model",
+	definition: "The LLM and host provider used."
+};
+
+export const RegionDefinition = {
+	title: "Region",
+	definition: "The region where the benchmark was run."
+};
+
+export const TTRDefinition = {
+	title: "TTR",
+	definition: "Time to (HTTP) Response. Indicates the overall speed of the serving infrastructure of the provider/model."
+};
+
+export const TTFTDefinition = {
+	title: "TTFT",
+	definition: "Time to First Token. Time to get first text from the model. This translates directly into how quickly the UI starts to update when displaying the response and indicates the overall speed with which the model begins working on the request and processing the input tokens."
+};
+
+export const TPSDefinition = {
+	title: "TPS",
+	definition: "Tokens per Second. This is how quickly text is emitted from the model and translates directly into how quickly the UI finishes displaying the response. It also indicates how quickly the model can produce each output token."
+};
+
+export const TokensDefinition = {
+	title: "Num Tokens",
+	definition: "The total number of output tokens. Longer responses take longer to produce."
+};
+
+export const TotalTimeDefinition = {
+	title: "Total Time",
+	definition: "The total time from the start of the request until the response is complete, i.e., the last token has been generated. Total time = TTFT + TPS * Tokens."
+};
+
+// Set-up all of our column definitions that will be used in the Data Grid
+const headerClass = "font-bold";
+
+// Model column
+const columnModel = {
+	field: "model",
+	headerName: ModelDefinition.title,
+	headerTooltip: ModelDefinition.definition,
+	headerClass: headerClass,
+	minWidth: 400, 
+	tooltipField: "output"
+};
+
+// Region column
+const columnRegion = {
+	field: "region",
+	headerName: RegionDefinition.title,
+	headerTooltip: RegionDefinition.definition,
+	headerClass: headerClass,
+	maxWidth: 100
+};
+
+// TTR column
+const columnTTR = {
+	field: "ttr",
+	headerName: TTRDefinition.title,
+	headerTooltip: TTRDefinition.definition,
+	headerClass: headerClass,
+	maxWidth: 90,
+	valueFormatter: (p: ValueFormatterParam) => p.value.toFixed(2)
+};
+
+// TTFT column
+const columnTTFT = {
+	field: "ttft",
+	headerName: TTFTDefinition.title,
+	headerTooltip: TTFTDefinition.definition,
+	headerClass: headerClass,
+	maxWidth: 90,
+	valueFormatter: (p: ValueFormatterParam) => p.value.toFixed(2)
+};
+
+// TPS column
+const columnTPS = {
+	field: "tps",
+	headerName: TPSDefinition.title,
+	headerTooltip: TPSDefinition.definition,
+	headerClass: headerClass,
+	maxWidth: 90,
+	valueFormatter: (p: ValueFormatterParam) => p.value.toFixed(2)
+};
+
+// Tokens column
+const columnNumTokens = {
+	field: "num_tokens",
+	headerName: TokensDefinition.title,
+	headerTooltip: TokensDefinition.definition,
+	headerClass: headerClass,
+	minWidth: 100,
+	maxWidth: 100,
+	wrapHeaderText: true
+};
+
+// Total Time column
+const columnTotalTime = {
+	field: "total_time",
+	headerName: TotalTimeDefinition.title,
+	headerTooltip: TotalTimeDefinition.definition,
+	headerClass: headerClass,
+	minWidth: 100,
+	maxWidth: 100,
+	wrapHeaderText: true,
+	valueFormatter: (p: ValueFormatterParam) => p.value.toFixed(2)
+};
+
+export const gridOptions = {
+  alwaysShowVerticalScroll: true,
+  autoSizeStrategy: { type: 'fitCellContents' },
+  defaultColDef: {
+    filter: true,
+    minWidth: 80,
+  },
+  pagination: true,
+  paginationPageSizeSelector: [50, 150, 500],
+  paginationPageSize: 150,
+  rowData: [],
+  // Columns to be displayed (Should match rowData properties)
+  columnDefs: [ columnModel, columnRegion, columnTTR, columnTTFT, columnTPS, columnNumTokens, columnTotalTime]
+};
diff --git a/website/tsconfig.json b/website/tsconfig.json
@@ -42,5 +42,6 @@
 			"@/*": ["./src/*"]
 		}
 	},
-	"include": ["src/**/*.ts", "src/**/*.tsx", "src/**/*.astro", "src/components/PostOverview.tsx"]
+	"include": ["src/**/*.ts", "src/**/*.tsx", "src/**/*.astro", "src/components/PostOverview.tsx"],
+	"exclude": ["src/components/DataGrid.astro"]
 }
diff --git a/website/yarn.lock b/website/yarn.lock
@@ -968,6 +968,19 @@ acorn@^8.11.2:
   resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.11.3.tgz#71e0b14e13a4ec160724b38fb7b0f233b1b81d7a"
   integrity sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==
 
+[email protected], ag-grid-community@^31.2.1:
+  version "31.2.1"
+  resolved "https://registry.yarnpkg.com/ag-grid-community/-/ag-grid-community-31.2.1.tgz#6031c91baa5caebccfa58a5131efc13339643e50"
+  integrity sha512-D+gnUQ4dHZ/EQJmupQnDqcEKiCEeuK5ZxlsIpdPKgHg/23dmW+aEdivtB9nLpSc2IEK0RUpchcSxeUT37Boo5A==
+
+ag-grid-react@^31.2.1:
+  version "31.2.1"
+  resolved "https://registry.yarnpkg.com/ag-grid-react/-/ag-grid-react-31.2.1.tgz#21be0703225e4c7ad88b1b6ca13491a6d9a072c1"
+  integrity sha512-9UH3xxXRwZfW97oz58KboyCJl4t+zdetopieeHVcttsXX1DvGFDUIEz7A1sQaG8e1DAXLMf3IxoIPrfWheH4XA==
+  dependencies:
+    ag-grid-community "31.2.1"
+    prop-types "^15.8.1"
+
 ansi-align@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/ansi-align/-/ansi-align-3.0.1.tgz#0cdf12e111ace773a86e9a1fad1225c43cb19a59"
@@ -2274,7 +2287,7 @@ longest-streak@^3.0.0:
   resolved "https://registry.yarnpkg.com/longest-streak/-/longest-streak-3.1.0.tgz#62fa67cd958742a1574af9f39866364102d90cd4"
   integrity sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==
 
-loose-envify@^1.1.0:
+loose-envify@^1.1.0, loose-envify@^1.4.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf"
   integrity sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==
@@ -2861,7 +2874,7 @@ npm-run-path@^5.1.0:
   dependencies:
     path-key "^4.0.0"
 
-object-assign@^4.0.1:
+object-assign@^4.0.1, object-assign@^4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863"
   integrity sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==
@@ -3172,6 +3185,15 @@ prompts@^2.4.2:
     kleur "^3.0.3"
     sisteransi "^1.0.5"
 
+prop-types@^15.8.1:
+  version "15.8.1"
+  resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.8.1.tgz#67d87bf1a694f48435cf332c24af10214a3140b5"
+  integrity sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==
+  dependencies:
+    loose-envify "^1.4.0"
+    object-assign "^4.1.1"
+    react-is "^16.13.1"
+
 property-information@^6.0.0:
   version "6.5.0"
   resolved "https://registry.yarnpkg.com/property-information/-/property-information-6.5.0.tgz#6212fbb52ba757e92ef4fb9d657563b933b7ffec"
@@ -3213,6 +3235,11 @@ react-dom@^18.2.0:
     loose-envify "^1.1.0"
     scheduler "^0.23.0"
 
+react-is@^16.13.1:
+  version "16.13.1"
+  resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.13.1.tgz#789729a4dc36de2999dc156dd6c1d9c18cea56a4"
+  integrity sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==
+
 react-refresh@^0.14.0:
   version "0.14.0"
   resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.14.0.tgz#4e02825378a5f227079554d4284889354e5f553e"