diff --git a/website/src/components/BenchmarkDefinitions.astro b/website/src/components/BenchmarkDefinitions.astro index 26bbfd0..b6e98e1 100644 --- a/website/src/components/BenchmarkDefinitions.astro +++ b/website/src/components/BenchmarkDefinitions.astro @@ -7,10 +7,10 @@ import { ModelDefinition, TTFTDefinition, TPSDefinition, TotalTimeDefinition } f

Definitions

\ No newline at end of file diff --git a/website/src/components/Methodology.astro b/website/src/components/Methodology.astro index 853f9ad..b826c54 100644 --- a/website/src/components/Methodology.astro +++ b/website/src/components/Methodology.astro @@ -5,12 +5,12 @@ import Section from './Section.astro';

Methodology

-

Distributed Footprint → We run our tools daily in multiple data centers using Fly.io. Currently we run in cdg, iad, and sea.

-

Short Requests → Input requests are a short, single message (~20 tokens) and ask for a brief output response.

-

Max Output → Max output tokens is set to 100 to avoid distortion of TPS values from long outputs.

-

Warmup for Consistent Setup → A warmup connection is made to remove any connection setup latency.

-

TTFT Roundtrip → The TTFT clock starts when the HTTP request is made and stops when the first token result is received in the response stream.

-

Try 3, Keep 1 → For each provider, three separate inferences are done, and the best result is kept (to remove any outliers due to queuing etc).

+

Distributed Footprint: We run our tools daily in multiple data centers using Fly.io. Currently we run in cdg, iad, and sea.

+

Short Requests: Input requests are a short, single message (~20 tokens) and ask for a brief output response.

+

Max Output: Max output tokens is set to 100 to avoid distortion of TPS values from long outputs.

+

Warmup for Consistent Setup: A warmup connection is made to remove any connection setup latency.

+

TTFT Roundtrip: The TTFT clock starts when the HTTP request is made and stops when the first token result is received in the response stream.

+

Try 3, Keep 1: For each provider, three separate inferences are done, and the best result is kept (to remove any outliers due to queuing etc).

diff --git a/website/src/components/Sources.astro b/website/src/components/Sources.astro index 8e314ab..551f4ff 100644 --- a/website/src/components/Sources.astro +++ b/website/src/components/Sources.astro @@ -5,9 +5,9 @@ import Section from './Section.astro';

Source

-

Raw Data → All data is in this public GCS bucket.

-

Benchmarking Tools → The full test suite is available in the ai-benchmarks repo.

-

Website → Full source code for this site is on GitHub.

+

Raw Data: All data is in this public GCS bucket.

+

Benchmarking Tools: The full test suite is available in the ai-benchmarks repo.

+

Website: Full source code for this site is on GitHub.

diff --git a/website/src/utils/DataGridDefinitions.ts b/website/src/utils/DataGridDefinitions.ts index 9d9c2cf..ed7bfaf 100644 --- a/website/src/utils/DataGridDefinitions.ts +++ b/website/src/utils/DataGridDefinitions.ts @@ -94,7 +94,8 @@ const columnTTFT = { headerName: TTFTDefinition.title, headerTooltip: TTFTDefinition.definition, headerClass: headerClass, - maxWidth: 90, + // minWidth: 80, + // maxWidth: 90, valueFormatter: (p: ValueFormatterParam) => p.value.toFixed(2) }; @@ -104,7 +105,8 @@ const columnTPS = { headerName: TPSDefinition.title, headerTooltip: TPSDefinition.definition, headerClass: headerClass, - maxWidth: 90, + // minWidth: 90, + // maxWidth: 90, valueFormatter: (p: ValueFormatterParam) => p.value.toFixed(2) }; @@ -114,8 +116,8 @@ const columnNumTokens = { headerName: TokensDefinition.title, headerTooltip: TokensDefinition.definition, headerClass: headerClass, - // minWidth: 100, - maxWidth: 100, + minWidth: 100, + // maxWidth: 100, wrapHeaderText: true }; @@ -126,9 +128,10 @@ const columnTotalTime = { headerTooltip: TotalTimeDefinition.definition, headerClass: headerClass, // minWidth: 100, - maxWidth: 100, + // maxWidth: 100, wrapHeaderText: true, valueFormatter: (p: ValueFormatterParam) => p.value.toFixed(2), + sort: 'asc', }; export const gridOptions = {