Merge branch 'main' into manu/fix-tracing-bug

js/env.ts

@@ -1,7 +1,9 @@
 export interface EnvI {
   OPENAI_API_KEY?: string;
+  OPENAI_BASE_URL?: string;
 }
 
 export const Env: EnvI = {
   OPENAI_API_KEY: undefined,
+  OPENAI_BASE_URL: undefined,
 };

js/json.ts

@@ -31,7 +31,7 @@ async function jsonDiff(
   o2: any,
   stringScorer: Scorer<string, {}>,
   numberScorer: Scorer<number, {}>
-): Promise<number> {
+): Promise<number | null> {
   if (isObject(o1) && isObject(o2)) {
     if (Object.keys(o1).length == 0 && Object.keys(o2).length == 0) {
       return 1;
@@ -45,20 +45,24 @@ async function jsonDiff(
       )
     );
 
-    const baseScores = await Promise.all(
-      allKeys.map((k) => jsonDiff(o1[k], o2[k], stringScorer, numberScorer))
-    );
+    const baseScores = (
+      await Promise.all(
+        allKeys.map((k) => jsonDiff(o1[k], o2[k], stringScorer, numberScorer))
+      )
+    ).filter((s) => s !== null) as number[];
     return baseScores.reduce((acc, s) => acc + s, 0) / baseScores.length;
   } else if (isArray(o1) && isArray(o2)) {
     if (o1.length === 0 && o2.length === 0) {
       return 1;
     }
 
-    const baseScores = await Promise.all(
-      Array.from({
-        length: Math.min(o1.length, o2.length),
-      }).map((_, i) => jsonDiff(o1[i], o2[i], stringScorer, numberScorer))
-    );
+    const baseScores = (
+      await Promise.all(
+        Array.from({
+          length: Math.min(o1.length, o2.length),
+        }).map((_, i) => jsonDiff(o1[i], o2[i], stringScorer, numberScorer))
+      )
+    ).filter((s) => s !== null) as number[];
     return (
       baseScores.reduce((acc, s) => acc + s, 0) / Math.max(o1.length, o2.length)
     );

js/node.ts

@@ -1,4 +1,5 @@
 import { Env } from "./env.js";
 Env.OPENAI_API_KEY = process.env.OPENAI_API_KEY;
+Env.OPENAI_BASE_URL = process.env.OPENAI_BASE_URL;
 
 export * from "./index.js";

js/oai.ts

@@ -44,7 +44,7 @@ export function buildOpenAIClient(options: OpenAIAuth): OpenAI {
   return new OpenAI({
     apiKey: openAiApiKey || Env.OPENAI_API_KEY,
     organization: openAiOrganizationId,
-    baseURL: openAiBaseUrl || PROXY_URL,
+    baseURL: openAiBaseUrl || Env.OPENAI_BASE_URL || PROXY_URL,
     defaultHeaders: openAiDefaultHeaders,
     dangerouslyAllowBrowser: openAiDangerouslyAllowBrowser,
   });

package.json

@@ -41,7 +41,7 @@
     "typescript": "^5.3.3"
   },
   "dependencies": {
-    "@braintrust/core": "^0.0.6",
+    "@braintrust/core": "^0.0.8",
     "@types/node": "^20.10.5",
     "compute-cosine-similarity": "^1.1.0",
     "js-levenshtein": "^1.1.6",

py/autoevals/json.py

@@ -26,11 +26,13 @@ def json_diff(self, o1, o2):
 
             all_keys = set(o1.keys()).union(set(o2.keys()))
             base_scores = [self.json_diff(o1.get(k), o2.get(k)) for k in all_keys]
+            base_scores = [s for s in base_scores if s is not None]
             return sum(base_scores) / len(base_scores)
         elif isinstance(o1, list) and isinstance(o2, list):
             if len(o1) == 0 and len(o2) == 0:
                 return 1
             base_scores = [self.json_diff(e1, e2) for (e1, e2) in zip(o1, o2)]
+            base_scores = [s for s in base_scores if s is not None]
             return sum(base_scores) / max(len(o1), len(o2))
         elif isinstance(o1, str) and isinstance(o2, str):
             return self.string_scorer.eval(o1, o2).score

py/autoevals/oai.py

@@ -1,4 +1,5 @@
 import asyncio
+import os
 import sys
 import textwrap
 import time
@@ -18,7 +19,7 @@ class OpenAIWrapper:
 
 def prepare_openai(is_async=False, api_key=None, base_url=None):
     if base_url is None:
-        base_url = PROXY_URL
+        base_url = os.environ.get("OPENAI_BASE_URL", PROXY_URL)
 
     try:
         import openai

py/autoevals/string.py

@@ -55,8 +55,8 @@ async def _run_eval_async(self, output, expected=None, **kwargs):
         if expected is None:
             raise ValueError("EmbeddingSimilarity requires an expected value")
 
-        output_embedding_p = arun_cached_request(input=f"{self.prefix}{output}", **self.extra_args)
-        expected_embedding_p = arun_cached_request(input=f"{self.prefix}{expected}", **self.extra_args)
+        output_embedding_p = arun_cached_request("embed", input=f"{self.prefix}{output}", **self.extra_args)
+        expected_embedding_p = arun_cached_request("embed", input=f"{self.prefix}{expected}", **self.extra_args)
 
         output_result, expected_result = await output_embedding_p, await expected_embedding_p
         return Score(

py/autoevals/version.py

@@ -1 +1 @@
-VERSION = "0.0.40"
+VERSION = "0.0.43"