Implement

braintrustdata · Sep 30, 2024 · 2b61b14 · 2b61b14
1 parent 108aad3
commit 2b61b14
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 4 deletions.
diff --git a/eval/src/braintrust.ts b/eval/src/braintrust.ts
@@ -64,6 +64,16 @@ export async function runEval(args: Params, onSummary: OnSummaryFn) {
   // Change working directory
   process.chdir(path.resolve(root));
 
-  const command = `npx braintrust eval --jsonl ${paths}`;
+  let command: string;
+  switch (args.runtime) {
+    case "node":
+      command = `npx braintrust eval --jsonl ${paths}`;
+      break;
+    case "python":
+      command = `braintrust eval --jsonl ${paths}`;
+      break;
+    default:
+      throw new Error(`Unsupported runtime: ${args.runtime}`);
+  }
   await runCommand(command, onSummary);
 }
diff --git a/eval/src/main.ts b/eval/src/main.ts
@@ -38,9 +38,6 @@ async function main(): Promise<void> {
       `Invalid arguments: ${args.error.errors.map(e => e.message).join("\n")}`,
     );
   }
-  if (args.data.runtime !== "node") {
-    throw new Error("Only Node.js runtime is supported");
-  }
 
   await upsertComment(`${TITLE}Evals in progress... ⌛`);