From 0a3f6dd2625738d84b069dd6e5b7d324712044a6 Mon Sep 17 00:00:00 2001 From: ecatkins Date: Sat, 14 Oct 2023 11:21:02 -0400 Subject: [PATCH 1/4] Enable azure openai engines --- py/autoevals/llm.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/py/autoevals/llm.py b/py/autoevals/llm.py index 38d1b99..e04329d 100644 --- a/py/autoevals/llm.py +++ b/py/autoevals/llm.py @@ -78,18 +78,21 @@ def __init__( render_args=None, max_tokens=None, temperature=None, + engine=None ): found = False - for m in SUPPORTED_MODELS: - # Prefixes are ok, because they are just time snapshots - if model.startswith(m): - found = True - break - if not found: - raise ValueError(f"Unsupported model: {model}. Currently only supports OpenAI chat models.") + if engine is None: + for m in SUPPORTED_MODELS: + # Prefixes are ok, because they are just time snapshots + if model.startswith(m): + found = True + break + if not found: + raise ValueError(f"Unsupported model: {model}. Currently only supports OpenAI chat models.") self.name = name self.model = model + self.engine = engine self.messages = messages self.choice_scores = choice_scores self.classification_functions = classification_functions @@ -136,6 +139,7 @@ def _request_args(self, output, expected, **kwargs): return dict( Completion=openai.ChatCompletion, model=self.model, + engine=self.engine, messages=self._render_messages(output=output, expected=expected, **kwargs), functions=self.classification_functions, function_call={"name": "select_choice"}, @@ -176,6 +180,7 @@ class ModelGradedSpec: prompt: str choice_scores: dict[str, float] model: Optional[str] = None + engine: Optional[str] = None use_cot: Optional[bool] = None temperature: Optional[float] = None @@ -195,6 +200,7 @@ def __init__( use_cot=True, max_tokens=512, temperature=0, + engine=None ): choice_strings = list(choice_scores.keys()) @@ -214,12 +220,13 @@ def __init__( classification_functions=build_classification_functions(use_cot), max_tokens=max_tokens, temperature=temperature, + engine=engine, render_args={"__choices": choice_strings}, ) @classmethod def from_spec(cls, name: str, spec: ModelGradedSpec, **kwargs): - return cls(name, spec.prompt, spec.choice_scores, **kwargs) + return cls(name, spec.prompt, spec.choice_scores, **kwargs) @classmethod def from_spec_file(cls, name: str, path: str, **kwargs): @@ -229,10 +236,12 @@ def from_spec_file(cls, name: str, path: str, **kwargs): class SpecFileClassifier(LLMClassifier): - def __new__(cls, model=None, use_cot=None, max_tokens=None, temperature=None): + def __new__(cls, model=None, engine=None, use_cot=None, max_tokens=None, temperature=None): kwargs = {} if model is not None: kwargs["model"] = model + if engine is not None: + kwargs["engine"] = engine if use_cot is not None: kwargs["use_cot"] = use_cot if max_tokens is not None: From f9ee9e0e22b26bfdfc3c78959725233e2ed3a44d Mon Sep 17 00:00:00 2001 From: ecatkins Date: Mon, 16 Oct 2023 10:10:55 -0400 Subject: [PATCH 2/4] remove error for unsupported models --- py/autoevals/llm.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/py/autoevals/llm.py b/py/autoevals/llm.py index e04329d..d97868e 100644 --- a/py/autoevals/llm.py +++ b/py/autoevals/llm.py @@ -80,16 +80,6 @@ def __init__( temperature=None, engine=None ): - found = False - if engine is None: - for m in SUPPORTED_MODELS: - # Prefixes are ok, because they are just time snapshots - if model.startswith(m): - found = True - break - if not found: - raise ValueError(f"Unsupported model: {model}. Currently only supports OpenAI chat models.") - self.name = name self.model = model self.engine = engine From becb7131e0f2c3076d06acb61ce06bd1cf436cce Mon Sep 17 00:00:00 2001 From: ecatkins Date: Mon, 16 Oct 2023 12:02:05 -0400 Subject: [PATCH 3/4] make fixup --- py/autoevals/llm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/py/autoevals/llm.py b/py/autoevals/llm.py index d97868e..870c2c2 100644 --- a/py/autoevals/llm.py +++ b/py/autoevals/llm.py @@ -78,7 +78,7 @@ def __init__( render_args=None, max_tokens=None, temperature=None, - engine=None + engine=None, ): self.name = name self.model = model @@ -190,7 +190,7 @@ def __init__( use_cot=True, max_tokens=512, temperature=0, - engine=None + engine=None, ): choice_strings = list(choice_scores.keys()) @@ -216,7 +216,7 @@ def __init__( @classmethod def from_spec(cls, name: str, spec: ModelGradedSpec, **kwargs): - return cls(name, spec.prompt, spec.choice_scores, **kwargs) + return cls(name, spec.prompt, spec.choice_scores, **kwargs) @classmethod def from_spec_file(cls, name: str, path: str, **kwargs): From 2021bc1a38c3c09fccc43cc695b59bc20a6bd899 Mon Sep 17 00:00:00 2001 From: Ankur Goyal Date: Mon, 16 Oct 2023 11:56:49 -0700 Subject: [PATCH 4/4] Update python tests --- .testcache/oai.sqlite | Bin 315392 -> 348160 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/.testcache/oai.sqlite b/.testcache/oai.sqlite index eca8f65d95fb704caaea926f1c219f04e50308a2..a273cf9682e88d4d6a129b503222b76f03cadb8c 100644 GIT binary patch delta 1670 zcmaJ>TTI(#6t?5vNyMlDZFHzqV9|>@t)q|&!IicS0wn=*0TO5t>*mHz;*vOt?cBK2 zO+u(6fP(r@Q#+{CJ&mCvwprz&+oWwz>nPe@ySJ?m+w(3`weEMIKnIV zle@mZ)e?;OxOiV1+u;c%Y(lmn)zawn!%e5|jKOAUG+AsmxcNEpwfg&gVhzuG?;eNu zpAZaqgn>s0@gXo?Kb#>ydVA)r{fKL;zdR=34I`+A8{dO+xa0&sL;5dWPJOco{DB`u z1h}p=z^ z3m+(;>e)sSY>I#r9}piO2&?rS@^mLHNwXv4mApPVo9&Av`r4h@1e~;jGi0If{O0X2 ztn=v3!_s?11w`n>@J^1O7b4a@A~2c=%aE=hUMURWhgo1km_yIS z+@t(Kn1e1)pWE8j$#4pvVB_9kpsTBnw>#uLn2lSQ@e*AzoS6k#fMZ5nEt>;>8jQ0e zBjm#C*E;r3U1fhHrtpIFDw;S?Xe~y4VkjKXT)d(vE#9=H-x}@-n3<5tr@`q9f;0JB z(a?_KpTN>;oP-Ns5T)?p8{i1U7l8@BdJdeVXjY~LQC6g6K;b!XJ$>?QXl3|xhl-C4fkw`2p@2?PuNfIAG1t7Hu=t~@*5>OqP z4ob1;{%ls{=(rS12^>mcMMP%%ds5V!w#2ev$+qQAgrg|$z0v(CY+ z$4bE|WDOUPxQm?5_KmZO2b+fC=MlVvJm1t}zFUE%6=r5Z^VL%yeD}^$pp% zsJFSn5^IV?d_GUBU6S@xZqR)FmFl_&?&A7d?1jVnTExFbZ8i-kOoDA#0Xb8V6IEb{FgHDF76lY^{6h~Lv0U&woMwJ!RSp997fYryLI3~& delta 67 zcmZozAlk4%c!D%*IRgWO??eTA#`28`OZYjw`1*hX48D_j1g2~jRFGoayi4&4PjjDK Td!HO55HkTW^Y%VDmZFUSlZX^d