Add LangSmith @unit test tracking

requirements.txt

@@ -23,7 +23,7 @@ idna==3.7
 iniconfig==2.0.0
 inline-snapshot==0.8.0
 instructor==1.2.0
-langsmith==0.1.48
+langsmith>=0.1.50,<0.2
 markdown-it-py==3.0.0
 mdurl==0.1.2
 multidict==6.0.5

tests/test_classification_literals.py

@@ -4,6 +4,7 @@
 from pydantic import BaseModel
 
 import pytest
+from langsmith import unit
 
 
 class ClassifySpam(BaseModel):
@@ -19,6 +20,11 @@ class ClassifySpam(BaseModel):
 @pytest.mark.asyncio_cooperative
 @pytest.mark.parametrize("client, data", product(clients, data))
 async def test_classification(client, data):
+    await check_classification(client, data)
+
+
+@unit
+async def check_classification(client, data):
     input, expected = data
     prediction = await client.create(
         response_model=ClassifySpam,

tests/test_retries.py

@@ -3,6 +3,7 @@
 from pydantic import AfterValidator, BaseModel, Field
 from typing import Annotated
 from util import clients
+from langsmith import unit
 
 
 def uppercase_validator(v):
@@ -37,6 +38,10 @@ class UserDetail(BaseModel):
 @pytest.mark.asyncio_cooperative
 @pytest.mark.parametrize("client, data", product(clients, data))
 async def test_retries(client, data):
+    await check_retries(client, data)
+
+@unit
+async def check_retries(client, data):
     query, expected = data
     response = await client.create(
         response_model=UserDetail,

tests/test_simple_validation.py

@@ -4,7 +4,7 @@
 
 from openai import OpenAI
 from pydantic import BaseModel, BeforeValidator, ValidationError
-
+from langsmith import unit
 import instructor
 from instructor import llm_validator
 from util import clients
@@ -36,6 +36,11 @@ class QuestionAnswerNoEvil(BaseModel):
 @pytest.mark.asyncio_cooperative
 @pytest.mark.parametrize("client, data", product(clients, data))
 async def test_simple_validation(client, data):
+    await check_simple_validation(client, data)
+
+
+@unit
+async def check_simple_validation(client, data):
     question, context = data
 
     with pytest.raises(ValidationError):
@@ -53,3 +58,4 @@ async def test_simple_validation(client, data):
             response_model=QuestionAnswerNoEvil,
             max_retries=0,
         )
+        return resp

tests/test_vision_table.py

@@ -12,6 +12,7 @@
 from itertools import product
 from util import clients
 from instructor import AsyncInstructor
+from langsmith import unit
 
 
 def md_to_df(data: Any) -> Any:
@@ -63,6 +64,11 @@ class MultipleTables(BaseModel):
 @pytest.mark.asyncio_cooperative
 @pytest.mark.parametrize("client, url", product(clients, urls))
 async def test_extract(client: AsyncInstructor, url: str):
+    await check_extract(client, url)
+
+
+@unit
+async def check_extract(client: AsyncInstructor, url: str):
     if client.kwargs["model"] != "gpt-4-turbo":
         pytest.skip("Only OpenAI supported for now, we need to support images for both")