more edits

python/bench/upload_examples_bench.py

@@ -1,11 +1,10 @@
 import statistics
 import time
 from typing import Dict
 from uuid import uuid4
 from langsmith.schemas import DataType, ExampleCreateWithAttachments
 import sys
-sys.path.append('./../langsmith')
-from client import Client
+from langsmith import Client
 
 def create_large_json(length: int) -> Dict:
     """Create a large JSON object for benchmarking purposes."""
@@ -38,7 +37,7 @@
         "outputs": create_large_json(json_size),
     })
 
-DATASET_NAME = "TEST DATASET"
+DATASET_NAME = "upsert_llm_evaluator_benchmark_dataset"
 def benchmark_example_uploading(num_examples: int, json_size: int, samples: int = 1) -> Dict:
     """
     Benchmark run creation with specified parameters.
@@ -64,13 +63,10 @@
         old_start = time.perf_counter()
         inputs=[e.inputs for e in examples]
         outputs=[e.outputs for e in examples]
-        # the create_examples endpoint fails above 20mb
-        try:
-            client.create_examples(inputs=inputs,
-                                outputs=outputs,dataset_id=dataset.id)
-            old_elapsed = time.perf_counter() - old_start
-        except:
-            old_elapsed = 1000000
+        # the create_examples endpoint fails above 20mb - so this will crash with json_size > ~100
+        client.create_examples(inputs=inputs,
+                            outputs=outputs,dataset_id=dataset.id)
+        old_elapsed = time.perf_counter() - old_start
 
         # New method
         multipart_start = time.perf_counter()
@@ -121,4 +117,4 @@
 
 
 if __name__ == "__main__":
-    main(json_size, num_examples)
+    main(json_size, num_examples)

python/langsmith/client.py

@@ -3373,14 +3373,14 @@ def create_example_from_run(
     def upsert_examples_multipart(
         self,
         *,
-        upserts: List[ls_schemas.ExampleCreateWithAttachments] = None,
-    ) -> dict: # Should we create an object for the return type - like UpsertExamplesResponse?
+        upserts: List[ls_schemas.ExampleUpsertWithAttachments] = None,
+    ) -> ls_schemas.UpsertExamplesResponse:
         """Upsert examples."""
-        if not (self.info.instance_flags or {}).get(
+        """ if not (self.info.instance_flags or {}).get(
                 "examples_multipart_enabled", False
             ):
             raise ValueError("Your LangSmith version does not allow using the multipart examples endpoint, please update to the latest version.")
-
+         """
         if upserts is None:
             upserts = []
         parts: list[MultipartPart] = []

python/langsmith/schemas.py

@@ -89,7 +89,7 @@ class ExampleCreate(ExampleBase):
     split: Optional[Union[str, List[str]]] = None
 
 
-class ExampleCreateWithAttachments(ExampleCreate):
+class ExampleUpsertWithAttachments(ExampleCreate):
     """Example create with attachments."""
 
     attachments: Optional[Attachments] = None
@@ -131,12 +131,6 @@ def url(self) -> Optional[str]:
         return None
 
 
-class ExampleSearch(ExampleBase):
-    """Example returned via search."""
-
-    id: UUID
-
-
 class ExampleUpdate(BaseModel):
     """Update class for Example."""
 
@@ -151,6 +145,10 @@ class Config:
 
         frozen = True
 
+class ExampleUpdateWithAttachments(ExampleUpdate):
+    """Example update with attachments."""
+    id: UUID
+    attachments: Optional[Attachments] = None
 
 class DataType(str, Enum):
     """Enum for dataset data types."""
@@ -988,3 +986,11 @@ class UsageMetadata(TypedDict):
 
     Does *not* need to sum to full output token count. Does *not* need to have all keys.
     """
+
+class UpsertExamplesResponse(TypedDict):
+    """Response object returned from the upsert_examples_multipart method."""
+
+    count: int
+    """The number of examples that were upserted."""
+    example_ids: List[str]
+    """The ids of the examples that were upserted."""

python/tests/integration_tests/test_client.py

@@ -20,7 +20,7 @@
 from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
 
 from langsmith.client import ID_TYPE, Client
-from langsmith.schemas import DataType, ExampleCreateWithAttachments
+from langsmith.schemas import DataType, ExampleUpsertWithAttachments
 from langsmith.utils import (
     LangSmithNotFoundError,
     LangSmithConnectionError,
@@ -373,7 +373,7 @@ def test_error_surfaced_invalid_uri(uri: str) -> None:
 def test_upsert_examples_multipart() -> None:
     """Test upserting examples with attachments via multipart endpoint."""
     dataset_name = "__test_upsert_examples_multipart" + uuid4().hex[:4]
-    langchain_client = Client(api_url="https://dev.api.smith.langchain.com")
+    langchain_client = Client(api_url="https://dev.api.smith.langchain.com", api_key="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")
     if langchain_client.has_dataset(dataset_name=dataset_name):
         langchain_client.delete_dataset(dataset_name=dataset_name)
 
@@ -385,7 +385,7 @@ def test_upsert_examples_multipart() -> None:
 
     # Test example with all fields
     example_id = uuid4()
-    example_1 = ExampleCreateWithAttachments(
+    example_1 = ExampleUpsertWithAttachments(
         id=example_id,
         dataset_id=dataset.id,
         inputs={"text": "hello world"},
@@ -395,7 +395,7 @@ def test_upsert_examples_multipart() -> None:
         },
     )
     # Test example without id
-    example_2 = ExampleCreateWithAttachments(
+    example_2 = ExampleUpsertWithAttachments(
         dataset_id=dataset.id,
         inputs={"text": "foo bar"},
         outputs={"response": "baz"},
@@ -419,8 +419,24 @@ def test_upsert_examples_multipart() -> None:
     all_examples_in_dataset = [example for example in langchain_client.list_examples(dataset_id=dataset.id)]
     assert len(all_examples_in_dataset) == 2
 
+    example_1_update = ExampleUpsertWithAttachments(
+        id=example_id,
+        dataset_id=dataset.id,
+        inputs={"text": "bar baz"},
+        outputs={"response": "foo"},
+        attachments={
+            "my_file": ("text/plain", b"more test content"),
+        },
+    )
+    updated_examples = langchain_client.upsert_examples_multipart(upserts=[example_1_update])
+    assert updated_examples['count'] == 1
+    assert updated_examples['example_ids'][0] == str(example_id)
+    updated_example = langchain_client.read_example(updated_examples['example_ids'][0])
+    assert updated_example.inputs['text'] == "bar baz"
+    assert updated_example.outputs['response'] == "foo"
+
     # Test that adding invalid example fails - even if valid examples are added alongside
-    example_3 = ExampleCreateWithAttachments(
+    example_3 = ExampleUpsertWithAttachments(
         dataset_id=uuid4(), # not a real dataset
         inputs={"text": "foo bar"},
         outputs={"response": "baz"},
@@ -435,7 +451,7 @@ def test_upsert_examples_multipart() -> None:
     all_examples_in_dataset = [example for example in langchain_client.list_examples(dataset_id=dataset.id)]
     assert len(all_examples_in_dataset) == 2
 
-    # Throw type errors when not passing ExampleCreateWithAttachments
+    # Throw type errors when not passing ExampleUpsertWithAttachments
     with pytest.raises(AttributeError):
         langchain_client.upsert_examples_multipart(upserts=[{"foo":"bar"}])