Merge branch 'main' into wfh/debug

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langsmith",
-  "version": "0.3.6",
+  "version": "0.3.7",
   "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.",
   "packageManager": "yarn@1.22.19",
   "files": [
@@ -162,7 +162,7 @@
     "typedoc": "^0.27.6",
     "typedoc-plugin-expand-object-like-types": "^0.1.2",
     "typescript": "^5.4.5",
-    "vitest": "^2.1.8",
+    "vitest": "^3.0.5",
     "zod": "^3.23.8"
   },
   "peerDependencies": {

js/src/index.ts

@@ -18,4 +18,4 @@ export { RunTree, type RunTreeConfig } from "./run_trees.js";
 export { overrideFetchImplementation } from "./singletons/fetch.js";
 
 // Update using yarn bump-version
-export const __version__ = "0.3.6";
+export const __version__ = "0.3.7";

js/src/utils/jestlike/index.ts

@@ -301,18 +301,18 @@ export function generateWrapperFromJestlikeMethods(
           experimentConfig?.metadata?.LANGSMITH_ENVIRONMENT ??
           getEnvironmentVariable("LANGSMITH_ENVIRONMENT");
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const metadata: Record<string, any> = {
+        const suiteMetadata: Record<string, any> = {
           ...experimentConfig?.metadata,
           __ls_runner: testRunnerName,
         };
         if (environment !== undefined) {
-          metadata.ENVIRONMENT = environment;
+          suiteMetadata.ENVIRONMENT = environment;
         }
         if (nodeEnv !== undefined) {
-          metadata.NODE_ENV = nodeEnv;
+          suiteMetadata.NODE_ENV = nodeEnv;
         }
         if (langsmithEnvironment !== undefined) {
-          metadata.LANGSMITH_ENVIRONMENT = langsmithEnvironment;
+          suiteMetadata.LANGSMITH_ENVIRONMENT = langsmithEnvironment;
         }
         const context = {
           suiteUuid,
@@ -321,7 +321,7 @@ export function generateWrapperFromJestlikeMethods(
           createdAt: new Date().toISOString(),
           projectConfig: {
             ...experimentConfig,
-            metadata,
+            metadata: suiteMetadata,
           },
           enableTestTracking: experimentConfig?.enableTestTracking,
         };
@@ -347,11 +347,13 @@ export function generateWrapperFromJestlikeMethods(
           const endTime = new Date();
           let branch;
           let commit;
+          let dirty;
           try {
             branch = execSync("git rev-parse --abbrev-ref HEAD")
               .toString()
               .trim();
             commit = execSync("git rev-parse HEAD").toString().trim();
+            dirty = execSync("git status --porcelain").toString().trim() !== "";
           } catch {
             return;
           }
@@ -376,23 +378,21 @@ export function generateWrapperFromJestlikeMethods(
               finalModifiedAt = endTime.toISOString();
             }
             const datasetInfo = datasetSetupInfo.get(suiteUuid);
-            const { as_of } = await client.readDatasetVersion({
+            await client.updateProject(datasetInfo.project.id, {
+              metadata: {
+                ...suiteMetadata,
+                commit,
+                branch,
+                dirty,
+              },
+            });
+            await client.updateDatasetTag({
               datasetId: datasetInfo.dataset.id,
               asOf: finalModifiedAt,
+              tag: `git:commit:${commit}`,
             });
-            await Promise.all([
-              client.updateDatasetTag({
-                datasetId: datasetInfo.dataset.id,
-                asOf: as_of,
-                tag: `git:branch:${branch}`,
-              }),
-              client.updateDatasetTag({
-                datasetId: datasetInfo.dataset.id,
-                asOf: as_of,
-                tag: `git:commit:${commit}`,
-              }),
-            ]);
-          } catch {
+          } catch (e) {
+            console.error(e);
             return;
           }
         });

python/langsmith/client.py

@@ -4422,7 +4422,7 @@ def list_examples(
             offset (int, default=0): The offset to start from. Defaults to 0.
             limit (Optional[int]): The maximum number of examples to return.
             metadata (Optional[dict]): A dictionary of metadata to filter by.
-            filter (Optional[str]): A structured fileter string to apply to
+            filter (Optional[str]): A structured filter string to apply to
                 the examples.
             include_attachments (bool, default=False): Whether to include the
                 attachments in the response. Defaults to False.

python/langsmith/testing/_internal.py

@@ -691,6 +691,8 @@ def __init__(
         self.pytest_plugin = pytest_plugin
         self.pytest_nodeid = pytest_nodeid
         self._logged_reference_outputs: Optional[dict] = None
+        self.inputs = inputs
+        self.reference_outputs = reference_outputs
 
         if pytest_plugin and pytest_nodeid:
             pytest_plugin.add_process_to_test_suite(
@@ -866,16 +868,14 @@ def _run_test(
         langtest_extra=langtest_extra,
     )
     _TEST_CASE.set(test_case)
-    func_sig = inspect.signature(func)
-    func_inputs = rh._get_inputs_safe(func_sig, *test_args, **test_kwargs)
 
     def _test():
         test_case.start_time()
         with rh.trace(
             name=getattr(func, "__name__", "Test"),
             run_id=test_case.run_id,
             reference_example_id=test_case.example_id,
-            inputs=func_inputs,
+            inputs=test_case.inputs,
             project_name=test_case.test_suite.name,
             exceptions_to_handle=(SkipException,),
             _end_on_exit=False,
@@ -936,16 +936,14 @@ async def _arun_test(
         langtest_extra=langtest_extra,
     )
     _TEST_CASE.set(test_case)
-    func_sig = inspect.signature(func)
-    func_inputs = rh._get_inputs_safe(func_sig, *test_args, **test_kwargs)
 
     async def _test():
         test_case.start_time()
         with rh.trace(
             name=getattr(func, "__name__", "Test"),
             run_id=test_case.run_id,
             reference_example_id=test_case.example_id,
-            inputs=func_inputs,
+            inputs=test_case.inputs,
             project_name=test_case.test_suite.name,
             exceptions_to_handle=(SkipException,),
             _end_on_exit=False,

python/tests/integration_tests/test_client.py

@@ -30,7 +30,6 @@
     Example,
     ExampleUpdateWithAttachments,
     ExampleUploadWithAttachments,
-    ExampleUpsertWithAttachments,
     Run,
 )
 from langsmith.utils import (
@@ -511,97 +510,6 @@ def test_upload_examples_multipart(langchain_client: Client):
     langchain_client.delete_dataset(dataset_name=dataset_name)
 
 
-def test_upsert_examples_multipart(langchain_client: Client) -> None:
-    """Test upserting examples with attachments via the multipart endpoint."""
-    dataset_name = "__test_upsert_examples_multipart" + uuid4().hex[:4]
-    if langchain_client.has_dataset(dataset_name=dataset_name):
-        langchain_client.delete_dataset(dataset_name=dataset_name)
-
-    dataset = langchain_client.create_dataset(
-        dataset_name,
-        description="Test dataset for multipart example upload",
-        data_type=DataType.kv,
-    )
-
-    # Test example with all fields
-    example_id = uuid4()
-    example_1 = ExampleUpsertWithAttachments(
-        id=example_id,
-        dataset_id=dataset.id,
-        inputs={"text": "hello world"},
-        # test without outputs
-        attachments={
-            "test_file": ("text/plain", b"test content"),
-        },
-    )
-    # Test example without id
-    example_2 = ExampleUpsertWithAttachments(
-        dataset_id=dataset.id,
-        inputs={"text": "foo bar"},
-        outputs={"response": "baz"},
-        attachments={
-            "my_file": ("text/plain", b"more test content"),
-        },
-    )
-    created_examples = langchain_client.upsert_examples_multipart(
-        upserts=[example_1, example_2]
-    )
-    assert created_examples["count"] == 2
-
-    created_example_1 = langchain_client.read_example(example_id)
-    assert created_example_1.inputs["text"] == "hello world"
-    assert created_example_1.outputs is None
-
-    created_example_2 = langchain_client.read_example(
-        [id_ for id_ in created_examples["example_ids"] if id_ != str(example_id)][0]
-    )
-    assert created_example_2.inputs["text"] == "foo bar"
-    assert created_example_2.outputs["response"] == "baz"
-
-    # make sure examples were sent to the correct dataset
-    all_examples_in_dataset = [
-        example for example in langchain_client.list_examples(dataset_id=dataset.id)
-    ]
-    assert len(all_examples_in_dataset) == 2
-
-    example_1_update = ExampleUpsertWithAttachments(
-        id=example_id,
-        dataset_id=dataset.id,
-        inputs={"text": "bar baz"},
-        outputs={"response": "foo"},
-        attachments={
-            "my_file": ("text/plain", b"more test content"),
-        },
-    )
-    updated_examples = langchain_client.upsert_examples_multipart(
-        upserts=[example_1_update]
-    )
-    assert updated_examples["count"] == 0
-    # Test that adding invalid example fails
-    # even if valid examples are added alongside
-    example_3 = ExampleUpsertWithAttachments(
-        dataset_id=uuid4(),  # not a real dataset
-        inputs={"text": "foo bar"},
-        outputs={"response": "baz"},
-        attachments={
-            "my_file": ("text/plain", b"more test content"),
-        },
-    )
-
-    with pytest.raises(LangSmithNotFoundError):
-        langchain_client.upsert_examples_multipart(upserts=[example_3])
-
-    all_examples_in_dataset = [
-        example for example in langchain_client.list_examples(dataset_id=dataset.id)
-    ]
-    assert len(all_examples_in_dataset) == 2
-
-    # Throw type errors when not passing ExampleUpsertWithAttachments
-    with pytest.raises(ValueError):
-        langchain_client.upsert_examples_multipart(upserts=[{"foo": "bar"}])
-    langchain_client.delete_dataset(dataset_name=dataset_name)
-
-
 def test_create_dataset(langchain_client: Client) -> None:
     dataset_name = "__test_create_dataset" + uuid4().hex[:4]
     if langchain_client.has_dataset(dataset_name=dataset_name):

python/tests/unit_tests/test_run_helpers.py

@@ -1391,11 +1391,9 @@ def the_parent(inputs: dict) -> dict:
     patches_dict = {d["id"]: d for d in patches_datas}
     child_patch = patches_dict[ids["child"]]
     assert child_patch["outputs"] == expected_at_stage["child_output"]
-    assert child_patch["inputs"] == expected_at_stage["child_input"]
     assert child_patch["name"] == "child"
     parent_patch = patches_dict[ids["the_parent"]]
     assert parent_patch["outputs"] == expected_at_stage["parent_output"]
-    assert parent_patch["inputs"] == expected_at_stage["parent_input"]
     assert parent_patch["name"] == "the_parent"
     for d in patches_datas:
         if d["id"] in ids_contains_serialized: