python[minor]: pytest integration (#1362)

Log pytest tests to LangSmith. Useful for:
- Evaluations where the eval logic is different for each datapoint,
making it difficult to use generic evaluators on a whole dataset
- Unit tests where you want both the local pytest experience and the
ability to trace and share results

### Install
```bash
pip install "langsmith[pytest]==0.2.11rc14"
```

### Simple usage

```python
# tests/test_simple.py
import pytest
from langsmith import testing as t

@pytest.mark.langsmith
def test_addition_single():
    x = 3
    y = 4
    # directly log example inputs if you don't want to use fixtures
    t.log_inputs({"x": x, "y": y})

    expected = 7
    # directly log example outputs if you don't want to use fixtures
    t.log_reference_outputs({"sum": expected})

    actual = x + y
    # directly log run outputs
    t.log_outputs({"sum": actual})
    
    # test pass/fail automatically logged to langsmith
    assert actual == expected
```

Run
```bash

pytest --outputs='ls' tests/test_foo.py
```

Results

<img width="952" alt="Screenshot 2025-01-08 at 2 53 04 AM"
src="https://github.com/user-attachments/assets/d695747b-cfdf-4248-b5fd-c5c420aa92ec"
/>


### Advanced usage

```python
#tests/test_advanced.py
import openai
import pytest

from langsmith import wrappers
from langsmith import testing as t

oai_client = wrappers.wrap_openai(openai.Client())

@pytest.mark.langsmith
def test_openai_says_hello():
    # Traced code will be included in the test case
    text = "Say hello!"
    response = oai_client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": text},
        ],
    )
    t.log_inputs({"text": text})
    t.log_outputs({"response": response.choices[0].message.content})
    t.log_reference_outputs({"response": "hello!"})

    # Use this context manager to trace any steps used for generating evaluation 
    # feedback separately from the main application logic
    with t.trace_feedback():
        grade = oai_client.chat.completions.create(
            model="gpt-4o-mini",
            messages=[
                {
                    "role": "system",
                    "content": "Return 1 if 'hello' is in the user message and 0 otherwise.",
                },
                {"role": "user", "content": response.choices[0].message.content},
            ],
        )
        t.log_feedback(
            key="llm_judge", score=float(grade.choices[0].message.content)
        )

    assert "hello" in response.choices[0].message.content.lower()


@pytest.mark.langsmith(output_keys=["expected"])
@pytest.mark.parametrize(
    "a, b, expected",
    [(1, 2, 3), (3, 4, 7)],
)
def test_addition_parametrized(a: int, b: int, expected: int):
    t.log_outputs({"sum": a + b})
    assert a + b == expected

```

Run using pytest-xdist to parallelize (`pip install pytest-xdist first`)
```bash
LANGSMITH_TEST_SUITE="Test suite name" LANGSMITH_EXPERIMENT="Experiment name" pytest  --outputs='ls' tests
```

Results:
https://dev.smith.langchain.com/public/cea0e7fd-2d27-47d1-8ada-141069acdf0d/d

<img width="1030" alt="Screenshot 2025-01-08 at 3 07 51 AM"
src="https://github.com/user-attachments/assets/db770d00-67b3-4b53-8e33-289c4f0edfeb"
/>

---------

Co-authored-by: isaac hershenson <ihershenson@hmc.edu>

Author: baskaryan

python/docs/create_api_rst.py

@@ -108,18 +108,6 @@ def _load_module_members(module_path: str, namespace: str) -> ModuleMembers:
                     else "Pydantic" if issubclass(type_, BaseModel) else "Regular"
                 )
             )
-            # if hasattr(type_, "__slots__"):
-            #     for func_name, func_type in inspect.getmembers(type_):
-            #         if inspect.isfunction(func_type):
-            #             functions.append(
-            #                 FunctionInfo(
-            #                     name=func_name,
-            #                     qualified_name=f"{namespace}.{name}.{func_name}",
-            #                     is_public=not func_name.startswith("_"),
-            #                     is_deprecated=".. deprecated::"
-            #                     in (func_type.__doc__ or ""),
-            #                 )
-            #             )
             classes_.append(
                 ClassInfo(
                     name=name,
@@ -156,7 +144,7 @@ def _load_package_modules(
             if file_path.name not in {
                 "_runner.py",
                 "_arunner.py",
-                "_testing.py",
+                "_internal.py",
                 "_expect.py",
                 "_openai.py",
             }:
@@ -200,6 +188,7 @@ def _load_package_modules(
     "utils",
     "anonymizer",
     "wrappers",
+    "testing",
 ]
 
 
@@ -387,20 +376,17 @@ def _build_index(package_version: str) -> None:
 | [AsyncClient](async_client/langsmith.async_client.AsyncClient) | Asynchronous client for interacting with the LangSmith API. |
 | [traceable](run_helpers/langsmith.run_helpers.traceable) | Wrapper/decorator for tracing any function. |
 | [wrap_openai](wrappers/langsmith.wrappers._openai.wrap_openai) | Wrapper for OpenAI client, adds LangSmith tracing to all OpenAI calls. |
-| [evaluate](evaluation/langsmith.evaluation._runner.evaluate) | Evaluate an application on a dataset. |
-| [aevaluate](evaluation/langsmith.evaluation._arunner.aevaluate) | Asynchronously evaluate an application on a dataset. |
-| [unit](_testing/langsmith._testing.unit) | Create a LangSmith unit test. |
+| [@pytest.mark.langsmith](/testing/langsmith.testing._internal.test) | LangSmith pytest integration. |
 
 ```{{toctree}}
 :maxdepth: 2
 :hidden:
 
   client<client>
   async_client<async_client>
-  evaluation<evaluation>
   run_helpers<run_helpers>
   wrappers<wrappers>
-  _testing<_testing>
+  testing<testing>
 ``` 
 
 """

python/langsmith/__init__.py

@@ -5,7 +5,6 @@
 
 if TYPE_CHECKING:
     from langsmith._expect import expect
-    from langsmith._testing import test, unit
     from langsmith.async_client import AsyncClient
     from langsmith.client import Client
     from langsmith.evaluation import aevaluate, evaluate
@@ -18,6 +17,7 @@
         tracing_context,
     )
     from langsmith.run_trees import RunTree
+    from langsmith.testing._internal import test, unit
     from langsmith.utils import (
         ContextThreadPoolExecutor,
     )
@@ -63,7 +63,7 @@ def __getattr__(name: str) -> Any:
         return traceable
 
     elif name == "test":
-        from langsmith._testing import test
+        from langsmith.testing._internal import test
 
         return test
 
@@ -104,7 +104,7 @@ def __getattr__(name: str) -> Any:
         return get_current_run_tree
 
     elif name == "unit":
-        from langsmith._testing import unit
+        from langsmith.testing._internal import unit
 
         return unit
     elif name == "ContextThreadPoolExecutor":

python/langsmith/_expect.py

@@ -379,7 +379,7 @@ def value(self, value: Any) -> _Matcher:
 
     def score(
         self,
-        score: Union[float, int],
+        score: Union[float, int, bool],
         *,
         key: str = "score",
         source_run_id: Optional[ls_client.ID_TYPE] = None,

python/langsmith/_internal/_beta_decorator.py

@@ -8,14 +8,14 @@ class LangSmithBetaWarning(UserWarning):
 
 
 @functools.lru_cache(maxsize=100)
-def _warn_once(message: str) -> None:
-    warnings.warn(message, LangSmithBetaWarning, stacklevel=2)
+def _warn_once(message: str, stacklevel: int = 2) -> None:
+    warnings.warn(message, LangSmithBetaWarning, stacklevel=stacklevel)
 
 
 def warn_beta(func: Callable) -> Callable:
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
-        _warn_once(f"Function {func.__name__} is in beta.")
+        _warn_once(f"Function {func.__name__} is in beta.", stacklevel=3)
         return func(*args, **kwargs)
 
     return wrapper