python[minor]: pytest integration

python/docs/create_api_rst.py

@@ -1,4 +1,4 @@
 """Script for auto-generating api_reference.rst."""
 
 import importlib
 import inspect
@@ -108,18 +108,6 @@
                     else "Pydantic" if issubclass(type_, BaseModel) else "Regular"
                 )
             )
-            # if hasattr(type_, "__slots__"):
-            #     for func_name, func_type in inspect.getmembers(type_):
-            #         if inspect.isfunction(func_type):
-            #             functions.append(
-            #                 FunctionInfo(
-            #                     name=func_name,
-            #                     qualified_name=f"{namespace}.{name}.{func_name}",
-            #                     is_public=not func_name.startswith("_"),
-            #                     is_deprecated=".. deprecated::"
-            #                     in (func_type.__doc__ or ""),
-            #                 )
-            #             )
             classes_.append(
                 ClassInfo(
                     name=name,
@@ -156,7 +144,7 @@
             if file_path.name not in {
                 "_runner.py",
                 "_arunner.py",
-                "_testing.py",
+                "_internal.py",
                 "_expect.py",
                 "_openai.py",
             }:
@@ -200,6 +188,7 @@
     "utils",
     "anonymizer",
     "wrappers",
+    "testing",
 ]
 
 
@@ -387,20 +376,17 @@
 | [AsyncClient](async_client/langsmith.async_client.AsyncClient) | Asynchronous client for interacting with the LangSmith API. |
 | [traceable](run_helpers/langsmith.run_helpers.traceable) | Wrapper/decorator for tracing any function. |
 | [wrap_openai](wrappers/langsmith.wrappers._openai.wrap_openai) | Wrapper for OpenAI client, adds LangSmith tracing to all OpenAI calls. |
-| [evaluate](evaluation/langsmith.evaluation._runner.evaluate) | Evaluate an application on a dataset. |
-| [aevaluate](evaluation/langsmith.evaluation._arunner.aevaluate) | Asynchronously evaluate an application on a dataset. |
-| [unit](_testing/langsmith._testing.unit) | Create a LangSmith unit test. |
+| [@pytest.mark.langsmith](/testing/langsmith.testing._internal.test) | LangSmith pytest integration. |
 
 ```{{toctree}}
 :maxdepth: 2
 :hidden:
 
   client<client>
   async_client<async_client>
-  evaluation<evaluation>
   run_helpers<run_helpers>
   wrappers<wrappers>
-  _testing<_testing>
+  testing<testing>
 ``` 
 
 """

python/langsmith/__init__.py

@@ -5,7 +5,6 @@
 
 if TYPE_CHECKING:
     from langsmith._expect import expect
-    from langsmith._testing import test, unit
     from langsmith.async_client import AsyncClient
     from langsmith.client import Client
     from langsmith.evaluation import aevaluate, evaluate
@@ -18,6 +17,7 @@
         tracing_context,
     )
     from langsmith.run_trees import RunTree
+    from langsmith.testing._internal import test, unit
     from langsmith.utils import (
         ContextThreadPoolExecutor,
     )
@@ -63,7 +63,7 @@ def __getattr__(name: str) -> Any:
         return traceable
 
     elif name == "test":
-        from langsmith._testing import test
+        from langsmith.testing._internal import test
 
         return test
 
@@ -104,7 +104,7 @@ def __getattr__(name: str) -> Any:
         return get_current_run_tree
 
     elif name == "unit":
-        from langsmith._testing import unit
+        from langsmith.testing._internal import unit
 
         return unit
     elif name == "ContextThreadPoolExecutor":

python/langsmith/_expect.py

@@ -379,7 +379,7 @@ def value(self, value: Any) -> _Matcher:
 
     def score(
         self,
-        score: Union[float, int],
+        score: Union[float, int, bool],
         *,
         key: str = "score",
         source_run_id: Optional[ls_client.ID_TYPE] = None,

python/langsmith/_internal/_beta_decorator.py

@@ -8,14 +8,14 @@ class LangSmithBetaWarning(UserWarning):
 
 
 @functools.lru_cache(maxsize=100)
-def _warn_once(message: str) -> None:
-    warnings.warn(message, LangSmithBetaWarning, stacklevel=2)
+def _warn_once(message: str, stacklevel: int = 2) -> None:
+    warnings.warn(message, LangSmithBetaWarning, stacklevel=stacklevel)
 
 
 def warn_beta(func: Callable) -> Callable:
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
-        _warn_once(f"Function {func.__name__} is in beta.")
+        _warn_once(f"Function {func.__name__} is in beta.", stacklevel=3)
         return func(*args, **kwargs)
 
     return wrapper