From 6b29f61645133c2c0bbbb0cf3be68cd6ee2325ee Mon Sep 17 00:00:00 2001 From: James Braza Date: Wed, 11 Sep 2024 14:14:28 -0700 Subject: [PATCH] Added mark vcr decorator to a bunch of tests, with fixtures --- .../TestHTTPAgentClient.test_lifecycle.yaml | 713 ++++++++++++++++++ .../TestLLMModel.test_output_schema.yaml | 101 +++ ...mage_message[claude-3-haiku-20240307].yaml | 62 ++ ....test_text_image_message[gpt-4-turbo].yaml | 98 +++ ...image_message[gpt-4o-mini-2024-07-18].yaml | 98 +++ .../TestMemoryAgent.test_agent_grad.yaml | 107 +++ ...test_dummyenv[gpt-4o-mini-2024-07-18].yaml | 121 +++ ...CompletionLLMModel.test_output_schema.yaml | 102 +++ ...image_message[gpt-4o-mini-2024-07-18].yaml | 98 +++ ...t_agent_grad[claude-3-haiku-20240307].yaml | 75 ++ ...ActAgent.test_agent_grad[gpt-4-turbo].yaml | 118 +++ ...act_dummyenv[claude-3-haiku-20240307].yaml | 75 ++ ...gent.test_react_dummyenv[gpt-4-turbo].yaml | 118 +++ ...t_agent_grad[claude-3-haiku-20240307].yaml | 72 ++ ...st_agent_grad[gpt-4o-mini-2024-07-18].yaml | 113 +++ ...est_dummyenv[claude-3-haiku-20240307].yaml | 73 ++ ...test_dummyenv[gpt-4o-mini-2024-07-18].yaml | 113 +++ tests/cassettes/test_reflect_module.yaml | 67 ++ .../test_tools[claude-3-haiku-20240307].yaml | 263 +++++++ .../test_tools[gpt-4o-mini-2024-07-18].yaml | 102 +++ tests/test_agents.py | 28 +- tests/test_llms.py | 7 +- tests/test_modules.py | 1 + 23 files changed, 2701 insertions(+), 24 deletions(-) create mode 100644 tests/cassettes/TestHTTPAgentClient.test_lifecycle.yaml create mode 100644 tests/cassettes/TestLLMModel.test_output_schema.yaml create mode 100644 tests/cassettes/TestLLMModel.test_text_image_message[claude-3-haiku-20240307].yaml create mode 100644 tests/cassettes/TestLLMModel.test_text_image_message[gpt-4-turbo].yaml create mode 100644 tests/cassettes/TestLLMModel.test_text_image_message[gpt-4o-mini-2024-07-18].yaml create mode 100644 tests/cassettes/TestMemoryAgent.test_agent_grad.yaml create mode 100644 tests/cassettes/TestMemoryAgent.test_dummyenv[gpt-4o-mini-2024-07-18].yaml create mode 100644 tests/cassettes/TestMultipleCompletionLLMModel.test_output_schema.yaml create mode 100644 tests/cassettes/TestMultipleCompletionLLMModel.test_text_image_message[gpt-4o-mini-2024-07-18].yaml create mode 100644 tests/cassettes/TestReActAgent.test_agent_grad[claude-3-haiku-20240307].yaml create mode 100644 tests/cassettes/TestReActAgent.test_agent_grad[gpt-4-turbo].yaml create mode 100644 tests/cassettes/TestReActAgent.test_react_dummyenv[claude-3-haiku-20240307].yaml create mode 100644 tests/cassettes/TestReActAgent.test_react_dummyenv[gpt-4-turbo].yaml create mode 100644 tests/cassettes/TestSimpleAgent.test_agent_grad[claude-3-haiku-20240307].yaml create mode 100644 tests/cassettes/TestSimpleAgent.test_agent_grad[gpt-4o-mini-2024-07-18].yaml create mode 100644 tests/cassettes/TestSimpleAgent.test_dummyenv[claude-3-haiku-20240307].yaml create mode 100644 tests/cassettes/TestSimpleAgent.test_dummyenv[gpt-4o-mini-2024-07-18].yaml create mode 100644 tests/cassettes/test_reflect_module.yaml create mode 100644 tests/cassettes/test_tools[claude-3-haiku-20240307].yaml create mode 100644 tests/cassettes/test_tools[gpt-4o-mini-2024-07-18].yaml diff --git a/tests/cassettes/TestHTTPAgentClient.test_lifecycle.yaml b/tests/cassettes/TestHTTPAgentClient.test_lifecycle.yaml new file mode 100644 index 00000000..bb5b406e --- /dev/null +++ b/tests/cassettes/TestHTTPAgentClient.test_lifecycle.yaml @@ -0,0 +1,713 @@ +interactions: + - request: + body: "" + headers: + accept: + - "*/*" + accept-encoding: + - gzip, deflate + connection: + - keep-alive + host: + - testserver + user-agent: + - python-httpx/0.27.2 + method: GET + uri: http://testserver/info + response: + body: + string: '{"agent_type":"SimpleAgent"}' + headers: + content-length: + - "28" + content-type: + - application/json + status: + code: 200 + message: OK + - request: + body: + '[{"type": "function", "info": {"name": "print_story", "description": "Print + a story.", "parameters": {"type": "object", "properties": {"story": {"description": + "Story to print.", "title": "Story", "type": "string"}}, "required": ["story"]}}}, + {"type": "function", "info": {"name": "cast_float", "description": "Cast the + input argument x to a float.", "parameters": {"type": "object", "properties": + {"x": {"title": "X", "type": "string"}}, "required": ["x"]}}}, {"type": "function", + "info": {"name": "cast_int", "description": "Cast the input argument x to an + integer.", "parameters": {"type": "object", "properties": {"x": {"title": "X", + "type": "number"}}, "required": ["x"]}}}]' + headers: + accept: + - "*/*" + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "679" + content-type: + - application/json + host: + - testserver + user-agent: + - python-httpx/0.27.2 + method: POST + uri: http://testserver/init_state + response: + body: + string: + '{"tools":[{"type":"function","function":{"name":"print_story","description":"Print + a story.","parameters":{"type":"object","properties":{"story":{"description":"Story + to print.","title":"Story","type":"string"}},"required":["story"]}}},{"type":"function","function":{"name":"cast_float","description":"Cast + the input argument x to a float.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"string"}},"required":["x"]}}},{"type":"function","function":{"name":"cast_int","description":"Cast + the input argument x to an integer.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"number"}},"required":["x"]}}}],"messages":[]}' + headers: + content-length: + - "660" + content-type: + - application/json + status: + code: 200 + message: OK + - request: + body: + '{"messages": [{"role": "user", "content": "Cast ''5.5'' to a float, then + to an integer, and finally use it to write a story of that many words."}], "model": + "gpt-4o-2024-08-06", "temperature": 0.1, "tool_choice": "required", "tools": + [{"type": "function", "function": {"name": "print_story", "description": "Print + a story.", "parameters": {"type": "object", "properties": {"story": {"description": + "Story to print.", "title": "Story", "type": "string"}}, "required": ["story"]}}}, + {"type": "function", "function": {"name": "cast_float", "description": "Cast + the input argument x to a float.", "parameters": {"type": "object", "properties": + {"x": {"title": "X", "type": "string"}}, "required": ["x"]}}}, {"type": "function", + "function": {"name": "cast_int", "description": "Cast the input argument x to + an integer.", "parameters": {"type": "object", "properties": {"x": {"title": + "X", "type": "number"}}, "required": ["x"]}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "924" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.44.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.44.1 + x-stainless-raw-response: + - "true" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//bFJNT+MwFLznV1jv3KCkH0mbGxII7QFBQWJXbFeR676k7jq2Zb8Apep/ + X+WDNlTrQ2TNeN5Mxj4EjIHcQMZAbDmJyqrwOnlMXzbPizdM+U96TXd3u5v3ZPlw93SPaxg1CrPe + oaAv1ZUwlVVI0uiOFg45YTM1TsdJNF/MF5OWqMwGVSMrLYVTE46j8TSM5mGU9MKtkQI9ZOx3wBhj + h/bbRNQb/ICMRaMvpELveYmQnQ4xBs6oBgHuvfTENcHoTAqjCXWTWtdKDQgyRuWCK3U27tZhsD/3 + xJXKH2/v98tlKqrSffpb9fTj+fN1IfmvgV83em/bQEWtxamfAX/CswszxkDzCjtDT3mhDKcLNWPA + XVlXqKlJDocVfKwgW8HsaraCI3w7fAz+t/8zqMFhUXuu+n56/HgqXJnSOrP2F/1BIbX029wh9+1/ + gCdjO+/Gp3WA+ttdgXWmspST+Yu6GRhP4m4enJ/SgJ32JBniaojPgj4h+L0nrPJC6hKddbK9Zyhs + Psc4TtP1JEogOAb/AAAA//8DAM1umOvwAgAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1aba2648fb176d-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:24:53 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=nGfgJdJZ_pAgme6BLaXdUvXt5eF_Va57hdGwO2EI0JM-1726089893-1.0.1.1-aiXuwj2qv5KPr7P1RKhnQdhPclQrNT8pJczWF7UO4GQEiUofEGZLUeMsjNSaqjn67bVPwISKBhXquEGSKNsz7Q; + path=/; expires=Wed, 11-Sep-24 21:54:53 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=Kgse9sWf7N1FmtSw9gZumAxV5yE3Fh0s1UZBElBJ4Aw-1726089893326-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "252" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=15552000; includeSubDomains; preload + x-ratelimit-limit-requests: + - "10000" + x-ratelimit-limit-tokens: + - "30000000" + x-ratelimit-remaining-requests: + - "9999" + x-ratelimit-remaining-tokens: + - "29999958" + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_d244f47ad44e97122336c8d2683ec3f8 + status: + code: 200 + message: OK + - request: + body: + '{"agent_state": {"tools": [{"type": "function", "info": {"name": "print_story", + "description": "Print a story.", "parameters": {"type": "object", "properties": + {"story": {"description": "Story to print.", "title": "Story", "type": "string"}}, + "required": ["story"]}}}, {"type": "function", "info": {"name": "cast_float", + "description": "Cast the input argument x to a float.", "parameters": {"type": + "object", "properties": {"x": {"title": "X", "type": "string"}}, "required": + ["x"]}}}, {"type": "function", "info": {"name": "cast_int", "description": "Cast + the input argument x to an integer.", "parameters": {"type": "object", "properties": + {"x": {"title": "X", "type": "number"}}, "required": ["x"]}}}], "messages": + []}, "obs": [{"role": "user", "content": "Cast ''5.5'' to a float, then to an + integer, and finally use it to write a story of that many words."}], "training": + false}' + headers: + accept: + - "*/*" + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "882" + content-type: + - application/json + host: + - testserver + user-agent: + - python-httpx/0.27.2 + method: POST + uri: http://testserver/get_asv + response: + body: + string: + '[{"call_id":{"run_id":"acba9f93-749f-444e-aafa-f2d3f42e3035","fwd_id":"de61b0ce-b246-4323-b7ae-689ee49321f1"},"op_name":"_llm_call_op","op_class_name":"ldp.graph.common_ops.LLMCallOp","value":{"role":"assistant","content":null,"function_call":null,"tool_calls":[{"id":"call_PEMyQQ7cmgrzsElRISzZ9iaX","type":"function","function":{"arguments":"{\"x\": + \"5.5\"}","name":"cast_float"}}]}},{"tools":[{"type":"function","function":{"name":"print_story","description":"Print + a story.","parameters":{"type":"object","properties":{"story":{"description":"Story + to print.","title":"Story","type":"string"}},"required":["story"]}}},{"type":"function","function":{"name":"cast_float","description":"Cast + the input argument x to a float.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"string"}},"required":["x"]}}},{"type":"function","function":{"name":"cast_int","description":"Cast + the input argument x to an integer.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"number"}},"required":["x"]}}}],"messages":[{"role":"user","content":"Cast + ''5.5'' to a float, then to an integer, and finally use it to write a story + of that many words."},{"role":"assistant","content":null,"function_call":null,"tool_calls":[{"id":"call_PEMyQQ7cmgrzsElRISzZ9iaX","type":"function","function":{"arguments":"{\"x\": + \"5.5\"}","name":"cast_float"}}]}]},0.0]' + headers: + content-length: + - "1370" + content-type: + - application/json + status: + code: 200 + message: OK + - request: + body: + '{"messages": [{"role": "user", "content": "Cast ''5.5'' to a float, then + to an integer, and finally use it to write a story of that many words."}], "model": + "gpt-4o-2024-08-06", "temperature": 0.1, "tool_choice": "required", "tools": + [{"type": "function", "function": {"name": "print_story", "description": "Print + a story.", "parameters": {"type": "object", "properties": {"story": {"description": + "Story to print.", "title": "Story", "type": "string"}}, "required": ["story"]}}}, + {"type": "function", "function": {"name": "cast_float", "description": "Cast + the input argument x to a float.", "parameters": {"type": "object", "properties": + {"x": {"title": "X", "type": "string"}}, "required": ["x"]}}}, {"type": "function", + "function": {"name": "cast_int", "description": "Cast the input argument x to + an integer.", "parameters": {"type": "object", "properties": {"x": {"title": + "X", "type": "number"}}, "required": ["x"]}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "924" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.44.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.44.1 + x-stainless-raw-response: + - "true" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA2xSXWvbMBR9968Q9zkutuOkjt9SOli3BTrYCu0yjCJfO25kXU2SWULIfy/+aOKG + 6sGIc3TuOT7S0WMMqhxSBmLLnai19Jfzx9un3V4+5D9WStzFX0zz63mVzOnl97clTFoFbV5RuHfV + jaBaS3QVqZ4WBrnDdmp4G82DZJEsph1RU46ylZXa+TH5URDFfpD4wXwQbqkSaCFlfzzGGDt23zai + ynEPKQsm70iN1vISIT0fYgwMyRYBbm1lHVcOJhdSkHKo2tSqkXJEOCKZCS7lxbhfx9H+0hOXMnt8 + +nrYheHdIvo32+ffEcMlrv7f/xz59aMPugtUNEqc+xnxZzy9MmMMFK+xN7QuKyRxd6VmDLgpmxqV + a5PDcQ37NaRrmN3M1nCCD4dP3mf7v6MaDBaN5XLoZ8BP58IlldrQxl71B0WlKrvNDHLb/QdYR7r3 + bn06B2g+3BVoQ7V2maMdqnZgOA37eXB5SiM2HkhHjssxPvOGhGAP1mGdFZUq0WhTdfcMhc7CeLrZ + JPEiEuCdvDcAAAD//wMAmBfOVfACAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1aba2a1cbc176d-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:24:53 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "271" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=15552000; includeSubDomains; preload + x-ratelimit-limit-requests: + - "10000" + x-ratelimit-limit-tokens: + - "30000000" + x-ratelimit-remaining-requests: + - "9999" + x-ratelimit-remaining-tokens: + - "29999958" + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_9728da4b9ff43ba096f41d51f5693669 + status: + code: 200 + message: OK + - request: + body: + '{"agent_state": {"tools": [{"type": "function", "info": {"name": "print_story", + "description": "Print a story.", "parameters": {"type": "object", "properties": + {"story": {"description": "Story to print.", "title": "Story", "type": "string"}}, + "required": ["story"]}}}, {"type": "function", "info": {"name": "cast_float", + "description": "Cast the input argument x to a float.", "parameters": {"type": + "object", "properties": {"x": {"title": "X", "type": "string"}}, "required": + ["x"]}}}, {"type": "function", "info": {"name": "cast_int", "description": "Cast + the input argument x to an integer.", "parameters": {"type": "object", "properties": + {"x": {"title": "X", "type": "number"}}, "required": ["x"]}}}], "messages": + []}, "obs": [{"role": "user", "content": "Cast ''5.5'' to a float, then to an + integer, and finally use it to write a story of that many words."}], "training": + false}' + headers: + accept: + - "*/*" + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "882" + content-type: + - application/json + host: + - testserver + user-agent: + - python-httpx/0.27.2 + method: POST + uri: http://testserver/get_asv + response: + body: + string: + '[{"call_id":{"run_id":"937b3bc0-d39f-44f5-ae3a-ac33bbf9a8f8","fwd_id":"1324e9d6-12c8-441a-8b30-dc11fd8766a1"},"op_name":"_llm_call_op","op_class_name":"ldp.graph.common_ops.LLMCallOp","value":{"role":"assistant","content":null,"function_call":null,"tool_calls":[{"id":"call_PVHyk11B92q5xdKee1AeMwDQ","type":"function","function":{"arguments":"{\"x\": + \"5.5\"}","name":"cast_float"}}]}},{"tools":[{"type":"function","function":{"name":"print_story","description":"Print + a story.","parameters":{"type":"object","properties":{"story":{"description":"Story + to print.","title":"Story","type":"string"}},"required":["story"]}}},{"type":"function","function":{"name":"cast_float","description":"Cast + the input argument x to a float.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"string"}},"required":["x"]}}},{"type":"function","function":{"name":"cast_int","description":"Cast + the input argument x to an integer.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"number"}},"required":["x"]}}}],"messages":[{"role":"user","content":"Cast + ''5.5'' to a float, then to an integer, and finally use it to write a story + of that many words."},{"role":"assistant","content":null,"function_call":null,"tool_calls":[{"id":"call_PVHyk11B92q5xdKee1AeMwDQ","type":"function","function":{"arguments":"{\"x\": + \"5.5\"}","name":"cast_float"}}]}]},0.0]' + headers: + content-length: + - "1370" + content-type: + - application/json + status: + code: 200 + message: OK + - request: + body: + '{"messages": [{"role": "user", "content": "Cast ''5.5'' to a float, then + to an integer, and finally use it to write a story of that many words."}], "model": + "gpt-4o-2024-08-06", "temperature": 0.1, "tool_choice": "required", "tools": + [{"type": "function", "function": {"name": "print_story", "description": "Print + a story.", "parameters": {"type": "object", "properties": {"story": {"description": + "Story to print.", "title": "Story", "type": "string"}}, "required": ["story"]}}}, + {"type": "function", "function": {"name": "cast_float", "description": "Cast + the input argument x to a float.", "parameters": {"type": "object", "properties": + {"x": {"title": "X", "type": "string"}}, "required": ["x"]}}}, {"type": "function", + "function": {"name": "cast_int", "description": "Cast the input argument x to + an integer.", "parameters": {"type": "object", "properties": {"x": {"title": + "X", "type": "number"}}, "required": ["x"]}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "924" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.44.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.44.1 + x-stainless-raw-response: + - "true" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//bFJNb6MwFLzzK6x3DhXQfHKr2kNXarM9bbttVsiYB6Fr/FzbqImi/PeV + gSY0Wh+QNeN5M4x9CBiDuoCUgdhyJxotw5v50+J59sNE610T/Xx+en1ZF3p/197er37fwsQrKH9H + 4b5UV4IaLdHVpHpaGOQO/dR4kcyj5Wq5mnZEQwVKL6u0C6cUJlEyDaNlGM0H4ZZqgRZS9hYwxtih + +/qIqsAdpCyafCENWssrhPR0iDEwJD0C3NraOq4cTM6kIOVQ+dSqlXJEOCKZCS7l2bhfh9H+3BOX + MvtoP4jfV4WO39d58vqZV2b7+JD8Gvn1o/e6C1S2Spz6GfEnPL0wYwwUb7A3tC4rJXF3oWYMuKna + BpXzyeGwgd0G0g3MrmYbOMK3w8fgf/s/oxoMlq3lcuhnwI+nwiVV2lBuL/qDsla13WYGue3+A6wj + 3Xt7n84B2m93BdpQo13m6C8qPzC+jvt5cH5KI3Y6kI4cl2N8FgwJwe6twyYra1Wh0abu7hlKnS0x + jheL/DqaQ3AM/gEAAP//AwD5mgcs8AIAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1aba2d1f6a176d-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:24:54 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "345" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=15552000; includeSubDomains; preload + x-ratelimit-limit-requests: + - "10000" + x-ratelimit-limit-tokens: + - "30000000" + x-ratelimit-remaining-requests: + - "9999" + x-ratelimit-remaining-tokens: + - "29999958" + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_f9352ae0515ce462949f20bf757c6f4f + status: + code: 200 + message: OK + - request: + body: + '{"agent_state": {"tools": [{"type": "function", "info": {"name": "print_story", + "description": "Print a story.", "parameters": {"type": "object", "properties": + {"story": {"description": "Story to print.", "title": "Story", "type": "string"}}, + "required": ["story"]}}}, {"type": "function", "info": {"name": "cast_float", + "description": "Cast the input argument x to a float.", "parameters": {"type": + "object", "properties": {"x": {"title": "X", "type": "string"}}, "required": + ["x"]}}}, {"type": "function", "info": {"name": "cast_int", "description": "Cast + the input argument x to an integer.", "parameters": {"type": "object", "properties": + {"x": {"title": "X", "type": "number"}}, "required": ["x"]}}}], "messages": + []}, "obs": [{"role": "user", "content": "Cast ''5.5'' to a float, then to an + integer, and finally use it to write a story of that many words."}], "training": + false}' + headers: + accept: + - "*/*" + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "882" + content-type: + - application/json + host: + - testserver + user-agent: + - python-httpx/0.27.2 + method: POST + uri: http://testserver/get_asv + response: + body: + string: + '[{"call_id":{"run_id":"055641df-3d69-4714-931b-0e09fca5b501","fwd_id":"506cf576-8260-4da3-84eb-ce6d92a8d866"},"op_name":"_llm_call_op","op_class_name":"ldp.graph.common_ops.LLMCallOp","value":{"role":"assistant","content":null,"function_call":null,"tool_calls":[{"id":"call_quqoaHgdp1jNb2ZwbgrhML2V","type":"function","function":{"arguments":"{\"x\": + \"5.5\"}","name":"cast_float"}}]}},{"tools":[{"type":"function","function":{"name":"print_story","description":"Print + a story.","parameters":{"type":"object","properties":{"story":{"description":"Story + to print.","title":"Story","type":"string"}},"required":["story"]}}},{"type":"function","function":{"name":"cast_float","description":"Cast + the input argument x to a float.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"string"}},"required":["x"]}}},{"type":"function","function":{"name":"cast_int","description":"Cast + the input argument x to an integer.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"number"}},"required":["x"]}}}],"messages":[{"role":"user","content":"Cast + ''5.5'' to a float, then to an integer, and finally use it to write a story + of that many words."},{"role":"assistant","content":null,"function_call":null,"tool_calls":[{"id":"call_quqoaHgdp1jNb2ZwbgrhML2V","type":"function","function":{"arguments":"{\"x\": + \"5.5\"}","name":"cast_float"}}]}]},0.0]' + headers: + content-length: + - "1370" + content-type: + - application/json + status: + code: 200 + message: OK + - request: + body: + '{"messages": [{"role": "user", "content": "Cast ''5.5'' to a float, then + to an integer, and finally use it to write a story of that many words."}, {"role": + "assistant", "content": null, "function_call": null, "tool_calls": [{"id": "call_quqoaHgdp1jNb2ZwbgrhML2V", + "type": "function", "function": {"arguments": "{\"x\": \"5.5\"}", "name": "cast_float"}}]}, + {"role": "tool", "content": "5.5", "name": "cast_float", "tool_call_id": "call_quqoaHgdp1jNb2ZwbgrhML2V"}], + "model": "gpt-4o-2024-08-06", "temperature": 0.1, "tool_choice": "required", + "tools": [{"type": "function", "function": {"name": "print_story", "description": + "Print a story.", "parameters": {"type": "object", "properties": {"story": {"description": + "Story to print.", "title": "Story", "type": "string"}}, "required": ["story"]}}}, + {"type": "function", "function": {"name": "cast_float", "description": "Cast + the input argument x to a float.", "parameters": {"type": "object", "properties": + {"x": {"title": "X", "type": "string"}}, "required": ["x"]}}}, {"type": "function", + "function": {"name": "cast_int", "description": "Cast the input argument x to + an integer.", "parameters": {"type": "object", "properties": {"x": {"title": + "X", "type": "number"}}, "required": ["x"]}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "1240" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.44.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.44.1 + x-stainless-raw-response: + - "true" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA2xS0WrjMBB891eIfY6L7XMSx29XelAoCQfpUY62GFmRHaWypEprmhDy74dsJ3HK + 6UGImd2d0UjHgBAQG8gJsC1F1hgZ/pz9nr8s1m7l9suD/rN+XK3+xs/7l8+sLiVMfIcud5zhueuO + 6cZIjkKrnmaWU+R+ajxPZlG2yBZpRzR6w6Vvqw2GqQ6TKEnDKAuj2dC41YJxBzl5DQgh5Njt3qLa + 8D3kJJqckYY7R2sO+aWIELBaegSoc8IhVQiTK8m0Qq68a9VKOSJQa1kwKuVVuF/H0fmaE5WySJhL + vpZrfGjTnaGz7dNz+ev+ax6P9PrRB9MZqlrFLvmM+AuefxMjBBRteC/osBA3dxkqqK3bhiv0vuH4 + Bvs3yKd30xPcFJ6C/53fRwFYXrWOyiGZAT9dopa6NlaX7ltyUAkl3LawnLruBuBQm17b63QK0N68 + EhirG4MF6g+u/MB4mvXz4PqJRmw6kKiRyhE+T4LBIbiDQ94UlVA1t8aK7oWhMkWc/ijLLF0kDIJT + 8A8AAP//AwB101uF6gIAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1aba3029e5176d-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:24:54 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "264" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=15552000; includeSubDomains; preload + x-ratelimit-limit-requests: + - "10000" + x-ratelimit-limit-tokens: + - "30000000" + x-ratelimit-remaining-requests: + - "9999" + x-ratelimit-remaining-tokens: + - "29999954" + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_868539a9c38578e65ef1d51e7f9df4fb + status: + code: 200 + message: OK + - request: + body: + '{"agent_state": {"tools": [{"type": "function", "info": {"name": "print_story", + "description": "Print a story.", "parameters": {"type": "object", "properties": + {"story": {"description": "Story to print.", "title": "Story", "type": "string"}}, + "required": ["story"]}}}, {"type": "function", "info": {"name": "cast_float", + "description": "Cast the input argument x to a float.", "parameters": {"type": + "object", "properties": {"x": {"title": "X", "type": "string"}}, "required": + ["x"]}}}, {"type": "function", "info": {"name": "cast_int", "description": "Cast + the input argument x to an integer.", "parameters": {"type": "object", "properties": + {"x": {"title": "X", "type": "number"}}, "required": ["x"]}}}], "messages": + [{"role": "user", "content": "Cast ''5.5'' to a float, then to an integer, and + finally use it to write a story of that many words."}, {"role": "assistant", + "content": null, "function_call": null, "tool_calls": [{"id": "call_quqoaHgdp1jNb2ZwbgrhML2V", + "type": "function", "function": {"arguments": "{\"x\": \"5.5\"}", "name": "cast_float"}}]}]}, + "obs": [{"role": "tool", "content": "5.5", "name": "cast_float", "tool_call_id": + "call_quqoaHgdp1jNb2ZwbgrhML2V"}], "training": false}' + headers: + accept: + - "*/*" + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "1196" + content-type: + - application/json + host: + - testserver + user-agent: + - python-httpx/0.27.2 + method: POST + uri: http://testserver/get_asv + response: + body: + string: + '[{"call_id":{"run_id":"5b25839e-2950-495c-acb8-fb71249e2d05","fwd_id":"e0b1a912-9551-4f5d-b7ee-157883d8fbc4"},"op_name":"_llm_call_op","op_class_name":"ldp.graph.common_ops.LLMCallOp","value":{"role":"assistant","content":null,"function_call":null,"tool_calls":[{"id":"call_2cs2wMStDu4jpa6hKTbEBw71","type":"function","function":{"arguments":"{\"x\": + 5.5}","name":"cast_int"}}]}},{"tools":[{"type":"function","function":{"name":"print_story","description":"Print + a story.","parameters":{"type":"object","properties":{"story":{"description":"Story + to print.","title":"Story","type":"string"}},"required":["story"]}}},{"type":"function","function":{"name":"cast_float","description":"Cast + the input argument x to a float.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"string"}},"required":["x"]}}},{"type":"function","function":{"name":"cast_int","description":"Cast + the input argument x to an integer.","parameters":{"type":"object","properties":{"x":{"title":"X","type":"number"}},"required":["x"]}}}],"messages":[{"role":"user","content":"Cast + ''5.5'' to a float, then to an integer, and finally use it to write a story + of that many words."},{"role":"assistant","content":null,"function_call":null,"tool_calls":[{"id":"call_quqoaHgdp1jNb2ZwbgrhML2V","type":"function","function":{"arguments":"{\"x\": + \"5.5\"}","name":"cast_float"}}]},{"role":"tool","content":"5.5","name":"cast_float","tool_call_id":"call_quqoaHgdp1jNb2ZwbgrhML2V"},{"role":"assistant","content":null,"function_call":null,"tool_calls":[{"id":"call_2cs2wMStDu4jpa6hKTbEBw71","type":"function","function":{"arguments":"{\"x\": + 5.5}","name":"cast_int"}}]}]},0.0]' + headers: + content-length: + - "1650" + content-type: + - application/json + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/TestLLMModel.test_output_schema.yaml b/tests/cassettes/TestLLMModel.test_output_schema.yaml new file mode 100644 index 00000000..95d87ba6 --- /dev/null +++ b/tests/cassettes/TestLLMModel.test_output_schema.yaml @@ -0,0 +1,101 @@ +interactions: + - request: + body: + '{"messages": [{"role": "system", "content": "Respond following this JSON + schema:\n\n{\n \"properties\": {\n \"name\": {\n \"title\": \"Name\",\n \"type\": + \"string\"\n },\n \"age\": {\n \"title\": \"Age\",\n \"type\": + \"integer\"\n }\n },\n \"required\": [\n \"name\",\n \"age\"\n ],\n \"title\": + \"DummyOutputSchema\",\n \"type\": \"object\"\n}"}, {"role": "user", "content": + "My name is Claude and I am 1 year old. What is my name and age?"}], "model": + "gpt-3.5-turbo", "response_format": {"type": "json_object"}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "559" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.44.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.44.1 + x-stainless-raw-response: + - "true" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA1SQzW7CMBCE73mK1Z4TREghIbeqVXurekFV21TIJEswdWzLdgQI8e6VTfjpxYf5 + POMZHyMA5A2WgPWGubrTInmcvRfZ4evjk78U+vU5e9i+LXb73aIxOcsx9g612lLtLq5RrTotyHEl + z7g2xBz51DSfzMbFfD7JA+hUQ8LbWu2SbDRNXG9WKhmnk+ng3Chek8USviMAgGM4fUfZ0B5LGMcX + pSNrWUtYXi8BoFHCK8is5dYx6TC+wVpJRzLUPlYSoELJOqqwhAqfBOsbqjA+A9YGPa3k6T7B0Lq3 + zC+QvRCDfrpWEqrVRq3swK/6mktuN0tDzCrpn7dOaQz0FAH8hOn9vzWojeq0Wzr1S9IHpmlxzsPb + b9/RfIBOOSbu9GwaDQ3RHqyjbrnmsiWjDQ8/EXacoj8AAAD//wMANu8s3AgCAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1abafcea3c171a-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:25:27 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "267" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=15552000; includeSubDomains; preload + x-ratelimit-limit-requests: + - "12000" + x-ratelimit-limit-tokens: + - "1000000" + x-ratelimit-remaining-requests: + - "11999" + x-ratelimit-remaining-tokens: + - "999838" + x-ratelimit-reset-requests: + - 5ms + x-ratelimit-reset-tokens: + - 9ms + x-request-id: + - req_65355d90c1cb42732d3c85abb2413f94 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/TestLLMModel.test_text_image_message[claude-3-haiku-20240307].yaml b/tests/cassettes/TestLLMModel.test_text_image_message[claude-3-haiku-20240307].yaml new file mode 100644 index 00000000..9897ebd3 --- /dev/null +++ b/tests/cassettes/TestLLMModel.test_text_image_message[claude-3-haiku-20240307].yaml @@ -0,0 +1,62 @@ +interactions: + - request: + body: + '{"messages": [{"role": "user", "content": [{"type": "image", "source": + {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAAKElEQVR4nO3NMQEAAAjDMMC/ZzDBvlRA01vZJvwHAAAAAAAAAAAAbx2jxAE/i2AjOgAAAABJRU5ErkJggg=="}}, + {"type": "text", "text": "What color is this square? Respond only with the color + name."}]}], "max_tokens": 4096, "model": "claude-3-haiku-20240307"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - "2023-06-01" + connection: + - keep-alive + content-length: + - "411" + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - litellm/1.44.24 + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA0yOT2vCQBDFv8s7byBNBOlehV5KRUS9SAlLdqrBdTZmZool5LuXiIWeHvzeH96I + LsLjKqemfNnFj3U4L982h9V2d/ve1/b+RQMc9KenOUUi4URwGHKaQRDpRAMrHK45UoJHm4JFKuri + HLqLFVVZLcq6XMKhzazECn8c/xaV7nP3IR5bipg+HURz3wwUJDM8iGOjNjCehtDNiFuCZ0vJwR6f + /IiOe9NG84VY4KtXh2z6Hy2m6RcAAP//AwB6WGlZ8AAAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1aba3c6c959664-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:24:56 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + request-id: + - req_01HzTrvjM4ny4Fa5TnY8n8pE + via: + - 1.1 google + x-cloud-trace-context: + - 4ab7e744703e1034c4b84f8d53549416 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/TestLLMModel.test_text_image_message[gpt-4-turbo].yaml b/tests/cassettes/TestLLMModel.test_text_image_message[gpt-4-turbo].yaml new file mode 100644 index 00000000..55c8ae2c --- /dev/null +++ b/tests/cassettes/TestLLMModel.test_text_image_message[gpt-4-turbo].yaml @@ -0,0 +1,98 @@ +interactions: + - request: + body: + '{"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": + {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAAKElEQVR4nO3NMQEAAAjDMMC/ZzDBvlRA01vZJvwHAAAAAAAAAAAAbx2jxAE/i2AjOgAAAABJRU5ErkJggg=="}}, + {"type": "text", "text": "What color is this square? Respond only with the color + name."}]}], "model": "gpt-4-turbo"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "362" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.44.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.44.1 + x-stainless-raw-response: + - "true" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA1SQS0/DMBCE7/kV1p4blIT0kdyQSrkgBEVwQShykk3i4tiWvUGgqv8dOU0fXPYw + n2d21vuAMRA15AyqjlPVGxneLZ6X5dP29n39+F12r5t487LbZe6eHtbxG8y8Q5c7rOjkuql0bySS + 0OqIK4uc0KfGy2QRrbJVlo2g1zVKb2sNhWlIgy11mERJGkZpGGWTu9OiQgc5+wgYY2w/Tt9T1fgD + OYtmJ6VH53iLkJ8fMQZWS68Ad0444opgdoGVVoRqrL7F+ppYbAbHfTs1SDnph/MqqVtjdekmftYb + oYTrCovcaeVjHWkDIz0EjH2OJw3/WoKxujdUkP5C5QOT5fyYB5efvNB4YqSJy2vTIpgKgvt1hH3R + CNWiNVYcD2xMkSXzuEnnNW8gOAR/AAAA//8DANYbuh/uAQAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1aba4488fdd035-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:25:00 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "1891" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=15552000; includeSubDomains; preload + x-ratelimit-limit-requests: + - "10000" + x-ratelimit-limit-tokens: + - "2000000" + x-ratelimit-remaining-requests: + - "9999" + x-ratelimit-remaining-tokens: + - "1999203" + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 23ms + x-request-id: + - req_70379d0b10dbfd23826f5319eb7c48f4 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/TestLLMModel.test_text_image_message[gpt-4o-mini-2024-07-18].yaml b/tests/cassettes/TestLLMModel.test_text_image_message[gpt-4o-mini-2024-07-18].yaml new file mode 100644 index 00000000..d73d01a6 --- /dev/null +++ b/tests/cassettes/TestLLMModel.test_text_image_message[gpt-4o-mini-2024-07-18].yaml @@ -0,0 +1,98 @@ +interactions: + - request: + body: + '{"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": + {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAAKElEQVR4nO3NMQEAAAjDMMC/ZzDBvlRA01vZJvwHAAAAAAAAAAAAbx2jxAE/i2AjOgAAAABJRU5ErkJggg=="}}, + {"type": "text", "text": "What color is this square? Respond only with the color + name."}]}], "model": "gpt-4o-mini-2024-07-18"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "373" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.44.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.44.1 + x-stainless-raw-response: + - "true" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA1SQP2/CMBTE93wKyzOpkjRAyNaBpVNF/wytqshxXoKL7WfZD9QK8d0rhxTK4uF+ + vvOdjwljXHW8ZlxuBUnjdPqweFoKef+46iQ9Z6+bl3YXDsq9rQ/v+ZrPogPbL5D057qTaJwGUmjP + WHoQBDE1XxaLrFpVq2oEBjvQ0TY4SktMjbIqLbKiTLNlmleTe4tKQuA1+0gYY+w4nrGn7eCb1yyb + /SkGQhAD8PpyiTHuUUeFixBUIGGJz65QoiWwY/UNdP+Jh34fRGxn91pP+unylMbBeWzDxC96r6wK + 28aDCGhjbCB0fKSnhLHPcdL+piV3Ho2jhnAHNgZW82LaxK9fecX5xAhJ6BtXnkwVefgJBKbplR3A + O6/OE3vXzEso+rIqW+DJKfkFAAD//wMAfqvd//ABAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1aba436f88aaa6-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:24:58 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "895" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=15552000; includeSubDomains; preload + x-ratelimit-limit-requests: + - "30000" + x-ratelimit-limit-tokens: + - "150000000" + x-ratelimit-remaining-requests: + - "29999" + x-ratelimit-remaining-tokens: + - "149999202" + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_93eab71f1f2d07d3a7cb0fbe4fe21df1 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/TestMemoryAgent.test_agent_grad.yaml b/tests/cassettes/TestMemoryAgent.test_agent_grad.yaml new file mode 100644 index 00000000..b5605916 --- /dev/null +++ b/tests/cassettes/TestMemoryAgent.test_agent_grad.yaml @@ -0,0 +1,107 @@ +interactions: + - request: + body: + '{"messages": [{"role": "user", "content": "Write a 5 word story via print_story"}], + "model": "gpt-4o-2024-08-06", "temperature": 0.1, "tool_choice": "required", + "tools": [{"type": "function", "function": {"name": "print_story", "description": + "Print a story.", "parameters": {"type": "object", "properties": {"story": {"description": + "Story to print.", "title": "Story", "type": "string"}}, "required": ["story"]}}}, + {"type": "function", "function": {"name": "cast_float", "description": "Cast + the input argument x to a float.", "parameters": {"type": "object", "properties": + {"x": {"title": "X", "type": "string"}}, "required": ["x"]}}}, {"type": "function", + "function": {"name": "cast_int", "description": "Cast the input argument x to + an integer.", "parameters": {"type": "object", "properties": {"x": {"title": + "X", "type": "number"}}, "required": ["x"]}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "862" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.44.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.44.1 + x-stainless-raw-response: + - "true" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//bFLLbptAFN3zFaO7NhEQ5GB2abJIValJlVZtU1doGA8w8bw6c0mLLf97 + xSM2scICje6Z8+Bc9gEhIDaQE2ANRaasDK+XD1ffl1RnP9SXP3efnm825f3t14fPP7v7eAuLnmHK + Z87wlXXBjLKSozB6hJnjFHmvGl8lyyhbZat0AJTZcNnTaothasIkStIwysJoOREbIxj3kJNfASGE + 7Id3H1Fv+D/ISbR4nSjuPa055MdLhIAzsp8A9V54pBphcQKZ0ch1n1q3Us4ANEYWjEp5Mh6f/ex8 + 6olKWWw/PFX+Yym76+VO7dqXp+7l7tvj7nbmN0p3dghUtZod+5nhx3l+ZkYIaKoGrnVCY+HRuO6M + TghQV7eKa+yjw34Nw7U15Gt4bLXnSP42wlvuPPGcOY6eoCFa1A1erOEAb+QOwXvn37OmHK9aT+VU + 4TQ/HHciTW2dKf1ZxVAJLXxTOE798Kl9Sjt69z6DA7Rv1gnWGWWxQLPluheM42TUg9PfNkOzCUSD + VM7ml1EwJQTfeeSqqISuuRtKHTZjizi9LMssXSUMgkPwHwAA//8DAJjMFkUTAwAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1aba2d2d9e1601-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:24:54 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "368" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=15552000; includeSubDomains; preload + x-ratelimit-limit-requests: + - "10000" + x-ratelimit-limit-tokens: + - "30000000" + x-ratelimit-remaining-requests: + - "9999" + x-ratelimit-remaining-tokens: + - "29999973" + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_fa05087f5334c0d12f533106097751dd + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/TestMemoryAgent.test_dummyenv[gpt-4o-mini-2024-07-18].yaml b/tests/cassettes/TestMemoryAgent.test_dummyenv[gpt-4o-mini-2024-07-18].yaml new file mode 100644 index 00000000..2433e801 --- /dev/null +++ b/tests/cassettes/TestMemoryAgent.test_dummyenv[gpt-4o-mini-2024-07-18].yaml @@ -0,0 +1,121 @@ +interactions: + - request: + body: + '{"messages": [{"role": "user", "content": "Write a 5 word story via print_story"}, + {"role": "user", "content": "\n\nThese are relevant + memories from previous attempts at similar tasks, along with the action taken + and the discounted cumulative reward from that action. A negative reward is + failure, a positive reward is success.\n\n \n Write + a 5 word story and call print\n \n \n Tool request message + \"Stories that start with ''Once there was'' are always interesting.\" for tool + calls: print_story(story=''Once there were was nothing.'') [id=c92b617a-7dcb-466e-a3ed-f61ec0590cb3]\n \n \n 1000.0\n \n\n\nConsidering + the memories, choose the next action."}], "model": "gpt-4o-mini-2024-07-18", + "temperature": 0.1, "tool_choice": "required", "tools": [{"type": "function", + "function": {"name": "print_story", "description": "Print a story.", "parameters": + {"type": "object", "properties": {"story": {"description": "Story to print.", + "title": "Story", "type": "string"}}, "required": ["story"]}}}, {"type": "function", + "function": {"name": "cast_float", "description": "Cast the input argument x + to a float.", "parameters": {"type": "object", "properties": {"x": {"title": + "X", "type": "string"}}, "required": ["x"]}}}, {"type": "function", "function": + {"name": "cast_int", "description": "Cast the input argument x to an integer.", + "parameters": {"type": "object", "properties": {"x": {"title": "X", "type": + "number"}}, "required": ["x"]}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "1578" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.44.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.44.1 + x-stainless-raw-response: + - "true" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA2xSW2+bMBR+51dY5zlUhFwgvE3t01qtm6ZJrZYJueYAbnybbZpGUf77ZEgDjcYD + ss7n78J3OEaEAK+gIMBa6pk0Iv6y/p79un9z7frp4W2Xfnv4encrs2Sxa5+Nhllg6JdXZP6DdcO0 + NAI912qAmUXqMajOs3Sd5Jt8k/aA1BWKQGuMj5c6llzxOE3SZZxk8Tw/s1vNGTooyO+IEEKO/Tvk + VBW+Q0GS2cdEonO0QSgulwgBq0WYAHWOO0+Vh9kIMq08qhBddUJMAK+1KBkVYjQenuPkPJZFhSif + HpsEX/O/99mev//Uq2a1P/yo2fPEb5A+mD5Q3Sl2KWmCX+bFlRkhoKjsucZy5UvntT1c0QkBaptO + ovIhOhy30F/bQrGFR8WQ+BYtkj11RGnfctXcbOEEn0RO0f/Ofyb9WKw7R8W5uPP8dNmE0I2x+sVd + FQs1V9y1pUXq+g8M2czgHXx6B+g+LRGM1dL40usdqiCY5stBD8YfbUTn6zPotadinC+SJDonBHdw + HmVZc9Wg7avs92HKZb6oFpsqX2UQnaJ/AAAA//8DAL8BsbwOAwAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1aba248f8bcf2f-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:24:52 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=viZdDbMlPim18zXA2VSk3Ps42oEX6OxE7sBVhT13fDE-1726089892-1.0.1.1-zUP54gGNaVXKUnFpkubX4TG1ojkxbIbWQOfOy8T728MOP.DLhzYV8D1zfbxiBdPPlq4wuFK3QDaHmdSZAywPOg; + path=/; expires=Wed, 11-Sep-24 21:54:52 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=o_nGJtTs1cXWPFPetWlXSw9gJf7WeFSTun8Ozgl__Gg-1726089892934-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "255" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=15552000; includeSubDomains; preload + x-ratelimit-limit-requests: + - "30000" + x-ratelimit-limit-tokens: + - "150000000" + x-ratelimit-remaining-requests: + - "29999" + x-ratelimit-remaining-tokens: + - "149999807" + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_c0ed70d37c948634bae4f2d44f0d8ec1 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/TestMultipleCompletionLLMModel.test_output_schema.yaml b/tests/cassettes/TestMultipleCompletionLLMModel.test_output_schema.yaml new file mode 100644 index 00000000..17bce6a7 --- /dev/null +++ b/tests/cassettes/TestMultipleCompletionLLMModel.test_output_schema.yaml @@ -0,0 +1,102 @@ +interactions: + - request: + body: + '{"messages": [{"role": "system", "content": "Respond following this JSON + schema:\n\n{\n \"properties\": {\n \"name\": {\n \"title\": \"Name\",\n \"type\": + \"string\"\n },\n \"age\": {\n \"title\": \"Age\",\n \"type\": + \"integer\"\n }\n },\n \"required\": [\n \"name\",\n \"age\"\n ],\n \"title\": + \"DummyOutputSchema\",\n \"type\": \"object\"\n}"}, {"role": "user", "content": + "My name is Claude and I am 1 year old. What is my name and age?"}], "model": + "gpt-3.5-turbo", "n": 2, "response_format": {"type": "json_object"}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "567" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.44.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.44.1 + x-stainless-raw-response: + - "true" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA9ySzU7DMBCE73kKa88JalraJrlFcEBwAHFBgqDKdTapwbEte8OPqr47cpr+IN6A + iw/zedYztrcRYyBrKBiIDSfRWZWUi4fls13cNHmW3Tflx5NIy+s7uqXHMv+EODjM+g0FHVwXwnRW + IUmj91g45IRharqcLiZZnuXLAXSmRhVsraVkdjFPqHdrk0zS6Xx0bowU6KFgLxFjjG2HNWTUNX5B + wSbxQenQe94iFMdNjIEzKijAvZeeuCaIT1AYTaiH2NtKM1aB5h1WULAKrhTva6wg3gPeDnpa6d35 + BIdN73looHulRn13jKRMa51Z+5Ef9UZq6Tcrh9wbHY73ZCxEZ+Y/PdP/1jNi7HV44v5XG7DOdJZW + ZN5Rh4Fpmu3nwelXnejscoRkiKsz13wajQnBf3vCbtVI3aKzTg43MfTYRT8AAAD//wMAt0bHfvAC + AAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1aba40e9e2ce5c-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:24:57 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "342" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=15552000; includeSubDomains; preload + x-ratelimit-limit-requests: + - "12000" + x-ratelimit-limit-tokens: + - "1000000" + x-ratelimit-remaining-requests: + - "11999" + x-ratelimit-remaining-tokens: + - "999877" + x-ratelimit-reset-requests: + - 5ms + x-ratelimit-reset-tokens: + - 7ms + x-request-id: + - req_134069e4a1883fe754d46858e6824763 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/TestMultipleCompletionLLMModel.test_text_image_message[gpt-4o-mini-2024-07-18].yaml b/tests/cassettes/TestMultipleCompletionLLMModel.test_text_image_message[gpt-4o-mini-2024-07-18].yaml new file mode 100644 index 00000000..b38dfabc --- /dev/null +++ b/tests/cassettes/TestMultipleCompletionLLMModel.test_text_image_message[gpt-4o-mini-2024-07-18].yaml @@ -0,0 +1,98 @@ +interactions: + - request: + body: + '{"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": + {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAAKElEQVR4nO3NMQEAAAjDMMC/ZzDBvlRA01vZJvwHAAAAAAAAAAAAbx2jxAE/i2AjOgAAAABJRU5ErkJggg=="}}, + {"type": "text", "text": "What color is this square? Respond only with the color + name."}]}], "model": "gpt-4o-mini-2024-07-18", "n": 2}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "381" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.44.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.44.1 + x-stainless-raw-response: + - "true" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA9SSy2rDMBBF9/4KMeu42K6TON7lC1Ia6INSgmKPbbWyRkgTaBvy70WO86Jf0I0W + 9+iOjpD2kRCgaigFVJ3kqrc6Xs4e5q/L5/VPkqpVu6q9vufkpVt31BZPMAkN2n5gxafWXUW91ciK + zBFXDiVjmJrOs1lSLIrFbAA91ahDrbUc5xT3yqg4S7I8TuZxWoztjlSFHkrxFgkhxH5Yg6ep8QtK + kUxOSY/eyxahPG8SAhzpkID0XnmWhmFygRUZRjOoP2J9TRw2Oy+DndlpPeaH81GaWuto60d+zhtl + lO82DqUnE8Z6JgvRVfmPf/pf/CMh3ocn2d1YgnXUW94wfaIJA4tpNr4JXL7CBWcjY2Kpb1pZNCqC + //aM/aZRpkVnnTpesbGbaY5Zkxf5FiE6RL8AAAD//wMAIMaR3bACAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1aba3908cdcfed-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:24:57 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "1107" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=15552000; includeSubDomains; preload + x-ratelimit-limit-requests: + - "30000" + x-ratelimit-limit-tokens: + - "150000000" + x-ratelimit-remaining-requests: + - "29999" + x-ratelimit-remaining-tokens: + - "149999187" + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_157e90ed4d9d67bdf5b00544b0abd92c + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/TestReActAgent.test_agent_grad[claude-3-haiku-20240307].yaml b/tests/cassettes/TestReActAgent.test_agent_grad[claude-3-haiku-20240307].yaml new file mode 100644 index 00000000..a9638d75 --- /dev/null +++ b/tests/cassettes/TestReActAgent.test_agent_grad[claude-3-haiku-20240307].yaml @@ -0,0 +1,75 @@ +interactions: + - request: + body: + '{"messages": [{"role": "user", "content": [{"type": "text", "text": "Write + a 5 word story via print_story"}]}], "temperature": 0.1, "stop_sequences": ["Observation:"], + "system": [{"type": "text", "text": "Answer the following questions as best + you can. You have access to the following tools:\n\nNAME: print_story\n\nSYNOPSIS:\n print_story(string + story)\n\nDESCRIPTION:\n Print a story.\n\nPARAMETERS:\n story (string): + Story to print.\n\nNAME: cast_float\n\nSYNOPSIS:\n cast_float(string x)\n\nDESCRIPTION:\n Cast + the input argument x to a float.\n\nPARAMETERS:\n x (string): No description + provided.\n\nNAME: cast_int\n\nSYNOPSIS:\n cast_int(number x)\n\nDESCRIPTION:\n Cast + the input argument x to an integer.\n\nPARAMETERS:\n x (number): No description + provided.\n\nUse the following format:\n\nThought: you should always think about + what to do\nAction: the action to take, should be one of [print_story, cast_float, + cast_int]\nAction Input: comma separated list of inputs to action as python + tuple\nObservation: the result of the action\n... (this Thought/Action/Action + Input/Observation can repeat N times)\n\nExample:\n\nThought: I need to use + the get_weather tool\nAction: get_weather\nAction Input: \"New York\", 7\nObservation: + The 7 day forecast for New York is [...]"}], "max_tokens": 4096, "model": "claude-3-haiku-20240307"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - "2023-06-01" + connection: + - keep-alive + content-length: + - "1366" + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - litellm/1.44.24 + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA1SQT2sCMRDFv8owpxaibLUq5NbSP9gevBRa6MoSN8NuWJ2kyUQr4ncvu8WDp+H9 + mHmPNyd0FjXuUlMVd6/71D3z4/y9frIhbj7Dy9sXz1ChHAP1W5SSaQgVRr/tgUnJJTEsqHDnLW1R + Y7012dJoOmqN6/JoUkzui2mxQIW1ZyEW1N+ni6PQb387DI0frc9NKxqWwEQWxMMhOiEwMIODjxaS + +Hgcl/xQi/OsIUTHUg30AmHJIYuGmxJXXBPk4BkMiNuRAmkJiO24xNuS8bxWmMSHKpJJnlH/q0Q/ + mbjue15rjatNorg3QzgqzMM79Aldn1mJ74gT6mmxUOizXLH5+fwHAAD//wMANd2UgG0BAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1aba247cd324df-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:24:53 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + request-id: + - req_01AP39pgvBs5GinrCu265JDv + via: + - 1.1 google + x-cloud-trace-context: + - 137e17d4feda2dee4ed60d2340ac0f00 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/TestReActAgent.test_agent_grad[gpt-4-turbo].yaml b/tests/cassettes/TestReActAgent.test_agent_grad[gpt-4-turbo].yaml new file mode 100644 index 00000000..dc345dc3 --- /dev/null +++ b/tests/cassettes/TestReActAgent.test_agent_grad[gpt-4-turbo].yaml @@ -0,0 +1,118 @@ +interactions: + - request: + body: + '{"messages": [{"role": "system", "content": "Answer the following questions + as best you can. You have access to the following tools:\n\nNAME: print_story\n\nSYNOPSIS:\n print_story(string + story)\n\nDESCRIPTION:\n Print a story.\n\nPARAMETERS:\n story (string): + Story to print.\n\nNAME: cast_float\n\nSYNOPSIS:\n cast_float(string x)\n\nDESCRIPTION:\n Cast + the input argument x to a float.\n\nPARAMETERS:\n x (string): No description + provided.\n\nNAME: cast_int\n\nSYNOPSIS:\n cast_int(number x)\n\nDESCRIPTION:\n Cast + the input argument x to an integer.\n\nPARAMETERS:\n x (number): No description + provided.\n\nUse the following format:\n\nThought: you should always think about + what to do\nAction: the action to take, should be one of [print_story, cast_float, + cast_int]\nAction Input: comma separated list of inputs to action as python + tuple\nObservation: the result of the action\n... (this Thought/Action/Action + Input/Observation can repeat N times)\n\nExample:\n\nThought: I need to use + the get_weather tool\nAction: get_weather\nAction Input: \"New York\", 7\nObservation: + The 7 day forecast for New York is [...]"}, {"role": "user", "content": "Write + a 5 word story via print_story"}], "model": "gpt-4-turbo", "stop": ["Observation:"], + "temperature": 0.1}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "1289" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.44.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.44.1 + x-stainless-raw-response: + - "true" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA1SRQWvbQBCF7/oVw55asIysOLalWxIoBBooofTQuoj17kjaWNpZdkdN3OD/XlaW + Hfeyh/fNG968fU8AhNGiBKFayap3XXq3+rb+ces13T/8fHre/JVe71V7//z0+qVQYhYdtHtBxWfX + XFHvOmRD9oSVR8kYty7W+SrbFJviZgQ9aeyirXGcLlMe/I7SPMuXabZMs2Jyt2QUBlHCrwQA4H18 + Y06r8U2UkM3OSo8hyAZFeRkCEJ66qAgZggksLYvZB1RkGe0Y/XtLQ9NyCY9gETUwwSk4SNh5gzUE + Jn+AIRjbAL5Jxd0BavMH4ZW8DvOtvVPx6BKcN5arcfwswqN1A5fwaSu+UmDY4yFATYPVMFiNHgLV + cr4Vn7f2Op/HeggydmSHrpv04+XgjhrnaRcmftFrY01oK48ykI3HBSYnRnpMAH6PxQ7/dSWcp95x + xbRHGxfmq/Vpn/j4zytaTJCJZXetr5IpoQiHwNhXtbEN+rGTGKV21aLO85uFXBa3Ijkm/wAAAP// + AwCnrkw8dQIAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1aba2bdbe5ce90-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:24:55 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=6q5OK8YOw4TpDZicGrlwpFodhg6BV25taNZ0U3Pm7nA-1726089895-1.0.1.1-iqAphWihSlws3LFtAGDmAlw6MS5MnYsAbg7y53FvZuRRf0bmCAmgZszJYiFn15AK5T0r1bpkbyHS1mKiIYLwWw; + path=/; expires=Wed, 11-Sep-24 21:54:55 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=Tk6wEhPWEpyORtlkLjrTQSFWntY37MYAwouqprtCAhU-1726089895219-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "1244" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=15552000; includeSubDomains; preload + x-ratelimit-limit-requests: + - "10000" + x-ratelimit-limit-tokens: + - "2000000" + x-ratelimit-remaining-requests: + - "9999" + x-ratelimit-remaining-tokens: + - "1999710" + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 8ms + x-request-id: + - req_e8d97f9cde79348fbaed1da39eb1d6e6 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/TestReActAgent.test_react_dummyenv[claude-3-haiku-20240307].yaml b/tests/cassettes/TestReActAgent.test_react_dummyenv[claude-3-haiku-20240307].yaml new file mode 100644 index 00000000..b1e6d24a --- /dev/null +++ b/tests/cassettes/TestReActAgent.test_react_dummyenv[claude-3-haiku-20240307].yaml @@ -0,0 +1,75 @@ +interactions: + - request: + body: + '{"messages": [{"role": "user", "content": [{"type": "text", "text": "Write + a 5 word story via print_story"}]}], "temperature": 0.1, "stop_sequences": ["Observation:"], + "system": [{"type": "text", "text": "Answer the following questions as best + you can. You have access to the following tools:\n\nNAME: print_story\n\nSYNOPSIS:\n print_story(string + story)\n\nDESCRIPTION:\n Print a story.\n\nPARAMETERS:\n story (string): + Story to print.\n\nNAME: cast_float\n\nSYNOPSIS:\n cast_float(string x)\n\nDESCRIPTION:\n Cast + the input argument x to a float.\n\nPARAMETERS:\n x (string): No description + provided.\n\nNAME: cast_int\n\nSYNOPSIS:\n cast_int(number x)\n\nDESCRIPTION:\n Cast + the input argument x to an integer.\n\nPARAMETERS:\n x (number): No description + provided.\n\nUse the following format:\n\nThought: you should always think about + what to do\nAction: the action to take, should be one of [print_story, cast_float, + cast_int]\nAction Input: comma separated list of inputs to action as python + tuple\nObservation: the result of the action\n... (this Thought/Action/Action + Input/Observation can repeat N times)\n\nExample:\n\nThought: I need to use + the get_weather tool\nAction: get_weather\nAction Input: \"New York\", 7\nObservation: + The 7 day forecast for New York is [...]"}], "max_tokens": 4096, "model": "claude-3-haiku-20240307"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - "2023-06-01" + connection: + - keep-alive + content-length: + - "1366" + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - litellm/1.44.24 + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA1SQQWvCQBCF/8owpxZWiVor7K0eSj0UKXiwNCVsk8Esmpm4O1trg/+9JMWDp+F9 + zLzHmw59hRabuCuyycvkfeObbXuk+Wn5u1m+bV+f5w4N6rmlfotidDtCg0EOPXAx+qiOFQ02UtEB + LZYHlyoazUa18/s0mmbTh2yWLdBgKazEivajuzoq/fS3w7C4qSXtarWwAiaqQAVOwSuBgzmcJFQQ + VcJ5nPNTqV7YQhs8azHQK4QVt0kt3OW45pIgtcLgQH1DBrQmIK7GOd7njJdPg1GlLQK5KIz2X0U6 + JuKy73mrLa6/IoVvN4SjwTS8w3bo+8xCZU8c0c6yhUFJesMeL5c/AAAA//8DAFHvxlNtAQAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1aba247b2ace8c-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:24:53 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + request-id: + - req_01J5QEgPTjLNb21napT1yMju + via: + - 1.1 google + x-cloud-trace-context: + - 8d6cc4291c877477bb67bbcb245affdf + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/TestReActAgent.test_react_dummyenv[gpt-4-turbo].yaml b/tests/cassettes/TestReActAgent.test_react_dummyenv[gpt-4-turbo].yaml new file mode 100644 index 00000000..83e3c296 --- /dev/null +++ b/tests/cassettes/TestReActAgent.test_react_dummyenv[gpt-4-turbo].yaml @@ -0,0 +1,118 @@ +interactions: + - request: + body: + '{"messages": [{"role": "system", "content": "Answer the following questions + as best you can. You have access to the following tools:\n\nNAME: print_story\n\nSYNOPSIS:\n print_story(string + story)\n\nDESCRIPTION:\n Print a story.\n\nPARAMETERS:\n story (string): + Story to print.\n\nNAME: cast_float\n\nSYNOPSIS:\n cast_float(string x)\n\nDESCRIPTION:\n Cast + the input argument x to a float.\n\nPARAMETERS:\n x (string): No description + provided.\n\nNAME: cast_int\n\nSYNOPSIS:\n cast_int(number x)\n\nDESCRIPTION:\n Cast + the input argument x to an integer.\n\nPARAMETERS:\n x (number): No description + provided.\n\nUse the following format:\n\nThought: you should always think about + what to do\nAction: the action to take, should be one of [print_story, cast_float, + cast_int]\nAction Input: comma separated list of inputs to action as python + tuple\nObservation: the result of the action\n... (this Thought/Action/Action + Input/Observation can repeat N times)\n\nExample:\n\nThought: I need to use + the get_weather tool\nAction: get_weather\nAction Input: \"New York\", 7\nObservation: + The 7 day forecast for New York is [...]"}, {"role": "user", "content": "Write + a 5 word story via print_story"}], "model": "gpt-4-turbo", "stop": ["Observation:"], + "temperature": 0.1}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "1289" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.44.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.44.1 + x-stainless-raw-response: + - "true" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA1SRT2+bQBDF73yK0Z5ayViAXdtwy6GHtK6SQ/8c6gotMMAmsLPeHZrYkb97tRjb + 6YXD++17evN4CwCEqkQGomwll73pwrvV4/pnvP/24/jrc7/ZD/Lh+Ljdf3kqtl8fjmLmHVQ8YckX + 17yk3nTIivQZlxYlo0+N18kq2qSbdDGCnirsvK0xHC5DHmxBYRIlyzBahlE6uVtSJTqRwe8AAOBt + /PqeusJXkUE0uyg9OicbFNn1EYCw1HlFSOeUY6lZzG6wJM2ox+rfWxqaljO4B41YAROci4OEwiqs + wTHZA5SkfZDSDVAN+CpL7g5Qq78IL2QrN9/pu9Ifn4GxSnM+2i4i3GszcAYfdmJLjuEZD24GNQ26 + Ao0vYCS3br4TH3f6fU+L9eCk30oPXTfpp+vhHTXGUuEmftVrpZVrc4vSkfZHOiYjRnoKAP6MAw// + bSaMpd5wzvSM2gcmq/U5T9z+640u4gkysezeudJNMDUU7uAY+7xWukE7buKr1CaP6yRZxHKZfhLB + KfgHAAD//wMAB+eXzX0CAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1aba2adb61986a-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:24:54 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=k6LGjHHIkVhEp6QnlyAXgEkCJanPkbZhZz4SVCHpmUk-1726089894-1.0.1.1-FZzGbppBKvd5I5uz.viWzOBlJbBwRTLJU1pCYSKAte9ZKV4V7j.b0e.ZkE3iD22LMWH1BfyidS5LEwOm6VByBQ; + path=/; expires=Wed, 11-Sep-24 21:54:54 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=JlAl4r6feqcQx1xq3RgOKZV5ZMaj3KeoVnGAkAW_oX4-1726089894996-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "1281" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=15552000; includeSubDomains; preload + x-ratelimit-limit-requests: + - "10000" + x-ratelimit-limit-tokens: + - "2000000" + x-ratelimit-remaining-requests: + - "9999" + x-ratelimit-remaining-tokens: + - "1999710" + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 8ms + x-request-id: + - req_56c6fc69f2afd0b3df14ae79de95f1dc + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/TestSimpleAgent.test_agent_grad[claude-3-haiku-20240307].yaml b/tests/cassettes/TestSimpleAgent.test_agent_grad[claude-3-haiku-20240307].yaml new file mode 100644 index 00000000..ba623fb5 --- /dev/null +++ b/tests/cassettes/TestSimpleAgent.test_agent_grad[claude-3-haiku-20240307].yaml @@ -0,0 +1,72 @@ +interactions: + - request: + body: + '{"messages": [{"role": "user", "content": [{"type": "text", "text": "Write + a 5 word story via print_story"}]}], "temperature": 0.1, "tools": [{"name": + "print_story", "description": "Print a story.", "input_schema": {"type": "object", + "properties": {"story": {"description": "Story to print.", "title": "Story", + "type": "string"}}, "required": ["story"]}}, {"name": "cast_float", "description": + "Cast the input argument x to a float.", "input_schema": {"type": "object", + "properties": {"x": {"title": "X", "type": "string"}}, "required": ["x"]}}, + {"name": "cast_int", "description": "Cast the input argument x to an integer.", + "input_schema": {"type": "object", "properties": {"x": {"title": "X", "type": + "number"}}, "required": ["x"]}}], "tool_choice": {"type": "any"}, "max_tokens": + 4096, "model": "claude-3-haiku-20240307"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-beta: + - tools-2024-05-16 + anthropic-version: + - "2023-06-01" + connection: + - keep-alive + content-length: + - "825" + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - litellm/1.44.24 + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA1SQW0vDQBCF/0qY562k6UXZt0J9MSXgDQsiYU2mMU0ym+7MqiHkv8u2CPo4Z845 + fJwR6hI0dFzl8fzhybnl9susuk3yIrdyeE8P2yMokKHH4EJmUyEocLYNgmGuWQwJKOhsiS1oKFrj + S5wtZh+mbvwsiZNlvIivQUFhSZAE9Ov42yjWtrnnUHnmCLfP43l2ujlJWmX32e64a57vskez36eg + gEwXcr2rSXIW64YQpd4L6BEugoZN+Ykk3qGLGLHhSBwa9g5VdKip5MgTfvdYCJbR0XpHOFzBNL2p + 0NDnwWzpP975wXjySAWCJt+2Cvx5Dj1eCHKxDRKDXq0TBdbLX225nqYfAAAA//8DAP317yttAQAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1aba247bffcf27-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:24:53 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + request-id: + - req_01M7Ex1HkXPXVfPGUAXSmnrx + via: + - 1.1 google + x-cloud-trace-context: + - 8fcecb13f13232d5f633d00dec0fbbad + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/TestSimpleAgent.test_agent_grad[gpt-4o-mini-2024-07-18].yaml b/tests/cassettes/TestSimpleAgent.test_agent_grad[gpt-4o-mini-2024-07-18].yaml new file mode 100644 index 00000000..b61ff104 --- /dev/null +++ b/tests/cassettes/TestSimpleAgent.test_agent_grad[gpt-4o-mini-2024-07-18].yaml @@ -0,0 +1,113 @@ +interactions: + - request: + body: + '{"messages": [{"role": "user", "content": "Write a 5 word story via print_story"}], + "model": "gpt-4o-mini-2024-07-18", "temperature": 0.1, "tool_choice": "required", + "tools": [{"type": "function", "function": {"name": "print_story", "description": + "Print a story.", "parameters": {"type": "object", "properties": {"story": {"description": + "Story to print.", "title": "Story", "type": "string"}}, "required": ["story"]}}}, + {"type": "function", "function": {"name": "cast_float", "description": "Cast + the input argument x to a float.", "parameters": {"type": "object", "properties": + {"x": {"title": "X", "type": "string"}}, "required": ["x"]}}}, {"type": "function", + "function": {"name": "cast_int", "description": "Cast the input argument x to + an integer.", "parameters": {"type": "object", "properties": {"x": {"title": + "X", "type": "number"}}, "required": ["x"]}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "867" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.44.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.44.1 + x-stainless-raw-response: + - "true" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA2xSy46bMBTd8xXWXYcR5MVjF6mLWUw70y5mWjUVcsyFeMb4IttUpVH+vTIwCROV + BbLu8XlwLqeAMZAl5AzEkTvRtCrcbZ+SZ+3sj3tKn76/RN3uUX1+2TzL+LiJYOEZdHhF4d5Zd4Ka + VqGTpEdYGOQOvWqcLLdRmqXZagAaKlF5Wt26cE1hI7UMl9FyHUZJGKcT+0hSoIWc/QwYY+w0vH1O + XeIfyFm0eJ80aC2vEfLLJcbAkPIT4NZK67h2sLiCgrRD7aPrTqkZ4IhUIbhSV+PxOc3O17K4UsWX + +sFs/37Nsj7bJjb9dngU2S7OPs38Rum+HQJVnRaXkmb4ZZ7fmDEGmjcDtzVSu8I6Mv0NnTHgpu4a + 1M5Hh9Mehmt7yPfwQNaxN+wXrCQyrCKDv9EwochiebeHM3zQOgf/O/+a1WSw6ixXU3/T/HxZiKK6 + NXSwN/1CJbW0x8Igt8N3+ojt6O19BgfoPuwSWkNN6wpHb6i9YBwvRz24/m8zNJ1AR46r2XwVBVNC + sL112BSV1DWaodFhLW2xTlflKivTTQLBOfgHAAD//wMAjgCXzxUDAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1aba2bcb73fa9e-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:24:54 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=FY9zJJk.AkvvBFb40KqpYxb.ksCTGQRszXXDeJ_Q7Ow-1726089894-1.0.1.1-fpfVV9Y7c_H5canwMeaOeEuupAKBaszki0DTAc_YeGtTXINXi6ghzWJsgCwZCKnG58iqk4sATa2Hr_k3AscLFA; + path=/; expires=Wed, 11-Sep-24 21:54:54 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=BCIddIQXcROP.EJte0kXuTtO67IB_JZ5yiMJ1P0vRJQ-1726089894188-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "262" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=15552000; includeSubDomains; preload + x-ratelimit-limit-requests: + - "30000" + x-ratelimit-limit-tokens: + - "150000000" + x-ratelimit-remaining-requests: + - "29999" + x-ratelimit-remaining-tokens: + - "149999973" + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_3b78d727d925adeb6e81fc5c6aff7a21 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/TestSimpleAgent.test_dummyenv[claude-3-haiku-20240307].yaml b/tests/cassettes/TestSimpleAgent.test_dummyenv[claude-3-haiku-20240307].yaml new file mode 100644 index 00000000..21ac7c87 --- /dev/null +++ b/tests/cassettes/TestSimpleAgent.test_dummyenv[claude-3-haiku-20240307].yaml @@ -0,0 +1,73 @@ +interactions: + - request: + body: + '{"messages": [{"role": "user", "content": [{"type": "text", "text": "Write + a 5 word story via print_story"}]}], "temperature": 0.1, "tools": [{"name": + "print_story", "description": "Print a story.", "input_schema": {"type": "object", + "properties": {"story": {"description": "Story to print.", "title": "Story", + "type": "string"}}, "required": ["story"]}}, {"name": "cast_float", "description": + "Cast the input argument x to a float.", "input_schema": {"type": "object", + "properties": {"x": {"title": "X", "type": "string"}}, "required": ["x"]}}, + {"name": "cast_int", "description": "Cast the input argument x to an integer.", + "input_schema": {"type": "object", "properties": {"x": {"title": "X", "type": + "number"}}, "required": ["x"]}}], "tool_choice": {"type": "any"}, "max_tokens": + 4096, "model": "claude-3-haiku-20240307"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-beta: + - tools-2024-05-16 + anthropic-version: + - "2023-06-01" + connection: + - keep-alive + content-length: + - "825" + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - litellm/1.44.24 + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA1SQUUvDQBCE/0rY54ukaa14b6W2FMSgWPVBJBzJto1J9uLtXmwJ+e9yLYI+7uzM + 8DEDVCVoaHmfJ5O328cs29wfFt364bnftq7f3333oEBOHQYXMps9ggJnmyAY5orFkICC1pbYgIai + Mb7EeBofTFX7OE3SWTJNbkBBYUmQBPT78Nso1ja551B55gi3z5PJ61PdrV6O2Wq93NDB7LZFlpol + KCDThlznKpKcxbpTiFLnBfQAF0HDouyRxDt0ESPWHIlDw96hinYVlRx5wmOHhWAZfVrvCE9XMI4f + KjR0eTBb+o93fjB+eaQCQZNvGgX+PIceLgS52BqJQV/PUwXWy19tNh/HHwAAAP//AwBX1dzRbQEA + AA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1aba247c0ccf5d-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:24:53 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + request-id: + - req_01VrQhCWEqKNRaxQ2vKLTXpH + via: + - 1.1 google + x-cloud-trace-context: + - 2f73b217496c2bc21b328bf1a748a629 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/TestSimpleAgent.test_dummyenv[gpt-4o-mini-2024-07-18].yaml b/tests/cassettes/TestSimpleAgent.test_dummyenv[gpt-4o-mini-2024-07-18].yaml new file mode 100644 index 00000000..7761efc5 --- /dev/null +++ b/tests/cassettes/TestSimpleAgent.test_dummyenv[gpt-4o-mini-2024-07-18].yaml @@ -0,0 +1,113 @@ +interactions: + - request: + body: + '{"messages": [{"role": "user", "content": "Write a 5 word story via print_story"}], + "model": "gpt-4o-mini-2024-07-18", "temperature": 0.1, "tool_choice": "required", + "tools": [{"type": "function", "function": {"name": "print_story", "description": + "Print a story.", "parameters": {"type": "object", "properties": {"story": {"description": + "Story to print.", "title": "Story", "type": "string"}}, "required": ["story"]}}}, + {"type": "function", "function": {"name": "cast_float", "description": "Cast + the input argument x to a float.", "parameters": {"type": "object", "properties": + {"x": {"title": "X", "type": "string"}}, "required": ["x"]}}}, {"type": "function", + "function": {"name": "cast_int", "description": "Cast the input argument x to + an integer.", "parameters": {"type": "object", "properties": {"x": {"title": + "X", "type": "number"}}, "required": ["x"]}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "867" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.44.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.44.1 + x-stainless-raw-response: + - "true" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA2xSTW+bQBC98ytWczaRjWnA3Fo1lVr54FRRqqiu0AYGvM2ys90d0riW/3vER2xi + lQNazdv3wRsOgRCgSsgEFDvJRWN1+PF6k/yIv9zd3iDLm4cNf//6eW3+3mtW+h/MOgY9/saC31hX + BTVWIysyA1w4lIyd6iKJrufpKl3FPdBQibqj1ZbDmMJGGRVG8ygO50m4SEf2jlSBHjLxMxBCiEP/ + 7nKaEl8gE/PZ26RB72WNkJ0uCQGOdDcB6b3yLA3D7AwWZBhNF920Wk8AJtJ5IbU+Gw/PYXI+lyW1 + zj85Hd29LJ9v/WYt7xP98Oeb9r7dTfwG6b3tA1WtKU4lTfDTPLswEwKMbHqudcpw7pnc/oIuBEhX + tw0a7qLDYQv9tS1kW1iTZ/GE+5koiZyoyOEzOlFo8lhebeEI77SOwf/OvyY1OaxaL/XY3zg/nhai + qbaOHv1Fv1Apo/wudyh9/51dRDt4dz69A7TvdgnWUWM5Z3pC0wkuFtGgB+f/bYKmI8jEUk/my3kw + JgS/94xNXilTo+sb7ddi8zhdlstVmX5IIDgGrwAAAP//AwB3Fgx6FQMAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1aba2c8fdbcf29-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:24:54 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=BwmSS43mmarya7_ibOS0g5eAoldYYWja.1i8yk_em38-1726089894-1.0.1.1-EIKFNI5MxGS3DXhWn8H7o.sCuigPgNp4hjyUYUmhs3_JoJj6UCuCrcwRZR9p.a4KsairNDaiDZ7Zp5k7pcuUTQ; + path=/; expires=Wed, 11-Sep-24 21:54:54 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=WTfbxZPAnf1ED3HCRks6o.bsMfHgxin_NmBR09BuOf4-1726089894817-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "784" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=15552000; includeSubDomains; preload + x-ratelimit-limit-requests: + - "30000" + x-ratelimit-limit-tokens: + - "150000000" + x-ratelimit-remaining-requests: + - "29999" + x-ratelimit-remaining-tokens: + - "149999973" + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_62f34426de9f766cee3cf0f4b99743e4 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/test_reflect_module.yaml b/tests/cassettes/test_reflect_module.yaml new file mode 100644 index 00000000..46f7af9d --- /dev/null +++ b/tests/cassettes/test_reflect_module.yaml @@ -0,0 +1,67 @@ +interactions: + - request: + body: + '{"messages": [{"role": "user", "content": [{"type": "text", "text": "\n I + am happy. How do I feel?\n\n\n You are sad.\n"}]}], + "temperature": 0, "system": [{"type": "text", "text": "Consider a proposed response + based on context. Reflect on the response within tags then conclude + with a possibly revised response within tags."}], "max_tokens": + 4096, "model": "claude-3-haiku-20240307"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-version: + - "2023-06-01" + connection: + - keep-alive + content-length: + - "452" + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - litellm/1.44.24 + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA3RRX0sjQQz/KiHPU69XtXJL8Qv4Jr6IK2W6k+7MuZuMk4xtKf3uxy7KycE9BX5/ + E3LGFLDBUfvt8udjDs8P6/e17fpjXJ9ud4f08DuiQztlmlSk6ntCh0WGCfCqSc2zocNRAg3YYDf4 + GmhxvYg+vdXFarm6WV4v79BhJ2zEhs3L+SvR6Dh559HgxqLUPtp9y0+RoJBmYSVo8Vkq+EKgPrQI + QUiBxWD01kWwSDBnHw1ykY8UKDg4xNRFUPNGCha9zbpMRYUhKUSf8+kKpiKfc5FckrdvpQepQ4Ad + gcmUvk9l/G+Mg51XCiA8s4n3UkZvSRj69EF81fLmx9/jWt7sE/th8VV23/J04Z5o+NxrMvyjwcur + QzXJ20JehbFB4rC1Whg/CaX3StwRNlyHwWGd39WcMXGutjV5I1Zs7lYOpdp3aP3rcvkDAAD//wMA + xRSOPQwCAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1aba490fb367bf-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:24:59 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + request-id: + - req_01BiuspyQPWq2rsnSvTuaQKJ + via: + - 1.1 google + x-cloud-trace-context: + - 3c185f043e8a4a8ce7ea8911547ea7dd + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/test_tools[claude-3-haiku-20240307].yaml b/tests/cassettes/test_tools[claude-3-haiku-20240307].yaml new file mode 100644 index 00000000..d09b31e6 --- /dev/null +++ b/tests/cassettes/test_tools[claude-3-haiku-20240307].yaml @@ -0,0 +1,263 @@ +interactions: + - request: + body: + '{"messages": [{"role": "user", "content": [{"type": "text", "text": "What + are three things I should do today?"}]}], "temperature": 0.1, "tools": [{"name": + "get_todo_list", "description": "Get todo list for today.", "input_schema": + {"type": "object", "properties": {"n": {"description": "number of items to return", + "title": "N", "type": "integer"}}, "required": ["n"]}}], "tool_choice": {"type": + "any"}, "max_tokens": 4096, "model": "claude-3-haiku-20240307"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-beta: + - tools-2024-05-16 + anthropic-version: + - "2023-06-01" + connection: + - keep-alive + content-length: + - "459" + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - litellm/1.44.24 + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA1SPQUvEMBCF/8s7p5CmXRZy1JvIHlbRRZEQtyENmybtzkTQ0v8urXjY47w38703 + M0IHjYG8kfXL/mibR3WalH8ozPbn9BqUhwB/j27dckTWOwhcc1wFSxSIbWIIDLlzERrnaEvnqqbq + bbiUSknVykbuIXDOiV1i6Pf5n8g5R1NoRW491rkYWd/Vu6+3w1NoP6l/PozK3/f1boJAssN65x0b + zl02MdAaHtJYGHpGgm6W5UOAOI/m6izldJuzGeSm4tLZQacSo0DZ/tLzH8hwvrhE0G2rBHLhG00u + yy8AAAD//wMAbVvKxDYBAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1acf2ecaba24f3-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:39:15 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + request-id: + - req_01MiAfxUZTYJLXnfbKzoun8A + via: + - 1.1 google + x-cloud-trace-context: + - 7bf875d346474586a91e14384ca3f6c6 + status: + code: 200 + message: OK + - request: + body: + '{"messages": [{"role": "user", "content": [{"type": "text", "text": "What + are three things I should do today?"}]}], "temperature": 0.1, "tools": [{"name": + "get_todo_list", "description": "Get todo list for today.", "input_schema": + {"type": "object", "properties": {"n": {"description": "number of items to return", + "title": "N", "type": "integer"}}, "required": ["n"]}}], "tool_choice": {"type": + "tool", "name": "get_todo_list"}, "max_tokens": 4096, "model": "claude-3-haiku-20240307"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-beta: + - tools-2024-05-16 + anthropic-version: + - "2023-06-01" + connection: + - keep-alive + content-length: + - "485" + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - litellm/1.44.24 + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA1SPQUvEMBCF/8s7p9DdVMXcrOLBlT3oKoJICO3YLU2Tupls2S3975KKoMd5b+Z7 + bya0NRT60Oh8dXvzePx8Kc14bsp2tzt241Y+vEKATwOlLQrBNASBg7dJMCG0gY1jCPS+JguFyppY + UyazvWm7mK3zdZHL/AoClXdMjqHep18ie291DAm59Ehz1Plqc3p7Om9Mf0eXzxeSSh724/0WAs70 + 6a4h1uxrr20bUnjrhshQExyUnOcPgcB+0Acywbv/OYsR6CuSqwjKRWsF4vKXmn5Amn1HLkAVxbWA + j/xXk3KevwEAAP//AwCFEGNiNgEAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1acf338e3124f3-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:39:16 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + request-id: + - req_01X9jheCq2r9TnpN1ZuAgPYt + via: + - 1.1 google + x-cloud-trace-context: + - e4c23da92650af2dbe414c2ac05ca4b9 + status: + code: 200 + message: OK + - request: + body: + '{"messages": [{"role": "user", "content": [{"type": "text", "text": "What + are three things I should do today?"}]}], "temperature": 0.1, "tools": [{"name": + "get_todo_list_no_args", "description": "Get todo list for today.", "input_schema": + {"type": "object", "properties": {}, "required": []}}], "tool_choice": {"type": + "any"}, "max_tokens": 4096, "model": "claude-3-haiku-20240307"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-beta: + - tools-2024-05-16 + anthropic-version: + - "2023-06-01" + connection: + - keep-alive + content-length: + - "382" + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - litellm/1.44.24 + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA1SPQWvDMAyF/4vOLqRJSJiPW9tdSunaQxljGDfR0hDHSiP5UEL++3DGYDu+p6dP + ehO0NWjouTHJunh6PZ6Lt5dNSoWthlPeHcv9ARTIY8CYQmbbICgYyUXDMrcs1gso6KlGBxoqZ0ON + q2x1s20XVmmS5kmWlKCgIi/oBfTH9EsUImcCR+TyR9TBJOvz9XB77N6/dqf7td5Xm212ufhnUOBt + H/caFCNUk3Eti/Fk7NhwhPghCOhpnj8VsNBgRrRM/v+pZcB4D+grBO2DcwrCUk1PPwwj1KFn0Hma + KKAgf720nOdvAAAA//8DAO0NjyA5AQAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1acf3a9bf324f3-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:39:17 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + request-id: + - req_01QHRmS9onkUCNH4scRM7Jqz + via: + - 1.1 google + x-cloud-trace-context: + - 9a772a4e16d3958380b298b61f7cbfa2 + status: + code: 200 + message: OK + - request: + body: + '{"messages": [{"role": "user", "content": [{"type": "text", "text": "What + items will I have time to accomplish on my todo list today based on my calendar?"}]}], + "temperature": 0.1, "tools": [{"name": "get_todo_list_no_args", "description": + "Get todo list for today.", "input_schema": {"type": "object", "properties": + {}, "required": []}}, {"name": "get_calendar", "description": "Get text version + of calendar for today.", "input_schema": {"type": "object", "properties": {}, + "required": []}}], "tool_choice": {"type": "any"}, "max_tokens": 4096, "model": + "claude-3-haiku-20240307"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + anthropic-beta: + - tools-2024-05-16 + anthropic-version: + - "2023-06-01" + connection: + - keep-alive + content-length: + - "581" + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - litellm/1.44.24 + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: !!binary | + H4sIAAAAAAAAA1SP3UrDQBCF32Wut5A/jOxtoYolEC9qa0WWJRli6GYnzcwKNeTdZSOCXp4zZ76Z + M0PfgoaBO5Oku3N+fGhfP6tjdbvbXh935/A11qBAbiPGFDLbDkHBRC4alrlnsV5AwUAtOtDQOBta + 3OSbD9tfwiZLsiLJkxIUNOQFvYB+m3+JQuRM4Ihc/4g6mCSt8FQXL9PTdP982Kf1dMBttj+BAm+H + uNehGKGWjOtZjCdjp44jxI9BQM/L8q6AhUYzoWXy/0+tA8ZrQN8gaB+cUxDWanr+YRihC3oGXZSF + Agry18vKZfkGAAD//wMAci3hAzkBAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1acf403f1924f3-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:39:17 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Robots-Tag: + - none + request-id: + - req_013NLgmQKyM8eTQ9Wk96rNGY + via: + - 1.1 google + x-cloud-trace-context: + - d7a9cae78b02a6c5b7d689e60011d033 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/cassettes/test_tools[gpt-4o-mini-2024-07-18].yaml b/tests/cassettes/test_tools[gpt-4o-mini-2024-07-18].yaml new file mode 100644 index 00000000..e16fceca --- /dev/null +++ b/tests/cassettes/test_tools[gpt-4o-mini-2024-07-18].yaml @@ -0,0 +1,102 @@ +interactions: + - request: + body: + '{"messages": [{"role": "user", "content": "What are three things I should + do today?"}], "model": "gpt-4o-mini-2024-07-18", "temperature": 0.1, "tool_choice": + "required", "tools": [{"type": "function", "function": {"name": "get_todo_list", + "description": "Get todo list for today.", "parameters": {"type": "object", + "properties": {"n": {"description": "number of items to return", "title": "N", + "type": "integer"}}, "required": ["n"]}}}]}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - "437" + content-type: + - application/json + host: + - api.openai.com + user-agent: + - AsyncOpenAI/Python 1.44.1 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.44.1 + x-stainless-raw-response: + - "true" + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.12.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAA2xSXW/TMBR9z6+w7vOCkrakbd6QBkUTIMSYJmAocp2b1Ju/ajuItep/n+yEJqvI + g2Wd43vO8XGOCSHAaygJsB31TBqRviu+frp77/R+XXwQh01u93v8c9gWzfWNuoWrMKG3j8j8v6k3 + TEsj0HOteppZpB6Dar6cFdk6WxZZJKSuUYSx1vh0oVPJFU9n2WyRZss0Xw3TO80ZOijJr4QQQo5x + DTlVjX+hJFErIhKdoy1CeT5ECFgtAgLUOe48VR6uRpJp5VGF6KoTYkJ4rUXFqBCjcf8dJ/uxLCpE + 9fFwfVB5sa7vN0+f7c/bH9+yzXf5ZTvx66WfTQzUdIqdS5rwZ7y8MCMEFJVxtkVfeV3rSnDnLwQI + AWrbTqLyITwcH0A9QDk/watjp+R/+9+TDiw2naNiKGfAT+e2hW6N1Vt3UR40XHG3qyxSFy8BzmvT + ewef6ADdq4cCY7U04UpPqIJgUfRyMP5LI5kvBtJrT8WIr7JkyAfu2XmUVcNVi9ZYHp8YGlMtVvN6 + vq5Xb5eQnJIXAAAA//8DAMuhQoPwAgAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c1acf541ba617e6-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 11 Sep 2024 21:39:20 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - future-house-xr4tdh + openai-processing-ms: + - "269" + openai-version: + - "2020-10-01" + strict-transport-security: + - max-age=15552000; includeSubDomains; preload + x-ratelimit-limit-requests: + - "30000" + x-ratelimit-limit-tokens: + - "150000000" + x-ratelimit-remaining-requests: + - "29999" + x-ratelimit-remaining-tokens: + - "149999973" + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_731d802bf8ded5b6a0956c2fd91a0fc7 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/test_agents.py b/tests/test_agents.py index 195b6579..a0351615 100644 --- a/tests/test_agents.py +++ b/tests/test_agents.py @@ -121,9 +121,7 @@ class TestSimpleAgent: "model_name", [CILLMModelNames.ANTHROPIC.value, CILLMModelNames.OPENAI.value] ) @pytest.mark.asyncio - @pytest.mark.flaky( # Rerun if LLM call does not return expected result - reruns=3, only_on=[AssertionError] - ) + @pytest.mark.vcr async def test_dummyenv(self, dummy_env: DummyEnv, model_name: str) -> None: obs, tools = await dummy_env.reset() @@ -157,9 +155,7 @@ async def test_dummyenv(self, dummy_env: DummyEnv, model_name: str) -> None: "model_name", [CILLMModelNames.ANTHROPIC.value, CILLMModelNames.OPENAI.value] ) @pytest.mark.asyncio - @pytest.mark.flaky( # Rerun if LLM call does not return expected result - reruns=3, only_on=[AssertionError] - ) + @pytest.mark.vcr async def test_agent_grad(self, dummy_env: DummyEnv, model_name: str) -> None: obs, tools = await dummy_env.reset() @@ -210,9 +206,7 @@ class TestMemoryAgent: # # On 5/14/2024, claude 3 opus would not follow its past memories @pytest.mark.parametrize("model_name", [CILLMModelNames.OPENAI.value]) @pytest.mark.asyncio - @pytest.mark.flaky( # Rerun if LLM call does not return expected result - reruns=3, only_on=[AssertionError] - ) + @pytest.mark.vcr async def test_dummyenv(self, dummy_env: DummyEnv, model_name: str) -> None: obs, tools = await dummy_env.reset() @@ -256,9 +250,7 @@ async def test_dummyenv(self, dummy_env: DummyEnv, model_name: str) -> None: raise RuntimeError("Could not find LLMCallOp in compute graph") @pytest.mark.asyncio - @pytest.mark.flaky( # Rerun if LLM call does not return expected result - reruns=3, only_on=[AssertionError] - ) + @pytest.mark.vcr async def test_agent_grad(self, dummy_env: DummyEnv) -> None: obs, tools = await dummy_env.reset() @@ -306,9 +298,7 @@ class TestReActAgent: "model_name", [CILLMModelNames.ANTHROPIC.value, "gpt-4-turbo"] ) @pytest.mark.asyncio - @pytest.mark.flaky( # Rerun if LLM call does not return expected result - reruns=3, only_on=[AssertionError] - ) + @pytest.mark.vcr async def test_react_dummyenv(self, dummy_env: DummyEnv, model_name: str) -> None: obs, tools = await dummy_env.reset() agent = ReActAgent(llm_model={"model": model_name, "temperature": 0.1}) @@ -377,9 +367,7 @@ def test_agent_op_naming(self) -> None: "model_name", [CILLMModelNames.ANTHROPIC.value, "gpt-4-turbo"] ) @pytest.mark.asyncio - @pytest.mark.flaky( # Rerun if LLM call does not return expected result - reruns=3, only_on=[AssertionError] - ) + @pytest.mark.vcr async def test_agent_grad(self, dummy_env: DummyEnv, model_name: str) -> None: obs, tools = await dummy_env.reset() @@ -568,9 +556,7 @@ async def test_complex_system_prompt( class TestHTTPAgentClient: @patch.dict("os.environ", {"AUTH_TOKEN": "stub"}) @pytest.mark.asyncio - @pytest.mark.flaky( # Rerun if LLM call does not return expected result - reruns=3, only_on=[AssertionError] - ) + @pytest.mark.vcr async def test_lifecycle(self, dummy_env: DummyEnv) -> None: obs, tools = await dummy_env.reset() # Let's turn the prompt to require multiple steps diff --git a/tests/test_llms.py b/tests/test_llms.py index 516baff5..2d524175 100644 --- a/tests/test_llms.py +++ b/tests/test_llms.py @@ -72,6 +72,7 @@ async def test_achat(model_name: str) -> None: "model_name", [CILLMModelNames.OPENAI.value, CILLMModelNames.ANTHROPIC.value] ) @pytest.mark.asyncio +@pytest.mark.vcr async def test_tools(dummy_env: DummyEnv, model_name: str) -> None: model = LLMModel(name=model_name) messages = [ @@ -232,7 +233,7 @@ def play(move: int | None) -> None: assert result.messages[0].tool_calls[0].function.arguments["move"] is None @pytest.mark.asyncio - @pytest.mark.flaky(reruns=3, only_on=[AssertionError]) + @pytest.mark.vcr async def test_output_schema(self) -> None: model = self.MODEL_CLS(name="gpt-3.5-turbo", config=self.DEFAULT_CONFIG) messages = [ @@ -252,7 +253,7 @@ async def test_output_schema(self) -> None: @pytest.mark.parametrize("model_name", [CILLMModelNames.OPENAI.value]) @pytest.mark.asyncio - @pytest.mark.flaky(reruns=3, only_on=[AssertionError]) + @pytest.mark.vcr async def test_text_image_message(self, model_name: str) -> None: model = self.MODEL_CLS(name=model_name, config=self.DEFAULT_CONFIG) @@ -334,6 +335,6 @@ class InstructionList(BaseModel): [CILLMModelNames.ANTHROPIC.value, "gpt-4-turbo", CILLMModelNames.OPENAI.value], ) @pytest.mark.asyncio - @pytest.mark.flaky(reruns=3, only_on=[AssertionError]) + @pytest.mark.vcr async def test_text_image_message(self, model_name: str) -> None: await super().test_text_image_message(model_name) diff --git a/tests/test_modules.py b/tests/test_modules.py index 5ef93ba5..5234f4ca 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -19,6 +19,7 @@ @pytest.mark.asyncio +@pytest.mark.vcr async def test_reflect_module() -> None: config = ReflectModuleConfig( llm_model={"model": CILLMModelNames.ANTHROPIC.value, "temperature": 0}