From 96bd422c61c21e681faf54c16012b1aa800a9788 Mon Sep 17 00:00:00 2001 From: Prateek Rungta Date: Fri, 11 Apr 2025 15:29:46 -0400 Subject: [PATCH] Add environment variables to configure SSL certificate handling When using LLM behind corporate proxies or firewalls that perform SSL inspection (like Zscaler), HTTPS certificate validation can fail with connection errors. This adds support for two environment variables: - LLM_SSL_CONFIG: Configure SSL verification behavior ('native_tls' or 'no_verify') - LLM_CA_BUNDLE: Path to a custom CA certificate bundle file The implementation adds a helper function that configures the OpenAI client's HTTP transport based on these settings. Fixes #772 --- docs/index.md | 13 +++ docs/setup.md | 80 +++++++++++++- llm/default_plugins/openai_models.py | 62 ++++++++++- tests/test_openai_ssl_config.py | 157 +++++++++++++++++++++++++++ 4 files changed, 310 insertions(+), 2 deletions(-) create mode 100644 tests/test_openai_ssl_config.py diff --git a/docs/index.md b/docs/index.md index 3034d5ed9..8e5fcd4c3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -33,19 +33,27 @@ First, install LLM using `pip` or Homebrew or `pipx`: ```bash pip install llm ``` + Or with Homebrew (see {ref}`warning note `): + ```bash brew install llm ``` + Or with [pipx](https://pypa.github.io/pipx/): + ```bash pipx install llm ``` + Or with [uv](https://docs.astral.sh/uv/guides/tools/) + ```bash uv tool install llm ``` + If you have an [OpenAI API key](https://platform.openai.com/api-keys) key you can run this: + ```bash # Paste your OpenAI API key into this llm keys set openai @@ -59,7 +67,9 @@ llm "extract text" -a scanned-document.jpg # Use a system prompt against a file cat myfile.py | llm -s "Explain this code" ``` + Or you can {ref}`install a plugin ` and use models that can run on your local device: + ```bash # Install the plugin llm install llm-gpt4all @@ -67,10 +77,13 @@ llm install llm-gpt4all # Download and run a prompt against the Orca Mini 7B model llm -m orca-mini-3b-gguf2-q4_0 'What is the capital of France?' ``` + To start {ref}`an interactive chat ` with a model, use `llm chat`: + ```bash llm chat -m gpt-4o ``` + ``` Chatting with gpt-4o Type 'exit' or 'quit' to exit diff --git a/docs/setup.md b/docs/setup.md index 72801ba68..2c4d9874e 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -3,18 +3,25 @@ ## Installation Install this tool using `pip`: + ```bash pip install llm ``` + Or using [pipx](https://pypa.github.io/pipx/): + ```bash pipx install llm ``` + Or using [uv](https://docs.astral.sh/uv/guides/tools/) ({ref}`more tips below `): + ```bash uv tool install llm ``` + Or using [Homebrew](https://brew.sh/) (see {ref}`warning note `): + ```bash brew install llm ``` @@ -22,27 +29,37 @@ brew install llm ## Upgrading to the latest version If you installed using `pip`: + ```bash pip install -U llm ``` + For `pipx`: + ```bash pipx upgrade llm ``` + For `uv`: + ```bash uv tool upgrade llm ``` + For Homebrew: + ```bash brew upgrade llm ``` + If the latest version is not yet available on Homebrew you can upgrade like this instead: + ```bash llm install -U llm ``` (setup-uvx)= + ## Using uvx If you have [uv](https://docs.astral.sh/uv/) installed you can also use the `uvx` command to try LLM without first installing it like this: @@ -51,20 +68,25 @@ If you have [uv](https://docs.astral.sh/uv/) installed you can also use the `uvx export OPENAI_API_KEY='sx-...' uvx llm 'fun facts about skunks' ``` + This will install and run LLM using a temporary virtual environment. You can use the `--with` option to add extra plugins. To use Anthropic's models, for example: + ```bash export ANTHROPIC_API_KEY='...' uvx --with llm-anthropic llm -m claude-3.5-haiku 'fun facts about skunks' ``` + All of the usual LLM commands will work with `uvx llm`. Here's how to set your OpenAI key without needing an environment variable for example: + ```bash uvx llm keys set openai # Paste key here ``` (homebrew-warning)= + ## A note about Homebrew and PyTorch The version of LLM packaged for Homebrew currently uses Python 3.12. The PyTorch project do not yet have a stable release of PyTorch for that version of Python. @@ -80,6 +102,7 @@ llm python -m pip install \ --index-url https://download.pytorch.org/whl/nightly/cpu llm install llm-sentence-transformers ``` + This should produce a working installation of that plugin. ## Installing plugins @@ -87,11 +110,13 @@ This should produce a working installation of that plugin. {ref}`plugins` can be used to add support for other language models, including models that can run on your own device. For example, the [llm-gpt4all](https://github.com/simonw/llm-gpt4all) plugin adds support for 17 new models that can be installed on your own machine. You can install that like so: + ```bash llm install llm-gpt4all ``` (api-keys)= + ## API key management Many LLM models require an API key. These API keys can be provided to this tool using several different mechanisms. @@ -105,11 +130,14 @@ The easiest way to store an API key is to use the `llm keys set` command: ```bash llm keys set openai ``` + You will be prompted to enter the key like this: + ``` % llm keys set openai Enter key: ``` + Once stored, this key will be automatically used for subsequent calls to the API: ```bash @@ -137,10 +165,13 @@ Keys can be passed directly using the `--key` option, like this: ```bash llm "Five names for pet weasels" --key sk-my-key-goes-here ``` + You can also pass the alias of a key stored in the `keys.json` file. For example, if you want to maintain a personal API key you could add that like this: + ```bash llm keys set personal ``` + And then use it for prompts like so: ```bash @@ -156,6 +187,7 @@ For OpenAI models the key will be read from the `OPENAI_API_KEY` environment var The environment variable will be used if no `--key` option is passed to the command and there is not a key configured in `keys.json` To use an environment variable in place of the `keys.json` key run the prompt like this: + ```bash llm 'my prompt' --key $OPENAI_API_KEY ``` @@ -165,6 +197,7 @@ llm 'my prompt' --key $OPENAI_API_KEY You can configure LLM in a number of different ways. (setup-default-model)= + ### Setting a custom default model The model used when calling `llm` without the `-m/--model` option defaults to `gpt-4o-mini` - the fastest and least expensive OpenAI model. @@ -174,10 +207,13 @@ You can use the `llm models default` command to set a different default model. F ```bash llm models default gpt-4o ``` + You can view the current model by running this: + ``` llm models default ``` + Any of the supported aliases for a model can be passed to this command. ### Setting a custom directory location @@ -193,16 +229,58 @@ You can set a custom location for this directory by setting the `LLM_USER_PATH` ```bash export LLM_USER_PATH=/path/to/my/custom/directory ``` + +(ssl-certificate-configuration)= + +### SSL Certificate Configuration + +When using LLM behind a corporate proxy or firewall (like Zscaler), you may encounter SSL certificate validation issues. You can configure SSL handling using environment variables: + +```bash +# Use your system's native certificate store (similar to UV's --native-tls option) +export LLM_SSL_CONFIG=native_tls + +# Or use a specific certificate bundle +export LLM_CA_BUNDLE=/path/to/certificate.pem +``` + +
+More SSL configuration options and details + +#### Environment Variables + +- `LLM_SSL_CONFIG`: Controls SSL verification behavior + + - `native_tls`: Uses your system's native certificate store + - `no_verify`: Disables SSL verification entirely (not recommended for production) + +- `LLM_CA_BUNDLE`: Path to a custom CA certificate bundle file + +#### Finding Your Corporate Certificate + +If you're behind a corporate proxy, you may need to export the certificate from your browser or obtain it from your IT department. + +Common certificate locations: + +- macOS: `~/Library/Application Support/Certificate Authority/` +- Linux: `/etc/ssl/certs/` +- Windows: The Windows Certificate Store +
+ ### Turning SQLite logging on and off By default, LLM will log every prompt and response you make to a SQLite database - see {ref}`logging` for more details. You can turn this behavior off by default by running: + ```bash llm logs off ``` + Or turn it back on again with: + ``` llm logs on ``` -Run `llm logs status` to see the current states of the setting. \ No newline at end of file + +Run `llm logs status` to see the current states of the setting. diff --git a/llm/default_plugins/openai_models.py b/llm/default_plugins/openai_models.py index 1e272c02e..3ca41d585 100644 --- a/llm/default_plugins/openai_models.py +++ b/llm/default_plugins/openai_models.py @@ -12,6 +12,7 @@ import httpx import openai import os +import warnings from pydantic import field_validator, Field @@ -535,8 +536,16 @@ def get_client(self, key, *, async_=False): kwargs["api_key"] = "DUMMY_KEY" if self.headers: kwargs["default_headers"] = self.headers + + # Configure SSL certificate handling from environment variables + ssl_client = _configure_ssl_client(self.model_id) + + if ssl_client: + kwargs["http_client"] = ssl_client + if os.environ.get("LLM_OPENAI_SHOW_RESPONSES"): - kwargs["http_client"] = logging_client() + if "http_client" not in kwargs: + kwargs["http_client"] = logging_client() if async_: return openai.AsyncOpenAI(**kwargs) else: @@ -798,3 +807,54 @@ def redact_data(input_dict): for item in input_dict: redact_data(item) return input_dict + + +def _configure_ssl_client(model_id): + """Configure SSL certificate handling based on environment variables.""" + # Check for SSL config in environment variables + ssl_config = os.environ.get("LLM_SSL_CONFIG") + ca_bundle = os.environ.get("LLM_CA_BUNDLE") + + if not ssl_config and not ca_bundle: + return None + + # Import here to handle potential import errors + try: + from openai import DefaultHttpxClient + import httpx + except ImportError: + warnings.warn( + "Unable to import DefaultHttpxClient from openai - SSL configuration not available." + ) + return None + + # Validate ssl_config value + valid_ssl_configs = ["native_tls", "no_verify"] + if ssl_config and ssl_config not in valid_ssl_configs: + warnings.warn( + f"Invalid ssl_config value: {ssl_config}. Valid values are: {', '.join(valid_ssl_configs)}" + ) + return None + + try: + if ssl_config == "native_tls": + # Use the system's native certificate store + return DefaultHttpxClient(transport=httpx.HTTPTransport(verify=True)) + elif ssl_config == "no_verify": + # Disable SSL verification entirely (less secure) + return DefaultHttpxClient(transport=httpx.HTTPTransport(verify=False)) + elif ca_bundle: + # Check if certificate file exists + if not os.path.exists(ca_bundle): + warnings.warn(f"Certificate file not found: {ca_bundle}") + return None + else: + # Use a specific CA bundle file + return DefaultHttpxClient( + transport=httpx.HTTPTransport(verify=ca_bundle) + ) + except Exception as e: + warnings.warn(f"Error configuring SSL client: {str(e)}") + return None + + return None diff --git a/tests/test_openai_ssl_config.py b/tests/test_openai_ssl_config.py new file mode 100644 index 000000000..c6281cdec --- /dev/null +++ b/tests/test_openai_ssl_config.py @@ -0,0 +1,157 @@ +import os +from unittest import mock +import pytest + +# Import the function we're testing +from llm.default_plugins.openai_models import _configure_ssl_client + + +@pytest.fixture +def mock_environ(): + with mock.patch.dict(os.environ, {}, clear=True): + yield + + +@mock.patch("openai.DefaultHttpxClient") +@mock.patch("httpx.HTTPTransport") +def test_default_ssl_config(mock_transport, mock_client, mock_environ): + # Test that without any environment variables, no special SSL config is returned + ssl_client = _configure_ssl_client("test-model") + assert ssl_client is None + + +@mock.patch("openai.DefaultHttpxClient") +@mock.patch("httpx.HTTPTransport") +def test_env_var_native_tls(mock_transport, mock_client, mock_environ): + # Set up mocks + mock_client_instance = mock.MagicMock() + mock_client.return_value = mock_client_instance + + # Set the environment variable + os.environ["LLM_SSL_CONFIG"] = "native_tls" + + # Test the helper function + ssl_client = _configure_ssl_client("test-model") + + # Should return the mock client + assert ssl_client is mock_client_instance + # Verify transport was created with verify=True + mock_transport.assert_called_once_with(verify=True) + + +@mock.patch("openai.DefaultHttpxClient") +@mock.patch("httpx.HTTPTransport") +def test_env_var_no_verify(mock_transport, mock_client, mock_environ): + # Set up mocks + mock_client_instance = mock.MagicMock() + mock_client.return_value = mock_client_instance + + # Set the environment variable + os.environ["LLM_SSL_CONFIG"] = "no_verify" + + # Test the helper function + ssl_client = _configure_ssl_client("test-model") + + # Should return the mock client + assert ssl_client is mock_client_instance + # Verify transport was created with verify=False + mock_transport.assert_called_once_with(verify=False) + + +@mock.patch("openai.DefaultHttpxClient") +@mock.patch("httpx.HTTPTransport") +@mock.patch("os.path.exists") +def test_env_var_ca_bundle(mock_exists, mock_transport, mock_client, mock_environ): + # Set up mocks + mock_client_instance = mock.MagicMock() + mock_client.return_value = mock_client_instance + mock_exists.return_value = True + + # Set environment variable + os.environ["LLM_CA_BUNDLE"] = "/path/to/ca-bundle.pem" + + # Test the helper function + ssl_client = _configure_ssl_client("test-model") + + # Should return the mock client + assert ssl_client is mock_client_instance + # Verify transport was created with verify pointing to certificate + mock_transport.assert_called_once_with(verify="/path/to/ca-bundle.pem") + + +def test_invalid_ssl_config(mock_environ): + # Set an invalid ssl_config value + os.environ["LLM_SSL_CONFIG"] = "invalid_value" + + # Should raise a warning and return None + with pytest.warns(UserWarning, match="Invalid ssl_config value"): + ssl_client = _configure_ssl_client("test-model") + assert ssl_client is None + + +@mock.patch("os.path.exists") +def test_missing_ca_bundle(mock_exists, mock_environ): + # Set a non-existent certificate file + os.environ["LLM_CA_BUNDLE"] = "/nonexistent/path/to/cert.pem" + mock_exists.return_value = False + + # Should raise a warning and return None + with pytest.warns(UserWarning, match="Certificate file not found"): + ssl_client = _configure_ssl_client("test-model") + assert ssl_client is None + + +# Integration test with mocked dependencies +class MockShared: + def __init__(self, model_id): + self.model_id = model_id + self.needs_key = None + self.api_base = None + self.api_type = None + self.api_version = None + self.api_engine = None + self.headers = None + + def get_key(self, key): + return "mock-key" + + def get_client(self, key, *, async_=False): + # Simple implementation that should call _configure_ssl_client + # and pass the result to OpenAI + from llm.default_plugins.openai_models import _configure_ssl_client, openai + + kwargs = {"api_key": self.get_key(key)} + + ssl_client = _configure_ssl_client(self.model_id) + if ssl_client: + kwargs["http_client"] = ssl_client + + if async_: + return openai.AsyncOpenAI(**kwargs) + else: + return openai.OpenAI(**kwargs) + + +@mock.patch("llm.default_plugins.openai_models._Shared", MockShared) +@mock.patch("llm.default_plugins.openai_models._configure_ssl_client") +@mock.patch("llm.default_plugins.openai_models.openai.OpenAI") +def test_get_client_with_ssl(mock_openai, mock_ssl_client): + # Import shared class after mocking + from llm.default_plugins.openai_models import _Shared + + # Set up a mock ssl client + mock_ssl = mock.MagicMock() + mock_ssl_client.return_value = mock_ssl + + # Create a client + shared = _Shared("test-model") + shared.needs_key = "openai" + shared.get_client(key="dummy-key") + + # _configure_ssl_client should be called with the model_id + mock_ssl_client.assert_called_once_with("test-model") + + # OpenAI client should be called with http_client parameter + mock_openai.assert_called_once() + kwargs = mock_openai.call_args[1] + assert kwargs["http_client"] == mock_ssl