24_pairwise_comparison.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Zuerst erstellen wir ein Dataset.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Animal:\n",
    "    def __init__(self, name: str, description: str):\n",
    "        self.name = name\n",
    "        self.description = description\n",
    "\n",
    "    def __repr__(self):\n",
    "        return f\"Animal(name='{self.name}', description='{self.description}')\"\n",
    "\n",
    "\n",
    "# Liste der Tiere mit deutschen Beschreibungen\n",
    "animals = [\n",
    "    Animal(\n",
    "        \"Löwe\",\n",
    "        \"Der Löwe ist ein großes, kräftiges Raubtier und bekannt als König der Tiere. Er lebt hauptsächlich in Afrika und jagt in Rudeln. Männliche Löwen haben eine markante Mähne. Sie sind für ihre Stärke und ihr lautes Brüllen bekannt.\",\n",
    "    ),\n",
    "    Animal(\n",
    "        \"Elefant\",\n",
    "        \"Der Elefant ist das größte Landtier der Welt und lebt in Afrika und Asien. Er hat große Ohren, einen langen Rüssel und dicke Beine. Elefanten sind für ihre Intelligenz und sozialen Bindungen bekannt. Sie leben oft in Herden und kommunizieren über lange Distanzen.\",\n",
    "    ),\n",
    "    Animal(\n",
    "        \"Panda\",\n",
    "        \"Der Panda ist ein großes, schwarz-weißes Säugetier, das hauptsächlich Bambus frisst. Er stammt aus China und ist ein Symbol für den Artenschutz. Pandas sind ruhig und verbringen die meiste Zeit mit der Nahrungssuche. Sie sind eine bedrohte Tierart.\",\n",
    "    ),\n",
    "    Animal(\n",
    "        \"Wolf\",\n",
    "        \"Der Wolf ist ein soziales Raubtier, das in Rudeln lebt und ein Vorfahre des Haushundes ist. Wölfe sind intelligente Jäger und kommunizieren über Heulen. Sie kommen in Nordamerika, Europa und Asien vor. Sie haben ein starkes Gemeinschaftsgefühl.\",\n",
    "    ),\n",
    "    Animal(\n",
    "        \"Känguru\",\n",
    "        \"Das Känguru ist ein Beuteltier, das in Australien heimisch ist und sich hüpfend fortbewegt. Es hat kräftige Hinterbeine und einen langen Schwanz. Kängurus leben oft in Gruppen und tragen ihre Jungen im Beutel. Sie sind Pflanzenfresser und an trockenes Klima angepasst.\",\n",
    "    ),\n",
    "]\n",
    "\n",
    "# Ausgabe der Liste\n",
    "print(animals)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Und laden Examples in das Dataset hoch.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langsmith import Client\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv()\n",
    "\n",
    "\n",
    "client = Client()\n",
    "\n",
    "dataset_name = \"animals\"\n",
    "if not client.has_dataset(dataset_name=dataset_name):\n",
    "    client.create_dataset(dataset_name=dataset_name)\n",
    "\n",
    "\n",
    "client.create_examples(\n",
    "    dataset_name=dataset_name,\n",
    "    inputs=animals,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Als nächstes erstellen wir uns eine Demo chain zum Übersetzen der Texte.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.prompts import PromptTemplate\n",
    "from langchain.chat_models import init_chat_model\n",
    "from anthropic import RateLimitError as AnthropicRateLimitError\n",
    "\n",
    "configurable_model = init_chat_model(\n",
    "    model=\"gpt-4o-mini\",\n",
    "    temperature=0,\n",
    "    configurable_fields=(\"model\", \"model_provider\", \"temperature\", \"max_tokens\"),\n",
    ")\n",
    "\n",
    "prompt = PromptTemplate.from_template(\"\"\"\n",
    "Übersetze die folgenden deutschen Beschreibungen von Tieren ins Englische und füge dabei keine Kommentare hinzu:\n",
    "                                      \n",
    "{description}\n",
    "                                                                                                                               \n",
    "\"\"\")\n",
    "\n",
    "configurable_chain = prompt | configurable_model | (lambda x: {\"animal\": x.content})\n",
    "\n",
    "\n",
    "async def oai_experiment(input: dict) -> dict:\n",
    "    return await configurable_chain.ainvoke(input)\n",
    "\n",
    "\n",
    "async def anthropic_experiment(input: dict) -> dict:\n",
    "    return (\n",
    "        await configurable_chain.with_config(model=\"claude-3-5-sonnet-20240620\")\n",
    "        .with_retry(\n",
    "            retry_if_exception_type=(\n",
    "                AnthropicRateLimitError,\n",
    "            ),  # Careful, must be a Tuple\n",
    "            wait_exponential_jitter=True,\n",
    "            stop_after_attempt=10,\n",
    "        )\n",
    "        .ainvoke(input)\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Jetzt führen wir die Experimente aus und speichern die Ergebnisse.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langsmith import aevaluate\n",
    "\n",
    "anthropic_experiment_results = await aevaluate(\n",
    "    anthropic_experiment,\n",
    "    data=dataset_name,\n",
    "    experiment_prefix=\"sonnet-3.5\",\n",
    ")\n",
    "print(f\"anthropic experiment name: {anthropic_experiment_results.experiment_name}\")\n",
    "oai_experiment_results = await aevaluate(\n",
    "    oai_experiment,\n",
    "    data=dataset_name,\n",
    "    experiment_prefix=\"gpt-4o\",\n",
    ")\n",
    "print(f\"oai experiment name: {oai_experiment_results.experiment_name}\")\n",
    "experiment_names = [\n",
    "    anthropic_experiment_results.experiment_name,\n",
    "    oai_experiment_results.experiment_name,\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import uuid\n",
    "from langsmith.schemas import ComparativeExperiment\n",
    "\n",
    "comparative_experiment_id = uuid.uuid4()\n",
    "comparative_experiment_name_suffix = \"testcomparison\"\n",
    "comparate_experiment_name = (\n",
    "    \"-vs-\".join(experiment_names) + \"-\" + comparative_experiment_name_suffix\n",
    ")\n",
    "experiments = [client.read_project(project_name=exp) for exp in experiment_names]\n",
    "experiment_ids = [str(exp.id) for exp in experiments]\n",
    "ref_datasets_ = [str(exp.reference_dataset_id) for exp in experiments]\n",
    "if not len(set(ref_datasets_)) == 1:\n",
    "    raise ValueError(\"All experiments must have the same reference dataset.\")\n",
    "\n",
    "dataset_id = client.read_dataset(dataset_name=dataset_name).id\n",
    "\n",
    "response = client.request_with_retries(\"GET\", f\"/datasets/{dataset_id}/comparative\")\n",
    "comp_exp = None\n",
    "for exp in response.json():\n",
    "    if exp[\"name\"] == comparate_experiment_name:\n",
    "        print(f\"comparative experiment already exists: {exp['name']}\")\n",
    "        comp_exp = ComparativeExperiment(**exp)\n",
    "        break\n",
    "if comp_exp is None:\n",
    "    comp_exp = client.create_comparative_experiment(\n",
    "        name=comparate_experiment_name,\n",
    "        experiments=experiment_ids,\n",
    "        id=comparative_experiment_id,\n",
    "    )\n",
    "    print(f\"comparative experiment created: {comp_exp.name}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}