Add cloud-tests (#206)

* Add cloud-tests

* Fix

* Add readme.md

package.json

@@ -5,11 +5,13 @@
     "repository": "https://github.com/cohere-ai/cohere-typescript",
     "main": "./index.js",
     "types": "./index.d.ts",
+    "bin": "./src/test/cloud-tests/script.ts",
     "scripts": {
         "format": "prettier . --write --ignore-unknown",
         "build": "tsc",
         "prepack": "cp -rv dist/. .",
-        "test": "jest"
+        "test": "jest",
+        "cloud-tests": "npx ts-node src/test/cloud-tests/script.ts"
     },
     "dependencies": {
         "url-join": "4.0.1",

src/test/cloud-tests/README.MD

@@ -0,0 +1,16 @@
+# Cloud Tests
+
+This directory contains tests that you can run on your cohere container to check that it supports all of the parameters that we would expect it to support.
+
+## Running the tests
+
+Running the tests from npm is very simple. Just run the following command:
+
+```
+npx cohere-ai@latest \
+    --baseUrl https://api.cohere.com \
+    --apiKey my_key \
+    --featuresToTest chat,embed,rerank,chat-2,rerank-2,embed-2
+```
+
+This will automatically download the latest test suite and execute them against whichever local service or contaienr you are running. Often the apiKey will be optional if you are running the tests against a local service.

src/test/cloud-tests/chat-2.test.ts

@@ -0,0 +1,45 @@
+import { expect } from "@jest/globals";
+import { cohere, skipIfTestTypeIsnt } from "./utils";
+
+
+skipIfTestTypeIsnt("chat-2")("chat-2", () => {
+    it("chat", async () => {
+        const response = await cohere.v2.chat({
+            model: "command-r-plus",
+            messages: [
+                {
+                    role: "user",
+                    content: "hello world!",
+                },
+            ],
+        });
+
+        expect(response.message).toBeDefined();
+    });
+
+    it("chatStream", async () => {
+        const stream = await cohere.v2.chatStream({
+            model: "command-r-plus",
+            messages: [
+                {
+                    role: "user",
+                    content: "hello world!",
+                },
+            ],
+        });
+
+        const events = new Set<string>();
+
+        for await (const chatEvent of stream) {
+            if (chatEvent) {
+                events.add(chatEvent.type);
+            }
+        }
+
+        expect(events.has("message-start")).toBeTruthy();
+        expect(events.has("content-start")).toBeTruthy();
+        expect(events.has("content-delta")).toBeTruthy();
+        expect(events.has("content-end")).toBeTruthy();
+        expect(events.has("message-end")).toBeTruthy();
+    });
+});

src/test/cloud-tests/chat.test.ts

@@ -0,0 +1,28 @@
+import { expect, test } from "@jest/globals";
+import { cohere, skipIfTestTypeIsnt } from "./utils";
+
+skipIfTestTypeIsnt("chat")("chat", () => {
+    test.concurrent("stream works", async () => {
+        const chat = await cohere.chatStream({
+            message: "Hello world!",
+        });
+
+        const chunks = [];
+
+        for await (const chunk of chat) {
+            chunks.push(chunk);
+        }
+
+        expect(chunks[0].eventType).toMatchInlineSnapshot(`"stream-start"`);
+        expect(chunks[1].eventType).toMatchInlineSnapshot(`"text-generation"`);
+        expect(chunks[chunks.length - 1].eventType).toMatchInlineSnapshot(`"stream-end"`);
+    });
+
+    test.concurrent("works", async () => {
+        const chat = await cohere.chat({
+            message: "Hello world!",
+        });
+
+        expect(chat.text).toBeDefined();
+    });
+});

src/test/cloud-tests/convict.ts

@@ -0,0 +1,32 @@
+import convict from "convict";
+
+const testables= ["chat","embed","rerank","chat-2","rerank-2","embed-2"]
+
+var config = convict({
+    baseUrl: {
+        doc: "The base url of the API eg https://api.cohere.com",
+        format: String,
+        default: "https://api.cohere.com",
+        env: "BASE_URL",
+        arg: "baseUrl"
+    },
+    apiKey: {
+        doc: "The API key",
+        format: String,
+        default: "none",
+        env: "CO_API_KEY",
+        arg: "apiKey"
+    },
+    featuresToTest: {
+        doc: `The features to test, can  be any of ${testables.join(", ")}`,
+        format: Array,
+        default: [],
+        env: "FEATURES_TO_TEST",
+        arg: "featuresToTest"
+    }
+});
+
+// Perform validation
+config.validate({ allowed: 'strict' });
+export { config };
+

src/test/cloud-tests/embed-2.test.ts

@@ -0,0 +1,15 @@
+import { test } from "@jest/globals";
+import { cohere, skipIfTestTypeIsnt } from "./utils";
+
+skipIfTestTypeIsnt("embed-2")("embed-2", () => {
+    test.concurrent("works", async () => {
+        const embed = await cohere.v2.embed({
+            texts: ["hello", "goodbye"],
+            model: "embed-english-v3.0",
+            inputType:"search_document",
+            embeddingTypes: ["float"]
+        });
+
+        expect(embed.embeddings.float).toBeDefined();
+    });
+});

src/test/cloud-tests/embed.test.ts

@@ -0,0 +1,14 @@
+import { test } from "@jest/globals";
+import { cohere, skipIfTestTypeIsnt } from "./utils";
+
+skipIfTestTypeIsnt("embed")("embed", () => {
+    test.concurrent("works", async () => {
+        const embed = await cohere.embed({
+            texts: ["hello", "goodbye"],
+            model: "embed-english-v3.0",
+            inputType: "search_document",
+        });
+
+        expect(embed.embeddings).toBeDefined();
+    });
+});

src/test/cloud-tests/rerank-2.test.ts

@@ -0,0 +1,26 @@
+import { test } from "@jest/globals";
+import { cohere, skipIfTestTypeIsnt } from "./utils";
+
+skipIfTestTypeIsnt("rerank-2")("rerank-2", () => {
+    test.concurrent("works", async () => {
+        const rerank = await cohere.v2.rerank({
+            documents: [
+                { text: "Carson City is the capital city of the American state of Nevada." },
+                {
+                    text: "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
+                },
+                {
+                    text: "Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district.",
+                },
+                {
+                    text: "Capital punishment (the death penalty) has existed in the United States since beforethe United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states.",
+                },
+            ],
+            query: "What is the capital of the United States?",
+            topN: 3,
+            model: "rerank-english-v3.0"
+        });
+
+        expect(rerank.results).toBeDefined();
+    });
+});

src/test/cloud-tests/rerank.test.ts

@@ -0,0 +1,25 @@
+import { test } from "@jest/globals";
+import { cohere, skipIfTestTypeIsnt } from "./utils";
+
+skipIfTestTypeIsnt("rerank")("rerank", () => {
+    test.concurrent("works", async () => {
+        const rerank = await cohere.rerank({
+            documents: [
+                { text: "Carson City is the capital city of the American state of Nevada." },
+                {
+                    text: "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.",
+                },
+                {
+                    text: "Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district.",
+                },
+                {
+                    text: "Capital punishment (the death penalty) has existed in the United States since beforethe United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states.",
+                },
+            ],
+            query: "What is the capital of the United States?",
+            topN: 3,
+        });
+
+        expect(rerank).toBeDefined();
+    });
+});

src/test/cloud-tests/script.ts

@@ -0,0 +1,15 @@
+#!/usr/bin/env ts-node
+
+import * as jest from "jest-cli"
+import { config } from "./convict"
+
+process.env.BASE_URL = config.get("baseUrl")
+process.env.CO_API_KEY = config.get("apiKey")
+process.env.FEATURES_TO_TEST = config.get("featuresToTest").join(",")
+
+console.log("Running tests with the following configuration:")
+console.log(`BASE_URL: ${process.env.BASE_URL}`)
+console.log(`CO_API_KEY: ${process.env.CO_API_KEY}`)
+console.log(`FEATURES_TO_TEST: ${process.env.FEATURES_TO_TEST}`)
+
+jest.run(["./src/test/cloud-tests"])

src/test/cloud-tests/utils.ts

@@ -0,0 +1,19 @@
+import { CohereClient } from "../../index";
+import { config } from "./convict";
+
+export const skipIf = (condition: boolean) => {
+    return condition ? describe.skip : describe;
+}
+
+export const skipIfTestTypeIsnt = (testType: string) => {
+    const featuresToTest = config.get("featuresToTest") as string[];
+    const shouldSkip = !featuresToTest.includes(testType);
+
+    return shouldSkip ? describe.skip : describe;
+}
+
+export const cohere = new CohereClient({
+    token: config.get('apiKey'),
+    environment: config.get('baseUrl'),
+    clientName: "typescript-e2e",
+});