[WIP] Add first flow template incl test framework based on stepzen snippets

flow-templates/.gitignore

@@ -0,0 +1,2 @@
+node_modules
+.env

flow-templates/__tests/.env.sample

@@ -0,0 +1 @@
+WXFLOWS_ENDPOINT=

flow-templates/__tests/README.md

@@ -0,0 +1,29 @@
+# Testing
+
+Initial test framework for flow templates.
+
+# Running
+
+Must have one environment variable set:
+
+ * `WXFLOWS_ENDPOINT` - wxflows endpoint all the tests will be deployed to (one at a time). For example:
+   ```
+   export WXFLOWS_ENDPOINT=https://YOUR_ACCOUNT.YOUR_REGION.stepzen.net/test/snippets/graphql
+   ```
+
+You can also set this environment variable in a `.env` file in the `tests` folder. By copying the `.env.example` file.
+
+Must be logged into your wxflows account (matching `YOUR_ACCOUNT` in `WXFLOWS_ENDPOINT`):
+
+```
+wxflows login
+```
+
+Execute at the root of the snippets repo:
+
+```
+cd tests
+npm ci
+npm test
+```
+

flow-templates/__tests/gqltest.js

@@ -0,0 +1,148 @@
+require("dotenv").config();
+const fetch = require("node-fetch");
+const { expect } = require("chai");
+const { execSync } = require("child_process");
+const { URL } = require("url");
+const path = require("node:path");
+
+const authTypes = {
+  adminKey: 1,
+  apiKey: 2,
+  jwt: 3,
+  noAuth: 4,
+};
+Object.freeze(authTypes);
+
+// We use admin key to test because there is a cache optimization for apikey's that is not conducive
+// to rapid deploy and run cycles that occur with this type of testing
+const adminKey =
+  `apikey ` + execSync(`wxflows whoami --adminkey`).toString().trim();
+const apiKey =
+  `apikey ` + execSync(`wxflows whoami --apikey`).toString().trim();
+
+const endpoint = process.env.WXFLOWS_ENDPOINT;
+
+// deploys the schema in a directory to a wxflows endpoint.
+// endpoint is the full URL.
+function deployEndpoint(endpoint, dirname) {
+  console.log(`deploying ${endpoint} from ${dirname}`);
+  const url = new URL(endpoint);
+  const endpointPath = url.pathname
+    .replace(/^\//, "")
+    .replace(/\/__graphql$/, "");
+
+  const configFilePath = dirname
+    .replace(/\/__tests$/, "");
+
+  const cmd = `wxflows deploy --configuration-file=${configFilePath}/wxflows.toml`;
+
+  // simple retry with backoff to avoid a deploy error
+  // when another client (typically a test) has deployed
+  // causing a 'Conflict: content has changed' error.
+  const stdout = execSync(
+    `${cmd} || (sleep 1; ${cmd}) || (sleep 3; ${cmd})`
+  ).toString();
+  console.log(stdout);
+}
+
+// Runs a GraphQL request against the endpoint
+// as a test returning the response.
+// The test will fail if the request does not
+// have status 200 or has any GraphQL errors.
+function runGqlOk(authType, endpoint, query, variables, operationName) {
+  switch (authType) {
+    case authTypes.adminKey:
+      authValue = adminKey;
+      break;
+    case authTypes.apiKey:
+      authValue = apiKey;
+      break;
+    //  Have not  implemented jwt and noAuth yet
+    case authTypes.jwt:
+    case authTypes.noAuth:
+    default:
+      authValue = "";
+  }
+  return fetch(endpoint, {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: authValue,
+    },
+    body: JSON.stringify({
+      query: query,
+      variables: variables,
+      operationName: operationName,
+    }),
+  })
+    .then(function (result) {
+      expect(result.status).to.equal(200);
+      return result;
+    })
+    .then(function (result) {
+      return result.json();
+    })
+    .then(function (response) {
+      expect(response.errors, `no errors should exist: ${JSON.stringify(response.errors)}`).to.be.undefined;
+      return response;
+    });
+}
+
+// tests that the data in a GraphQL response exists
+function expectData(response, value, assert) {
+  expect(response.data).to[assert](value);
+}
+
+// tests that the key in a GraphQL response exists
+function expectField(response, field) {
+  expect(response.data).to.have.nested.property(field);
+}
+
+// deploys graphql schema located in dirname to the test endpoint provided by the environment (process.env.WXFLOWS_ENDPOINT),
+// and then runs through all fo the field selection tests.
+function deployAndRun(dirname, tests) {
+  it("deploy", function () {
+    // deployEndpoint will try up to three times to deploy
+    // the schema with a backoff that can total four seconds.
+    // So set the timeout to be (3*10)+4 seconds to cover a worst case scenario.
+    this.timeout(34000);
+    return deployEndpoint(endpoint, dirname);
+  });
+
+  tests.forEach(
+    ({ label, query, variables, operationName, expectedResponse, expectedFields, authType, assert = 'equal' }) => {
+      it(label, function () {
+        this.timeout(8000); // Occasional requests take > 2s
+
+        return runGqlOk(
+          authType,
+          endpoint,
+          query,
+          variables,
+          operationName
+        ).then(function (response) {
+          if (expectedResponse) {
+            expectData(response, expectedResponse, assert);
+          }
+          if (expectedFields && expectedFields.length > 0) {
+            expectedFields.forEach((field) => expectField(response, field))
+          }
+        });
+      });
+    }
+  );
+}
+
+function getTestDescription(testRoot, fullDirName) {
+  segments = fullDirName.split(path.sep);
+  rootIndex = segments.findIndex((element) => element == testRoot);
+  // Construct the test description from the unique path from testRoot, which is likely the root of the git repo.
+  // Intentionally not using `path.sep` as this is not a path to a file now, but a test description.
+  return segments.slice(rootIndex + 1, -1).join("/");
+}
+
+exports.runGqlOk = runGqlOk;
+exports.expectData = expectData;
+exports.deployAndRun = deployAndRun;
+exports.authTypes = authTypes;
+exports.getTestDescription = getTestDescription;

flow-templates/__tests/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "wxflows-flow-templates-test",
+  "version": "0.0.1",
+  "type": "commonjs",
+  "scripts": {
+    "test": "mocha ../**/__tests/*test.js"
+  },
+  "dependencies": {
+    "chai": "^4.3.6",
+    "chain": "^0.2.1",
+    "dotenv": "^16.0.3",
+    "mocha": "^10.0.0",
+    "node-fetch": "~2.6.1"
+  },
+  "keywords": [],
+  "author": ""
+}

flow-templates/summarization/.env.sample

@@ -0,0 +1,17 @@
+# -- Auto-generated once by wxflows init. Edit as required and save as .env --
+# STEPZEN_BAM_GENAI_TOKEN=
+# STEPZEN_ELASTIC_BM25_BASICAUTH=
+# STEPZEN_ELASTIC_BM25_CACERT=
+# STEPZEN_ELASTIC_BM25_CLUSTER=
+# STEPZEN_ELASTIC_BM25_TLS=
+# STEPZEN_MILVUS_ADDRESS=
+# STEPZEN_MILVUS_COLLECTION=
+# STEPZEN_MILVUS_PASSWORD=
+# STEPZEN_MILVUS_USERNAME=
+# STEPZEN_VECTORSERVICES_ACCOUNT=
+# STEPZEN_VECTORSERVICES_APIKEY=
+# STEPZEN_WATSONX_AI_TOKEN=
+# STEPZEN_WATSONX_HOST=
+# STEPZEN_WATSONX_PROJECTID=
+# STEPZEN_ZILLIS_API_KEY=
+# STEPZEN_ZILLIS_CLUSTER=

flow-templates/summarization/__tests/test.js

@@ -0,0 +1,31 @@
+const {
+    deployAndRun,
+    authTypes,
+    getTestDescription,
+} = require('../../__tests/gqltest.js');
+
+testDescription = getTestDescription('flow-templates', __dirname);
+
+describe(testDescription, function () {
+    const tests = [
+        {
+            label: 'summarization',
+            query: `{ summarization(
+                        question: "What is Lorem Ipsum? Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum. Why do we use it? It is a long established fact that a reader will be distracted by the readable content of a page when looking at its layout. The point of using Lorem Ipsum is that it has a more-or-less normal distribution of letters, as opposed to using 'Content here, content here', making it look like readable English. Many desktop publishing packages and web page editors now use Lorem Ipsum as their default model text, and a search for 'lorem ipsum' will uncover many web sites still in their infancy. Various versions have evolved over the years, sometimes by accident, sometimes on purpose (injected humour and the like). Where does it come from? Contrary to popular belief, Lorem Ipsum is not simply random text. It has roots in a piece of classical Latin literature from 45 BC, making it over 2000 years old. Richard McClintock, a Latin professor at Hampden-Sydney College in Virginia, looked up one of the more obscure Latin words, consectetur, from a Lorem Ipsum passage, and going through the cites of the word in classical literature, discovered the undoubtable source. Lorem Ipsum comes from sections 1.10.32 and 1.10.33 of 'de Finibus Bonorum et Malorum' (The Extremes of Good and Evil) by Cicero, written in 45 BC. This book is a treatise on the theory of ethics, very popular during the Renaissance. The first line of Lorem Ipsum, 'Lorem ipsum dolor sit amet..', comes from a line in section 1.10.32."
+                        parameters: {
+                            min_new_tokens: 100, 
+                            max_new_tokens: 1000, 
+                            stop_sequences: []
+                        }
+                    ) {
+                        out
+                    }
+                }`,
+            expectedFields: ['summarization.out.results[0].generated_text', 'summarization.out.results[0].generated_token_count'],
+            expected: null,
+            authType: authTypes.adminKey,
+            assert: 'include'
+        }
+    ];
+    return deployAndRun(__dirname, tests);
+});

flow-templates/summarization/summarization.flow

@@ -0,0 +1 @@
+summarization = templatedPrompt(promptTemplate: "<|system|>You are Granite Chat, an AI language model developed by IBM. You are a cautious assistant. You carefully follow instructions. You are helpful and harmless and you follow ethical guidelines and promote positive behavior.<|user|>Create a concise summary of the document provided below.[Document]{question}[End]<|assistant|>") | completion(parameters: summarization.parameters)

flow-templates/summarization/wxflows.toml

@@ -0,0 +1,8 @@
+[wxflows.deployment]
+endpoint_name = "wxflows-genai/summarization"
+flows="""
+    summarization = templatedPrompt(promptTemplate: "<|system|>You are Granite Chat, an AI language model developed by IBM. You are a cautious assistant. You carefully follow instructions. You are helpful and harmless and you follow ethical guidelines and promote positive behavior.<|user|>Create a concise summary of the document provided below.[Document]{question}[End]<|assistant|>") | completion(parameters: summarization.parameters)
+"""
+# flow_files=["summarization.flow"]
+ai_engine = "WATSONX"
+