From b6e4c5d4c59032a615fea6d12623a881a6211917 Mon Sep 17 00:00:00 2001 From: James Sumners Date: Tue, 12 Nov 2024 14:35:43 -0500 Subject: [PATCH] chore: Updated kafkajs, langchain, & openai tests to node:test (#2723) --- .../kafkajs/{kafka.tap.js => kafka.test.js} | 191 ++++--- test/versioned/kafkajs/package.json | 2 +- test/versioned/kafkajs/utils.js | 37 +- test/versioned/langchain/common.js | 61 +- test/versioned/langchain/package.json | 8 +- ...ing.tap.js => runnables-streaming.test.js} | 367 ++++++------ test/versioned/langchain/runnables.tap.js | 445 --------------- test/versioned/langchain/runnables.test.js | 434 ++++++++++++++ test/versioned/langchain/tools.tap.js | 168 ------ test/versioned/langchain/tools.test.js | 167 ++++++ test/versioned/langchain/vectorstore.tap.js | 256 --------- test/versioned/langchain/vectorstore.test.js | 242 ++++++++ test/versioned/openai/chat-completions.tap.js | 511 ----------------- .../versioned/openai/chat-completions.test.js | 529 ++++++++++++++++++ test/versioned/openai/common.js | 64 +-- test/versioned/openai/embeddings.tap.js | 171 ------ test/versioned/openai/embeddings.test.js | 182 ++++++ .../versioned/openai/feedback-messages.tap.js | 66 --- .../openai/feedback-messages.test.js | 86 +++ test/versioned/openai/package.json | 6 +- 20 files changed, 2014 insertions(+), 1979 deletions(-) rename test/versioned/kafkajs/{kafka.tap.js => kafka.test.js} (64%) rename test/versioned/langchain/{runnables-streaming.tap.js => runnables-streaming.test.js} (59%) delete mode 100644 test/versioned/langchain/runnables.tap.js create mode 100644 test/versioned/langchain/runnables.test.js delete mode 100644 test/versioned/langchain/tools.tap.js create mode 100644 test/versioned/langchain/tools.test.js delete mode 100644 test/versioned/langchain/vectorstore.tap.js create mode 100644 test/versioned/langchain/vectorstore.test.js delete mode 100644 test/versioned/openai/chat-completions.tap.js create mode 100644 test/versioned/openai/chat-completions.test.js delete mode 100644 test/versioned/openai/embeddings.tap.js create mode 100644 test/versioned/openai/embeddings.test.js delete mode 100644 test/versioned/openai/feedback-messages.tap.js create mode 100644 test/versioned/openai/feedback-messages.test.js diff --git a/test/versioned/kafkajs/kafka.tap.js b/test/versioned/kafkajs/kafka.test.js similarity index 64% rename from test/versioned/kafkajs/kafka.tap.js rename to test/versioned/kafkajs/kafka.test.js index d585bfafe8..63f1b1572a 100644 --- a/test/versioned/kafkajs/kafka.tap.js +++ b/test/versioned/kafkajs/kafka.test.js @@ -5,28 +5,32 @@ 'use strict' -const tap = require('tap') -const helper = require('../../lib/agent_helper') -const params = require('../../lib/params') +const test = require('node:test') +const tspl = require('@matteo.collina/tspl') + const { removeModules } = require('../../lib/cache-buster') +const { assertSegments, match } = require('../../lib/custom-assertions') +const params = require('../../lib/params') +const helper = require('../../lib/agent_helper') const utils = require('./utils') -const SEGMENT_PREFIX = 'kafkajs.Kafka.consumer#' +const SEGMENT_PREFIX = 'kafkajs.Kafka.consumer#' const broker = `${params.kafka_host}:${params.kafka_port}` -tap.beforeEach(async (t) => { - t.context.agent = helper.instrumentMockedAgent({ +test.beforeEach(async (ctx) => { + ctx.nr = {} + ctx.nr.agent = helper.instrumentMockedAgent({ feature_flag: { kafkajs_instrumentation: true } }) const { Kafka, logLevel } = require('kafkajs') - t.context.Kafka = Kafka + ctx.nr.Kafka = Kafka const topic = utils.randomString() - t.context.topic = topic + ctx.nr.topic = topic const clientId = utils.randomString('kafka-test') - t.context.clientId = clientId + ctx.nr.clientId = clientId const kafka = new Kafka({ clientId, @@ -37,52 +41,47 @@ tap.beforeEach(async (t) => { const producer = kafka.producer() await producer.connect() - t.context.producer = producer + ctx.nr.producer = producer const consumer = kafka.consumer({ groupId: 'kafka' }) await consumer.connect() - t.context.consumer = consumer + ctx.nr.consumer = consumer }) -tap.afterEach(async (t) => { - helper.unloadAgent(t.context.agent) +test.afterEach(async (ctx) => { + helper.unloadAgent(ctx.nr.agent) removeModules(['kafkajs']) - await t.context.consumer.disconnect() - await t.context.producer.disconnect() + await ctx.nr.consumer.disconnect() + await ctx.nr.producer.disconnect() }) -tap.test('send records correctly', (t) => { - t.plan(8) - - const { agent, consumer, producer, topic } = t.context +test('send records correctly', async (t) => { + const plan = tspl(t, { plan: 8 }) + const { agent, consumer, producer, topic } = t.nr const message = 'test message' const expectedName = 'produce-tx' - let txCount = 0 agent.on('transactionFinished', (tx) => { - txCount++ - if (tx.name === expectedName) { - const name = `MessageBroker/Kafka/Topic/Produce/Named/${topic}` - const segment = tx.agent.tracer.getSegment() + if (tx.name !== expectedName) { + return + } - const foundSegment = segment.children.find((s) => s.name.endsWith(topic)) - t.equal(foundSegment.name, name) + const name = `MessageBroker/Kafka/Topic/Produce/Named/${topic}` + const segment = tx.agent.tracer.getSegment() - const metric = tx.metrics.getMetric(name) - t.equal(metric.callCount, 1) - const sendMetric = agent.metrics.getMetric( - 'Supportability/Features/Instrumentation/kafkajs/send' - ) - t.equal(sendMetric.callCount, 1) + const foundSegment = segment.children.find((s) => s.name.endsWith(topic)) + plan.equal(foundSegment.name, name) - const produceTrackingMetric = agent.metrics.getMetric( - `MessageBroker/Kafka/Nodes/${broker}/Produce/${topic}` - ) - t.equal(produceTrackingMetric.callCount, 1) - } + const metric = tx.metrics.getMetric(name) + plan.equal(metric.callCount, 1) + const sendMetric = agent.metrics.getMetric( + 'Supportability/Features/Instrumentation/kafkajs/send' + ) + plan.equal(sendMetric.callCount, 1) - if (txCount === 2) { - t.end() - } + const produceTrackingMetric = agent.metrics.getMetric( + `MessageBroker/Kafka/Nodes/${broker}/Produce/${topic}` + ) + plan.equal(produceTrackingMetric.callCount, 1) }) helper.runInTransaction(agent, async (tx) => { @@ -91,10 +90,10 @@ tap.test('send records correctly', (t) => { const promise = new Promise((resolve) => { consumer.run({ eachMessage: async ({ message: actualMessage }) => { - t.equal(actualMessage.value.toString(), message) - t.equal(actualMessage.headers['x-foo'].toString(), 'foo') - t.equal(actualMessage.headers.newrelic.toString(), '') - t.equal(actualMessage.headers.traceparent.toString().startsWith('00-'), true) + plan.equal(actualMessage.value.toString(), message) + plan.equal(actualMessage.headers['x-foo'].toString(), 'foo') + plan.equal(actualMessage.headers.newrelic.toString(), '') + plan.equal(actualMessage.headers.traceparent.toString().startsWith('00-'), true) resolve() } }) @@ -117,13 +116,15 @@ tap.test('send records correctly', (t) => { tx.end() }) + + await plan.completed }) -tap.test('send passes along DT headers', (t) => { +test('send passes along DT headers', async (t) => { + const plan = tspl(t, { plan: 13 }) + const { agent, consumer, producer, topic } = t.nr const expectedName = 'produce-tx' - const { agent, consumer, producer, topic } = t.context - // These agent.config lines are utilized to simulate the inbound // distributed trace that we are trying to validate. agent.config.account_id = 'account_1' @@ -143,8 +144,7 @@ tap.test('send passes along DT headers', (t) => { } if (txCount === 3) { - utils.verifyDistributedTrace({ t, consumeTxs, produceTx }) - t.end() + utils.verifyDistributedTrace({ plan, consumeTxs, produceTx }) } }) @@ -178,12 +178,13 @@ tap.test('send passes along DT headers', (t) => { tx.end() }) -}) -tap.test('sendBatch records correctly', (t) => { - t.plan(9) + await plan.completed +}) - const { agent, consumer, producer, topic } = t.context +test('sendBatch records correctly', async (t) => { + const plan = tspl(t, { plan: 9 }) + const { agent, consumer, producer, topic } = t.nr const message = 'test message' const expectedName = 'produce-tx' @@ -193,23 +194,21 @@ tap.test('sendBatch records correctly', (t) => { const segment = tx.agent.tracer.getSegment() const foundSegment = segment.children.find((s) => s.name.endsWith(topic)) - t.equal(foundSegment.name, name) + plan.equal(foundSegment.name, name) const metric = tx.metrics.getMetric(name) - t.equal(metric.callCount, 1) + plan.equal(metric.callCount, 1) - t.equal(tx.isDistributedTrace, true) + plan.equal(tx.isDistributedTrace, true) const sendMetric = agent.metrics.getMetric( 'Supportability/Features/Instrumentation/kafkajs/sendBatch' ) - t.equal(sendMetric.callCount, 1) + plan.equal(sendMetric.callCount, 1) const produceTrackingMetric = agent.metrics.getMetric( `MessageBroker/Kafka/Nodes/${broker}/Produce/${topic}` ) - t.equal(produceTrackingMetric.callCount, 1) - - t.end() + plan.equal(produceTrackingMetric.callCount, 1) } }) @@ -219,10 +218,10 @@ tap.test('sendBatch records correctly', (t) => { const promise = new Promise((resolve) => { consumer.run({ eachMessage: async ({ message: actualMessage }) => { - t.equal(actualMessage.value.toString(), message) - t.match(actualMessage.headers['x-foo'].toString(), 'foo') - t.equal(actualMessage.headers.newrelic.toString(), '') - t.equal(actualMessage.headers.traceparent.toString().startsWith('00-'), true) + plan.equal(actualMessage.value.toString(), message) + match(actualMessage.headers['x-foo'].toString(), 'foo', { assert: plan }) + plan.equal(actualMessage.headers.newrelic.toString(), '') + plan.equal(actualMessage.headers.traceparent.toString().startsWith('00-'), true) resolve() } }) @@ -247,24 +246,27 @@ tap.test('sendBatch records correctly', (t) => { tx.end() }) + + await plan.completed }) -tap.test('consume outside of a transaction', async (t) => { - const { agent, consumer, producer, topic, clientId } = t.context +test('consume outside of a transaction', async (t) => { + const plan = tspl(t, { plan: 16 }) + const { agent, consumer, producer, topic, clientId } = t.nr const message = 'test message' const txPromise = new Promise((resolve) => { agent.on('transactionFinished', (tx) => { - utils.verifyConsumeTransaction({ t, tx, topic, clientId }) + utils.verifyConsumeTransaction({ plan, tx, topic, clientId }) const sendMetric = agent.metrics.getMetric( 'Supportability/Features/Instrumentation/kafkajs/eachMessage' ) - t.equal(sendMetric.callCount, 1) + plan.equal(sendMetric.callCount, 1) const consumeTrackingMetric = agent.metrics.getMetric( `MessageBroker/Kafka/Nodes/${broker}/Consume/${topic}` ) - t.equal(consumeTrackingMetric.callCount, 1) + plan.equal(consumeTrackingMetric.callCount, 1) resolve() }) @@ -274,7 +276,7 @@ tap.test('consume outside of a transaction', async (t) => { const testPromise = new Promise((resolve) => { consumer.run({ eachMessage: async ({ message: actualMessage }) => { - t.equal(actualMessage.value.toString(), message) + plan.equal(actualMessage.value.toString(), message) resolve() } }) @@ -286,11 +288,13 @@ tap.test('consume outside of a transaction', async (t) => { messages: [{ key: 'key', value: message }] }) - return Promise.all([txPromise, testPromise]) + await Promise.all([txPromise, testPromise]) + await plan.completed }) -tap.test('consume inside of a transaction', async (t) => { - const { agent, consumer, producer, topic, clientId } = t.context +test('consume inside of a transaction', async (t) => { + const plan = tspl(t, { plan: 44 }) + const { agent, consumer, producer, topic, clientId } = t.nr const expectedName = 'testing-tx-consume' const messages = ['one', 'two', 'three'] @@ -301,11 +305,16 @@ tap.test('consume inside of a transaction', async (t) => { agent.on('transactionFinished', (tx) => { txCount++ if (tx.name === expectedName) { - t.assertSegments(tx.trace.root, [`${SEGMENT_PREFIX}subscribe`, `${SEGMENT_PREFIX}run`], { - exact: false - }) + assertSegments( + tx.trace.root, + [`${SEGMENT_PREFIX}subscribe`, `${SEGMENT_PREFIX}run`], + { + exact: false + }, + { assert: plan } + ) } else { - utils.verifyConsumeTransaction({ t, tx, topic, clientId }) + utils.verifyConsumeTransaction({ plan, tx, topic, clientId }) } if (txCount === messages.length + 1) { @@ -321,7 +330,7 @@ tap.test('consume inside of a transaction', async (t) => { consumer.run({ eachMessage: async ({ message: actualMessage }) => { msgCount++ - t.ok(messages.includes(actualMessage.value.toString())) + plan.ok(messages.includes(actualMessage.value.toString())) if (msgCount === messages.length) { resolve() } @@ -339,19 +348,25 @@ tap.test('consume inside of a transaction', async (t) => { tx.end() return Promise.all([txPromise, testPromise]) }) + + await plan.completed }) -tap.test('consume batch inside of a transaction', async (t) => { - const { agent, consumer, producer, topic } = t.context +test('consume batch inside of a transaction', async (t) => { + const plan = tspl(t, { plan: 10 }) + const { agent, consumer, producer, topic } = t.nr const expectedName = 'testing-tx-consume' const messages = ['one', 'two', 'three', 'four', 'five'] const txPromise = new Promise((resolve) => { agent.on('transactionFinished', (tx) => { - t.assertSegments(tx.trace.root, [`${SEGMENT_PREFIX}subscribe`, `${SEGMENT_PREFIX}run`], { - exact: false - }) + assertSegments( + tx.trace.root, + [`${SEGMENT_PREFIX}subscribe`, `${SEGMENT_PREFIX}run`], + { exact: false }, + { assert: plan } + ) resolve() }) }) @@ -362,23 +377,23 @@ tap.test('consume batch inside of a transaction', async (t) => { const testPromise = new Promise((resolve) => { consumer.run({ eachBatch: async ({ batch }) => { - t.equal( + plan.equal( batch.messages.length, messages.length, `should have ${messages.length} messages in batch` ) batch.messages.forEach((m) => { - t.ok(messages.includes(m.value.toString()), 'should have message') + plan.ok(messages.includes(m.value.toString()), 'should have message') }) const sendMetric = agent.metrics.getMetric( 'Supportability/Features/Instrumentation/kafkajs/eachBatch' ) - t.equal(sendMetric.callCount, 1) + plan.equal(sendMetric.callCount, 1) const consumeTrackingMetric = agent.metrics.getMetric( `MessageBroker/Kafka/Nodes/${broker}/Consume/${topic}` ) - t.equal(consumeTrackingMetric.callCount, 1) + plan.equal(consumeTrackingMetric.callCount, 1) resolve() } @@ -395,4 +410,6 @@ tap.test('consume batch inside of a transaction', async (t) => { tx.end() return Promise.all([txPromise, testPromise]) }) + + await plan.completed }) diff --git a/test/versioned/kafkajs/package.json b/test/versioned/kafkajs/package.json index 1c13a78547..e0b19adb94 100644 --- a/test/versioned/kafkajs/package.json +++ b/test/versioned/kafkajs/package.json @@ -12,7 +12,7 @@ "kafkajs": ">=2.0.0" }, "files": [ - "kafka.tap.js" + "kafka.test.js" ] } ] diff --git a/test/versioned/kafkajs/utils.js b/test/versioned/kafkajs/utils.js index 9fd9c58c82..92a9306221 100644 --- a/test/versioned/kafkajs/utils.js +++ b/test/versioned/kafkajs/utils.js @@ -4,6 +4,8 @@ */ 'use strict' + +const { assertMetrics } = require('../../lib/custom-assertions') const { makeId } = require('../../../lib/util/hashes') const utils = module.exports const metrics = require('../../lib/metrics_helper') @@ -70,14 +72,14 @@ utils.waitForConsumersToJoinGroup = ({ consumer, maxWait = 10000 }) => * and the relevant tx attributes * * @param {object} params function params - * @param {object} params.t test instance + * @param {object} params.plan assertion library instance with plan support * @param {object} params.tx consumer transaction * @param {string} params.topic topic name * @params {string} params.clientId client id */ -utils.verifyConsumeTransaction = ({ t, tx, topic, clientId }) => { +utils.verifyConsumeTransaction = ({ plan, tx, topic, clientId }) => { const expectedName = `OtherTransaction/Message/Kafka/Topic/Consume/Named/${topic}` - t.assertMetrics( + assertMetrics( tx.metrics, [ [{ name: expectedName }], @@ -88,34 +90,35 @@ utils.verifyConsumeTransaction = ({ t, tx, topic, clientId }) => { [{ name: 'OtherTransactionTotalTime' }] ], false, - false + false, + { assert: plan } ) - t.equal(tx.getFullName(), expectedName) + plan.equal(tx.getFullName(), expectedName) const consume = metrics.findSegment(tx.trace.root, expectedName) - t.equal(consume, tx.baseSegment) + plan.equal(consume, tx.baseSegment) const attributes = tx.trace.attributes.get(DESTINATIONS.TRANS_SCOPE) - t.ok(attributes['kafka.consume.byteCount'], 'should have byteCount') - t.equal(attributes['kafka.consume.client_id'], clientId, 'should have client_id') + plan.ok(attributes['kafka.consume.byteCount'], 'should have byteCount') + plan.equal(attributes['kafka.consume.client_id'], clientId, 'should have client_id') } /** * Asserts the properties on both the produce and consume transactions * @param {object} params function params - * @param {object} params.t test instance + * @param {object} params.plan assertion library instance with plan support * @param {object} params.consumeTxs consumer transactions * @param {object} params.produceTx produce transaction */ -utils.verifyDistributedTrace = ({ t, consumeTxs, produceTx }) => { - t.ok(produceTx.isDistributedTrace, 'should mark producer as distributed') +utils.verifyDistributedTrace = ({ plan, consumeTxs, produceTx }) => { + plan.ok(produceTx.isDistributedTrace, 'should mark producer as distributed') const produceSegment = produceTx.trace.root.children[3] consumeTxs.forEach((consumeTx) => { - t.ok(consumeTx.isDistributedTrace, 'should mark consumer as distributed') - t.equal(consumeTx.incomingCatId, null, 'should not set old CAT properties') - t.equal(produceTx.id, consumeTx.parentId, 'should have proper parent id') - t.equal(produceTx.traceId, consumeTx.traceId, 'should have proper trace id') - t.equal(produceSegment.id, consumeTx.parentSpanId, 'should have proper parentSpanId') - t.equal(consumeTx.parentTransportType, 'Kafka', 'should have correct transport type') + plan.ok(consumeTx.isDistributedTrace, 'should mark consumer as distributed') + plan.equal(consumeTx.incomingCatId, null, 'should not set old CAT properties') + plan.equal(produceTx.id, consumeTx.parentId, 'should have proper parent id') + plan.equal(produceTx.traceId, consumeTx.traceId, 'should have proper trace id') + plan.equal(produceSegment.id, consumeTx.parentSpanId, 'should have proper parentSpanId') + plan.equal(consumeTx.parentTransportType, 'Kafka', 'should have correct transport type') }) } diff --git a/test/versioned/langchain/common.js b/test/versioned/langchain/common.js index f03ab6f8ed..ebd48728cd 100644 --- a/test/versioned/langchain/common.js +++ b/test/versioned/langchain/common.js @@ -5,7 +5,7 @@ 'use strict' -const tap = require('tap') +const { match } = require('../../lib/custom-assertions') function filterLangchainEvents(events) { return events.filter((event) => { @@ -21,7 +21,10 @@ function filterLangchainEventsByType(events, msgType) { }) } -function assertLangChainVectorSearch({ tx, vectorSearch, responseDocumentSize }) { +function assertLangChainVectorSearch( + { tx, vectorSearch, responseDocumentSize }, + { assert = require('node:assert') } = {} +) { const expectedSearch = { 'id': /[a-f0-9]{36}/, 'appName': 'New Relic for Node.js tests', @@ -36,11 +39,14 @@ function assertLangChainVectorSearch({ tx, vectorSearch, responseDocumentSize }) 'duration': tx.trace.root.children[0].getDurationInMillis() } - this.equal(vectorSearch[0].type, 'LlmVectorSearch') - this.match(vectorSearch[1], expectedSearch, 'should match vector search') + assert.equal(vectorSearch[0].type, 'LlmVectorSearch') + match(vectorSearch[1], expectedSearch, { assert }) } -function assertLangChainVectorSearchResult({ tx, vectorSearchResult, vectorSearchId }) { +function assertLangChainVectorSearchResult( + { tx, vectorSearchResult, vectorSearchId }, + { assert = require('node:assert') } = {} +) { const baseSearchResult = { 'id': /[a-f0-9]{36}/, 'search_id': vectorSearchId, @@ -63,12 +69,15 @@ function assertLangChainVectorSearchResult({ tx, vectorSearchResult, vectorSearc expectedChatMsg.page_content = '212 degrees Fahrenheit is equal to 100 degrees Celsius.' } - this.equal(search[0].type, 'LlmVectorSearchResult') - this.match(search[1], expectedChatMsg, 'should match vector search result') + assert.equal(search[0].type, 'LlmVectorSearchResult') + match(search[1], expectedChatMsg, { assert }) }) } -function assertLangChainChatCompletionSummary({ tx, chatSummary, withCallback }) { +function assertLangChainChatCompletionSummary( + { tx, chatSummary, withCallback }, + { assert = require('node:assert') } = {} +) { const expectedSummary = { 'id': /[a-f0-9]{36}/, 'appName': 'New Relic for Node.js tests', @@ -90,18 +99,21 @@ function assertLangChainChatCompletionSummary({ tx, chatSummary, withCallback }) expectedSummary.id = /[a-f0-9\-]{36}/ } - this.equal(chatSummary[0].type, 'LlmChatCompletionSummary') - this.match(chatSummary[1], expectedSummary, 'should match chat summary message') + assert.equal(chatSummary[0].type, 'LlmChatCompletionSummary') + match(chatSummary[1], expectedSummary, { assert }) } -function assertLangChainChatCompletionMessages({ - tx, - chatMsgs, - chatSummary, - withCallback, - input = '{"topic":"scientist"}', - output = '212 degrees Fahrenheit is equal to 100 degrees Celsius.' -}) { +function assertLangChainChatCompletionMessages( + { + tx, + chatMsgs, + chatSummary, + withCallback, + input = '{"topic":"scientist"}', + output = '212 degrees Fahrenheit is equal to 100 degrees Celsius.' + }, + { assert = require('node:assert') } = {} +) { const baseMsg = { id: /[a-f0-9]{36}/, appName: 'New Relic for Node.js tests', @@ -131,17 +143,16 @@ function assertLangChainChatCompletionMessages({ expectedChatMsg.is_response = true } - this.equal(msg[0].type, 'LlmChatCompletionMessage') - this.match(msg[1], expectedChatMsg, 'should match chat completion message') + assert.equal(msg[0].type, 'LlmChatCompletionMessage') + match(msg[1], expectedChatMsg, { assert }) }) } -tap.Test.prototype.addAssert('langchainMessages', 1, assertLangChainChatCompletionMessages) -tap.Test.prototype.addAssert('langchainSummary', 1, assertLangChainChatCompletionSummary) -tap.Test.prototype.addAssert('langchainVectorSearch', 1, assertLangChainVectorSearch) -tap.Test.prototype.addAssert('langchainVectorSearchResult', 1, assertLangChainVectorSearchResult) - module.exports = { + assertLangChainChatCompletionMessages, + assertLangChainChatCompletionSummary, + assertLangChainVectorSearch, + assertLangChainVectorSearchResult, filterLangchainEvents, filterLangchainEventsByType } diff --git a/test/versioned/langchain/package.json b/test/versioned/langchain/package.json index acd5f736ab..b6ef66da84 100644 --- a/test/versioned/langchain/package.json +++ b/test/versioned/langchain/package.json @@ -16,9 +16,9 @@ "@langchain/openai": ">=0.0.34" }, "files": [ - "tools.tap.js", - "runnables.tap.js", - "runnables-streaming.tap.js" + "tools.test.js", + "runnables.test.js", + "runnables-streaming.test.js" ] }, { @@ -32,7 +32,7 @@ "@elastic/elasticsearch": "8.13.1" }, "files": [ - "vectorstore.tap.js" + "vectorstore.test.js" ] } ] diff --git a/test/versioned/langchain/runnables-streaming.tap.js b/test/versioned/langchain/runnables-streaming.test.js similarity index 59% rename from test/versioned/langchain/runnables-streaming.tap.js rename to test/versioned/langchain/runnables-streaming.test.js index 118c32e6a4..1c81f61ddf 100644 --- a/test/versioned/langchain/runnables-streaming.tap.js +++ b/test/versioned/langchain/runnables-streaming.test.js @@ -5,60 +5,63 @@ 'use strict' -const tap = require('tap') -const helper = require('../../lib/agent_helper') +const test = require('node:test') +const assert = require('node:assert') + const { removeModules } = require('../../lib/cache-buster') -// load the assertSegments assertion -require('../../lib/metrics_helper') -const { filterLangchainEvents, filterLangchainEventsByType } = require('./common') +const { assertSegments, match } = require('../../lib/custom-assertions') +const { + assertLangChainChatCompletionMessages, + assertLangChainChatCompletionSummary, + filterLangchainEvents, + filterLangchainEventsByType +} = require('./common') const { version: pkgVersion } = require('@langchain/core/package.json') const createOpenAIMockServer = require('../openai/mock-server') const mockResponses = require('../openai/mock-responses') +const helper = require('../../lib/agent_helper') + const config = { ai_monitoring: { - enabled: true, - streaming: { - enabled: true - } + enabled: true } } - const { DESTINATIONS } = require('../../../lib/config/attribute-filter') -async function beforeEach({ enabled, t }) { +async function beforeEach({ enabled, ctx }) { + ctx.nr = {} const { host, port, server } = await createOpenAIMockServer() - t.context.server = server - t.context.agent = helper.instrumentMockedAgent(config) - t.context.agent.config.ai_monitoring.streaming.enabled = enabled + ctx.nr.server = server + ctx.nr.agent = helper.instrumentMockedAgent(config) + ctx.nr.agent.config.ai_monitoring.streaming.enabled = enabled const { ChatPromptTemplate } = require('@langchain/core/prompts') const { StringOutputParser } = require('@langchain/core/output_parsers') const { ChatOpenAI } = require('@langchain/openai') - t.context.prompt = ChatPromptTemplate.fromMessages([['assistant', '{topic} response']]) - t.context.model = new ChatOpenAI({ + ctx.nr.prompt = ChatPromptTemplate.fromMessages([['assistant', '{topic} response']]) + ctx.nr.model = new ChatOpenAI({ openAIApiKey: 'fake-key', maxRetries: 0, configuration: { baseURL: `http://${host}:${port}` } }) - t.context.outputParser = new StringOutputParser() + ctx.nr.outputParser = new StringOutputParser() } -async function afterEach(t) { - t.context?.server?.close() - helper.unloadAgent(t.context.agent) +async function afterEach(ctx) { + ctx.nr?.server?.close() + helper.unloadAgent(ctx.nr.agent) // bust the require-cache so it can re-instrument removeModules(['@langchain/core', 'openai']) } -tap.test('Langchain instrumentation - chain streaming', (t) => { - t.beforeEach(beforeEach.bind(null, { enabled: true, t })) - - t.afterEach(afterEach.bind(null, t)) +test('streaming enabled', async (t) => { + t.beforeEach((ctx) => beforeEach({ enabled: true, ctx })) + t.afterEach((ctx) => afterEach(ctx)) - t.test('should create langchain events for every stream call', (t) => { - const { agent, prompt, outputParser, model } = t.context + await t.test('should create langchain events for every stream call', (t, end) => { + const { agent, prompt, outputParser, model } = t.nr helper.runInTransaction(agent, async (tx) => { const input = { topic: 'Streamed' } @@ -71,49 +74,52 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { } const { streamData: expectedContent } = mockResponses.get('Streamed response') - t.equal(content, expectedContent) + assert.equal(content, expectedContent) const events = agent.customEventAggregator.events.toArray() - t.equal(events.length, 6, 'should create 6 events') + assert.equal(events.length, 6, 'should create 6 events') const langchainEvents = events.filter((event) => { const [, chainEvent] = event return chainEvent.vendor === 'langchain' }) - t.equal(langchainEvents.length, 3, 'should create 3 langchain events') + assert.equal(langchainEvents.length, 3, 'should create 3 langchain events') tx.end() - t.end() + end() }) }) - t.test('should increment tracking metric for each langchain chat prompt event', (t) => { - const { agent, prompt, outputParser, model } = t.context + await t.test( + 'should increment tracking metric for each langchain chat prompt event', + (t, end) => { + const { agent, prompt, outputParser, model } = t.nr - helper.runInTransaction(agent, async (tx) => { - const input = { topic: 'Streamed' } + helper.runInTransaction(agent, async (tx) => { + const input = { topic: 'Streamed' } - const chain = prompt.pipe(model).pipe(outputParser) - const stream = await chain.stream(input) - for await (const chunk of stream) { - chunk - // no-op - } + const chain = prompt.pipe(model).pipe(outputParser) + const stream = await chain.stream(input) + for await (const chunk of stream) { + chunk + // no-op + } - const metrics = agent.metrics.getOrCreateMetric( - `Supportability/Nodejs/ML/Langchain/${pkgVersion}` - ) - t.equal(metrics.callCount > 0, true) + const metrics = agent.metrics.getOrCreateMetric( + `Supportability/Nodejs/ML/Langchain/${pkgVersion}` + ) + assert.equal(metrics.callCount > 0, true) - tx.end() - t.end() - }) - }) + tx.end() + end() + }) + } + ) - t.test( + await t.test( 'should create langchain events for every stream call on chat prompt + model + parser', - (t) => { - const { agent, prompt, outputParser, model } = t.context + (t, end) => { + const { agent, prompt, outputParser, model } = t.nr helper.runInTransaction(agent, async (tx) => { const input = { topic: 'Streamed' } @@ -138,12 +144,12 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { 'LlmChatCompletionSummary' ) - t.langchainSummary({ + assertLangChainChatCompletionSummary({ tx, chatSummary: langChainSummaryEvents[0] }) - t.langchainMessages({ + assertLangChainChatCompletionMessages({ tx, chatMsgs: langChainMessageEvents, chatSummary: langChainSummaryEvents[0][1], @@ -152,60 +158,63 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { }) tx.end() - t.end() + end() }) } ) - t.test('should create langchain events for every stream call on chat prompt + model', (t) => { - const { agent, prompt, model } = t.context + await t.test( + 'should create langchain events for every stream call on chat prompt + model', + (t, end) => { + const { agent, prompt, model } = t.nr - helper.runInTransaction(agent, async (tx) => { - const input = { topic: 'Streamed' } - const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } + helper.runInTransaction(agent, async (tx) => { + const input = { topic: 'Streamed' } + const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } - const chain = prompt.pipe(model) - const stream = await chain.stream(input, options) - let content = '' - for await (const chunk of stream) { - content += chunk - } + const chain = prompt.pipe(model) + const stream = await chain.stream(input, options) + let content = '' + for await (const chunk of stream) { + content += chunk + } - const events = agent.customEventAggregator.events.toArray() + const events = agent.customEventAggregator.events.toArray() - const langchainEvents = filterLangchainEvents(events) - const langChainMessageEvents = filterLangchainEventsByType( - langchainEvents, - 'LlmChatCompletionMessage' - ) - const langChainSummaryEvents = filterLangchainEventsByType( - langchainEvents, - 'LlmChatCompletionSummary' - ) - - t.langchainSummary({ - tx, - chatSummary: langChainSummaryEvents[0] - }) + const langchainEvents = filterLangchainEvents(events) + const langChainMessageEvents = filterLangchainEventsByType( + langchainEvents, + 'LlmChatCompletionMessage' + ) + const langChainSummaryEvents = filterLangchainEventsByType( + langchainEvents, + 'LlmChatCompletionSummary' + ) - t.langchainMessages({ - tx, - chatMsgs: langChainMessageEvents, - chatSummary: langChainSummaryEvents[0][1], - input: '{"topic":"Streamed"}', - output: content - }) + assertLangChainChatCompletionSummary({ + tx, + chatSummary: langChainSummaryEvents[0] + }) - tx.end() - t.end() - }) - }) + assertLangChainChatCompletionMessages({ + tx, + chatMsgs: langChainMessageEvents, + chatSummary: langChainSummaryEvents[0][1], + input: '{"topic":"Streamed"}', + output: content + }) - t.test( + tx.end() + end() + }) + } + ) + + await t.test( 'should create langchain events for every stream call with parser that returns an array as output', - (t) => { + (t, end) => { const { CommaSeparatedListOutputParser } = require('@langchain/core/output_parsers') - const { agent, prompt, model } = t.context + const { agent, prompt, model } = t.nr helper.runInTransaction(agent, async (tx) => { const parser = new CommaSeparatedListOutputParser() @@ -232,12 +241,12 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { 'LlmChatCompletionSummary' ) - t.langchainSummary({ + assertLangChainChatCompletionSummary({ tx, chatSummary: langChainSummaryEvents[0] }) - t.langchainMessages({ + assertLangChainChatCompletionMessages({ tx, chatMsgs: langChainMessageEvents, chatSummary: langChainSummaryEvents[0][1], @@ -246,12 +255,12 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { }) tx.end() - t.end() + end() }) } ) - t.test('should add runId when a callback handler exists', (t) => { + await t.test('should add runId when a callback handler exists', (t, end) => { const { BaseCallbackHandler } = require('@langchain/core/callbacks/base') let runId const cbHandler = BaseCallbackHandler.fromMethods({ @@ -260,7 +269,7 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { } }) - const { agent, prompt, outputParser, model } = t.context + const { agent, prompt, outputParser, model } = t.nr helper.runInTransaction(agent, async (tx) => { const input = { topic: 'Streamed' } @@ -280,22 +289,22 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { const events = agent.customEventAggregator.events.toArray() const langchainEvents = filterLangchainEvents(events) - t.equal(langchainEvents[0][1].request_id, runId) + assert.equal(langchainEvents[0][1].request_id, runId) tx.end() - t.end() + end() }) }) - t.test( + await t.test( 'should create langchain events for every stream call on chat prompt + model + parser with callback', - (t) => { + (t, end) => { const { BaseCallbackHandler } = require('@langchain/core/callbacks/base') const cbHandler = BaseCallbackHandler.fromMethods({ handleChainStart() {} }) - const { agent, prompt, outputParser, model } = t.context + const { agent, prompt, outputParser, model } = t.nr helper.runInTransaction(agent, async (tx) => { const input = { topic: 'Streamed' } @@ -324,13 +333,13 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { langchainEvents, 'LlmChatCompletionSummary' ) - t.langchainSummary({ + assertLangChainChatCompletionSummary({ tx, chatSummary: langChainSummaryEvents[0], withCallback: cbHandler }) - t.langchainMessages({ + assertLangChainChatCompletionMessages({ tx, chatMsgs: langChainMessageEvents, chatSummary: langChainSummaryEvents[0][1], @@ -340,13 +349,13 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { }) tx.end() - t.end() + end() }) } ) - t.test('should not create langchain events when not in a transaction', async (t) => { - const { agent, prompt, outputParser, model } = t.context + await t.test('should not create langchain events when not in a transaction', async (t) => { + const { agent, prompt, outputParser, model } = t.nr const input = { topic: 'Streamed' } @@ -358,12 +367,11 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { } const events = agent.customEventAggregator.events.toArray() - t.equal(events.length, 0, 'should not create langchain events') - t.end() + assert.equal(events.length, 0, 'should not create langchain events') }) - t.test('should add llm attribute to transaction', (t) => { - const { agent, prompt, model } = t.context + await t.test('should add llm attribute to transaction', (t, end) => { + const { agent, prompt, model } = t.nr const input = { topic: 'Streamed' } @@ -376,15 +384,15 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { } const attributes = tx.trace.attributes.get(DESTINATIONS.TRANS_EVENT) - t.equal(attributes.llm, true) + assert.equal(attributes.llm, true) tx.end() - t.end() + end() }) }) - t.test('should create span on successful runnables create', (t) => { - const { agent, prompt, model } = t.context + await t.test('should create span on successful runnables create', (t, end) => { + const { agent, prompt, model } = t.nr const input = { topic: 'Streamed' } @@ -396,18 +404,18 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { // no-op } - t.assertSegments(tx.trace.root, ['Llm/chain/Langchain/stream'], { exact: false }) + assertSegments(tx.trace.root, ['Llm/chain/Langchain/stream'], { exact: false }) tx.end() - t.end() + end() }) }) // testing JSON.stringify on request (input) during creation of LangChainCompletionMessage event - t.test( + await t.test( 'should use empty string for content property on completion message event when invalid input is used - circular reference', - (t) => { - const { agent, prompt, outputParser, model } = t.context + (t, end) => { + const { agent, prompt, outputParser, model } = t.nr helper.runInTransaction(agent, async (tx) => { const input = { topic: 'Streamed' } @@ -432,20 +440,24 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { (event) => event[1].content === '' ) - t.equal(msgEventEmptyContent.length, 1, 'should have 1 event with empty content property') + assert.equal( + msgEventEmptyContent.length, + 1, + 'should have 1 event with empty content property' + ) tx.end() - t.end() + end() }) } ) - t.test('should create error events from input', (t) => { + await t.test('should create error events from input', (t, end) => { const { ChatPromptTemplate } = require('@langchain/core/prompts') const prompt = ChatPromptTemplate.fromMessages([ ['assistant', 'tell me short joke about {topic}'] ]) - const { agent, outputParser, model } = t.context + const { agent, outputParser, model } = t.nr helper.runInTransaction(agent, async (tx) => { const chain = prompt.pipe(model).pipe(outputParser) @@ -453,32 +465,32 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { try { await chain.stream('') } catch (error) { - t.ok(error) + assert.ok(error) } // No openai events as it errors before talking to LLM const events = agent.customEventAggregator.events.toArray() - t.equal(events.length, 2, 'should create 2 events') + assert.equal(events.length, 2, 'should create 2 events') const summary = events.find((e) => e[0].type === 'LlmChatCompletionSummary')?.[1] - t.equal(summary.error, true) + assert.equal(summary.error, true) // But, we should also get two error events: 1xLLM and 1xLangChain const exceptions = tx.exceptions for (const e of exceptions) { const str = Object.prototype.toString.call(e.customAttributes) - t.equal(str, '[object LlmErrorMessage]') + assert.equal(str, '[object LlmErrorMessage]') } tx.end() - t.end() + end() }) }) - t.test('should create error events when stream fails', (t) => { + await t.test('should create error events when stream fails', (t, end) => { const { ChatPromptTemplate } = require('@langchain/core/prompts') const prompt = ChatPromptTemplate.fromMessages([['assistant', '{topic} stream']]) - const { agent, model, outputParser } = t.context + const { agent, model, outputParser } = t.nr helper.runInTransaction(agent, async (tx) => { const chain = prompt.pipe(model).pipe(outputParser) @@ -490,28 +502,28 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { // no-op } } catch (error) { - t.ok(error) + assert.ok(error) } // We should still get the same 3xLangChain and 3xLLM events as in the // success case: const events = agent.customEventAggregator.events.toArray() - t.equal(events.length, 6, 'should create 6 events') + assert.equal(events.length, 6, 'should create 6 events') const langchainEvents = events.filter((event) => { const [, chainEvent] = event return chainEvent.vendor === 'langchain' }) - t.equal(langchainEvents.length, 3, 'should create 3 langchain events') + assert.equal(langchainEvents.length, 3, 'should create 3 langchain events') const summary = langchainEvents.find((e) => e[0].type === 'LlmChatCompletionSummary')?.[1] - t.equal(summary.error, true) + assert.equal(summary.error, true) // But, we should also get two error events: 1xLLM and 1xLangChain const exceptions = tx.exceptions for (const e of exceptions) { const str = Object.prototype.toString.call(e.customAttributes) - t.equal(str, '[object LlmErrorMessage]') - t.match(e, { + assert.equal(str, '[object LlmErrorMessage]') + match(e, { customAttributes: { 'error.message': 'Premature close', 'completion_id': /\w{32}/ @@ -519,51 +531,52 @@ tap.test('Langchain instrumentation - chain streaming', (t) => { }) } tx.end() - t.end() + end() }) }) - t.end() }) -tap.test('Langchain instrumentation - streaming disabled', (t) => { - t.beforeEach(beforeEach.bind(null, { enabled: false, t })) +test('streaming disabled', async (t) => { + t.beforeEach((ctx) => beforeEach({ enabled: false, ctx })) + t.afterEach((ctx) => afterEach(ctx)) - t.afterEach(afterEach.bind(null, t)) + await t.test( + 'should not create llm events when `ai_monitoring.streaming.enabled` is false', + (t, end) => { + const { agent, prompt, outputParser, model } = t.nr - t.test('should not create llm events when `ai_monitoring.streaming.enabled` is false', (t) => { - const { agent, prompt, outputParser, model } = t.context + helper.runInTransaction(agent, async (tx) => { + const input = { topic: 'Streamed' } - helper.runInTransaction(agent, async (tx) => { - const input = { topic: 'Streamed' } + const chain = prompt.pipe(model).pipe(outputParser) + const stream = await chain.stream(input) + let content = '' + for await (const chunk of stream) { + content += chunk + } - const chain = prompt.pipe(model).pipe(outputParser) - const stream = await chain.stream(input) - let content = '' - for await (const chunk of stream) { - content += chunk - } + const { streamData: expectedContent } = mockResponses.get('Streamed response') + assert.equal(content, expectedContent) + const events = agent.customEventAggregator.events.toArray() + assert.equal(events.length, 0, 'should not create llm events when streaming is disabled') + const metrics = agent.metrics.getOrCreateMetric( + `Supportability/Nodejs/ML/Langchain/${pkgVersion}` + ) + assert.equal(metrics.callCount > 0, true) + const attributes = tx.trace.attributes.get(DESTINATIONS.TRANS_EVENT) + assert.equal(attributes.llm, true) + const streamingDisabled = agent.metrics.getOrCreateMetric( + 'Supportability/Nodejs/ML/Streaming/Disabled' + ) + assert.equal( + streamingDisabled.callCount, + 2, + 'should increment streaming disabled in both langchain and openai' + ) - const { streamData: expectedContent } = mockResponses.get('Streamed response') - t.equal(content, expectedContent) - const events = agent.customEventAggregator.events.toArray() - t.equal(events.length, 0, 'should not create llm events when streaming is disabled') - const metrics = agent.metrics.getOrCreateMetric( - `Supportability/Nodejs/ML/Langchain/${pkgVersion}` - ) - t.equal(metrics.callCount > 0, true) - const attributes = tx.trace.attributes.get(DESTINATIONS.TRANS_EVENT) - t.equal(attributes.llm, true) - const streamingDisabled = agent.metrics.getOrCreateMetric( - 'Supportability/Nodejs/ML/Streaming/Disabled' - ) - t.equal( - streamingDisabled.callCount, - 2, - 'should increment streaming disabled in both langchain and openai' - ) - tx.end() - t.end() - }) - }) - t.end() + tx.end() + end() + }) + } + ) }) diff --git a/test/versioned/langchain/runnables.tap.js b/test/versioned/langchain/runnables.tap.js deleted file mode 100644 index a2d3aacc7f..0000000000 --- a/test/versioned/langchain/runnables.tap.js +++ /dev/null @@ -1,445 +0,0 @@ -/* - * Copyright 2024 New Relic Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -'use strict' - -const tap = require('tap') -const helper = require('../../lib/agent_helper') -const { removeModules } = require('../../lib/cache-buster') -// load the assertSegments assertion -require('../../lib/metrics_helper') -const { filterLangchainEvents, filterLangchainEventsByType } = require('./common') -const { version: pkgVersion } = require('@langchain/core/package.json') -const createOpenAIMockServer = require('../openai/mock-server') -const config = { - ai_monitoring: { - enabled: true - } -} - -const { DESTINATIONS } = require('../../../lib/config/attribute-filter') - -tap.test('Langchain instrumentation - runnable sequence', (t) => { - t.autoend() - - t.beforeEach(async (t) => { - const { host, port, server } = await createOpenAIMockServer() - t.context.server = server - t.context.agent = helper.instrumentMockedAgent(config) - const { ChatPromptTemplate } = require('@langchain/core/prompts') - const { StringOutputParser } = require('@langchain/core/output_parsers') - const { ChatOpenAI } = require('@langchain/openai') - - t.context.prompt = ChatPromptTemplate.fromMessages([['assistant', 'You are a {topic}.']]) - t.context.model = new ChatOpenAI({ - openAIApiKey: 'fake-key', - maxRetries: 0, - configuration: { - baseURL: `http://${host}:${port}` - } - }) - t.context.outputParser = new StringOutputParser() - }) - - t.afterEach(async (t) => { - t.context?.server?.close() - helper.unloadAgent(t.context.agent) - // bust the require-cache so it can re-instrument - removeModules(['@langchain/core', 'openai']) - }) - - t.test('should create langchain events for every invoke call', (t) => { - const { agent, prompt, outputParser, model } = t.context - helper.runInTransaction(agent, async (tx) => { - const input = { topic: 'scientist' } - const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } - - const chain = prompt.pipe(model).pipe(outputParser) - await chain.invoke(input, options) - - const events = agent.customEventAggregator.events.toArray() - t.equal(events.length, 6, 'should create 6 events') - - const langchainEvents = events.filter((event) => { - const [, chainEvent] = event - return chainEvent.vendor === 'langchain' - }) - - t.equal(langchainEvents.length, 3, 'should create 3 langchain events') - - tx.end() - t.end() - }) - }) - - t.test('should increment tracking metric for each langchain chat prompt event', (t) => { - const { agent, prompt, outputParser, model } = t.context - - helper.runInTransaction(agent, async (tx) => { - const input = { topic: 'scientist' } - const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } - - const chain = prompt.pipe(model).pipe(outputParser) - await chain.invoke(input, options) - - const metrics = agent.metrics.getOrCreateMetric( - `Supportability/Nodejs/ML/Langchain/${pkgVersion}` - ) - t.equal(metrics.callCount > 0, true) - - tx.end() - t.end() - }) - }) - - t.test('should support custom attributes on the LLM events', (t) => { - const { agent, prompt, outputParser, model } = t.context - const api = helper.getAgentApi() - helper.runInTransaction(agent, async (tx) => { - api.withLlmCustomAttributes({ 'llm.contextAttribute': 'someValue' }, async () => { - const input = { topic: 'scientist' } - const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } - - const chain = prompt.pipe(model).pipe(outputParser) - await chain.invoke(input, options) - const events = agent.customEventAggregator.events.toArray() - - const [[, message]] = events - t.equal(message['llm.contextAttribute'], 'someValue') - - tx.end() - t.end() - }) - }) - }) - - t.test( - 'should create langchain events for every invoke call on chat prompt + model + parser', - (t) => { - const { agent, prompt, outputParser, model } = t.context - helper.runInTransaction(agent, async (tx) => { - const input = { topic: 'scientist' } - const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } - - const chain = prompt.pipe(model).pipe(outputParser) - await chain.invoke(input, options) - - const events = agent.customEventAggregator.events.toArray() - - const langchainEvents = filterLangchainEvents(events) - const langChainMessageEvents = filterLangchainEventsByType( - langchainEvents, - 'LlmChatCompletionMessage' - ) - const langChainSummaryEvents = filterLangchainEventsByType( - langchainEvents, - 'LlmChatCompletionSummary' - ) - - t.langchainSummary({ - tx, - chatSummary: langChainSummaryEvents[0] - }) - - t.langchainMessages({ - tx, - chatMsgs: langChainMessageEvents, - chatSummary: langChainSummaryEvents[0][1] - }) - - tx.end() - t.end() - }) - } - ) - - t.test('should create langchain events for every invoke call on chat prompt + model', (t) => { - const { agent, prompt, model } = t.context - - helper.runInTransaction(agent, async (tx) => { - const input = { topic: 'scientist' } - const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } - - const chain = prompt.pipe(model) - await chain.invoke(input, options) - - const events = agent.customEventAggregator.events.toArray() - - const langchainEvents = filterLangchainEvents(events) - const langChainMessageEvents = filterLangchainEventsByType( - langchainEvents, - 'LlmChatCompletionMessage' - ) - const langChainSummaryEvents = filterLangchainEventsByType( - langchainEvents, - 'LlmChatCompletionSummary' - ) - - t.langchainSummary({ - tx, - chatSummary: langChainSummaryEvents[0] - }) - - t.langchainMessages({ - tx, - chatMsgs: langChainMessageEvents, - chatSummary: langChainSummaryEvents[0][1] - }) - - tx.end() - t.end() - }) - }) - - t.test( - 'should create langchain events for every invoke call with parser that returns an array as output', - (t) => { - const { CommaSeparatedListOutputParser } = require('@langchain/core/output_parsers') - const { agent, prompt, model } = t.context - - helper.runInTransaction(agent, async (tx) => { - const parser = new CommaSeparatedListOutputParser() - - const input = { topic: 'scientist' } - const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } - - const chain = prompt.pipe(model).pipe(parser) - await chain.invoke(input, options) - - const events = agent.customEventAggregator.events.toArray() - - const langchainEvents = filterLangchainEvents(events) - const langChainMessageEvents = filterLangchainEventsByType( - langchainEvents, - 'LlmChatCompletionMessage' - ) - const langChainSummaryEvents = filterLangchainEventsByType( - langchainEvents, - 'LlmChatCompletionSummary' - ) - - t.langchainSummary({ - tx, - chatSummary: langChainSummaryEvents[0] - }) - - t.langchainMessages({ - tx, - chatMsgs: langChainMessageEvents, - chatSummary: langChainSummaryEvents[0][1] - }) - - tx.end() - t.end() - }) - } - ) - - t.test('should add runId when a callback handler exists', (t) => { - const { BaseCallbackHandler } = require('@langchain/core/callbacks/base') - let runId - const cbHandler = BaseCallbackHandler.fromMethods({ - handleChainStart(...args) { - runId = args?.[2] - } - }) - - const { agent, prompt, outputParser, model } = t.context - - helper.runInTransaction(agent, async (tx) => { - const input = { topic: 'scientist' } - const options = { - metadata: { key: 'value', hello: 'world' }, - callbacks: [cbHandler], - tags: ['tag1', 'tag2'] - } - - const chain = prompt.pipe(model).pipe(outputParser) - await chain.invoke(input, options) - - const events = agent.customEventAggregator.events.toArray() - - const langchainEvents = filterLangchainEvents(events) - t.equal(langchainEvents[0][1].request_id, runId) - - tx.end() - t.end() - }) - }) - - t.test( - 'should create langchain events for every invoke call on chat prompt + model + parser with callback', - (t) => { - const { BaseCallbackHandler } = require('@langchain/core/callbacks/base') - const cbHandler = BaseCallbackHandler.fromMethods({ - handleChainStart() {} - }) - - const { agent, prompt, outputParser, model } = t.context - - helper.runInTransaction(agent, async (tx) => { - const input = { topic: 'scientist' } - const options = { - metadata: { key: 'value', hello: 'world' }, - callbacks: [cbHandler], - tags: ['tag1', 'tag2'] - } - - const chain = prompt.pipe(model).pipe(outputParser) - await chain.invoke(input, options) - - const events = agent.customEventAggregator.events.toArray() - - const langchainEvents = filterLangchainEvents(events) - const langChainMessageEvents = filterLangchainEventsByType( - langchainEvents, - 'LlmChatCompletionMessage' - ) - const langChainSummaryEvents = filterLangchainEventsByType( - langchainEvents, - 'LlmChatCompletionSummary' - ) - t.langchainSummary({ - tx, - chatSummary: langChainSummaryEvents[0], - withCallback: cbHandler - }) - - t.langchainMessages({ - tx, - chatMsgs: langChainMessageEvents, - chatSummary: langChainSummaryEvents[0][1], - withCallback: cbHandler - }) - - tx.end() - t.end() - }) - } - ) - - t.test('should not create langchain events when not in a transaction', async (t) => { - const { agent, prompt, outputParser, model } = t.context - - const input = { topic: 'scientist' } - const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } - - const chain = prompt.pipe(model).pipe(outputParser) - await chain.invoke(input, options) - - const events = agent.customEventAggregator.events.toArray() - t.equal(events.length, 0, 'should not create langchain events') - t.end() - }) - - t.test('should add llm attribute to transaction', (t) => { - const { agent, prompt, model } = t.context - - const input = { topic: 'scientist' } - const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } - - helper.runInTransaction(agent, async (tx) => { - const chain = prompt.pipe(model) - await chain.invoke(input, options) - - const attributes = tx.trace.attributes.get(DESTINATIONS.TRANS_EVENT) - t.equal(attributes.llm, true) - - tx.end() - t.end() - }) - }) - - t.test('should create span on successful runnables create', (t) => { - const { agent, prompt, model } = t.context - - const input = { topic: 'scientist' } - const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } - - helper.runInTransaction(agent, async (tx) => { - const chain = prompt.pipe(model) - const result = await chain.invoke(input, options) - - t.ok(result) - t.assertSegments(tx.trace.root, ['Llm/chain/Langchain/invoke'], { exact: false }) - - tx.end() - t.end() - }) - }) - - // testing JSON.stringify on request (input) during creation of LangChainCompletionMessage event - t.test( - 'should use empty string for content property on completion message event when invalid input is used - circular reference', - (t) => { - const { agent, prompt, outputParser, model } = t.context - - helper.runInTransaction(agent, async (tx) => { - const input = { topic: 'scientist' } - input.myself = input - const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } - - const chain = prompt.pipe(model).pipe(outputParser) - await chain.invoke(input, options) - - const events = agent.customEventAggregator.events.toArray() - - const langchainEvents = filterLangchainEvents(events) - const langChainMessageEvents = filterLangchainEventsByType( - langchainEvents, - 'LlmChatCompletionMessage' - ) - - const msgEventEmptyContent = langChainMessageEvents.filter( - (event) => event[1].content === '' - ) - - t.equal(msgEventEmptyContent.length, 1, 'should have 1 event with empty content property') - - tx.end() - t.end() - }) - } - ) - - t.test('should create error events', (t) => { - const { ChatPromptTemplate } = require('@langchain/core/prompts') - const prompt = ChatPromptTemplate.fromMessages([['assistant', 'Invalid API key.']]) - const { agent, outputParser, model } = t.context - - helper.runInTransaction(agent, async (tx) => { - const chain = prompt.pipe(model).pipe(outputParser) - - try { - await chain.invoke('') - } catch (error) { - t.ok(error) - } - - // We should still get the same 3xLangChain and 3xLLM events as in the - // success case: - const events = agent.customEventAggregator.events.toArray() - t.equal(events.length, 6, 'should create 6 events') - - const langchainEvents = events.filter((event) => { - const [, chainEvent] = event - return chainEvent.vendor === 'langchain' - }) - t.equal(langchainEvents.length, 3, 'should create 3 langchain events') - const summary = langchainEvents.find((e) => e[0].type === 'LlmChatCompletionSummary')?.[1] - t.equal(summary.error, true) - - // But, we should also get two error events: 1xLLM and 1xLangChain - const exceptions = tx.exceptions - for (const e of exceptions) { - const str = Object.prototype.toString.call(e.customAttributes) - t.equal(str, '[object LlmErrorMessage]') - } - - tx.end() - t.end() - }) - }) -}) diff --git a/test/versioned/langchain/runnables.test.js b/test/versioned/langchain/runnables.test.js new file mode 100644 index 0000000000..8a2037624b --- /dev/null +++ b/test/versioned/langchain/runnables.test.js @@ -0,0 +1,434 @@ +/* + * Copyright 2024 New Relic Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +'use strict' + +const test = require('node:test') +const assert = require('node:assert') + +const { removeModules } = require('../../lib/cache-buster') +const { assertSegments } = require('../../lib/custom-assertions') +const { + assertLangChainChatCompletionMessages, + assertLangChainChatCompletionSummary, + filterLangchainEvents, + filterLangchainEventsByType +} = require('./common') +const { version: pkgVersion } = require('@langchain/core/package.json') +const createOpenAIMockServer = require('../openai/mock-server') +const helper = require('../../lib/agent_helper') + +const config = { + ai_monitoring: { + enabled: true + } +} +const { DESTINATIONS } = require('../../../lib/config/attribute-filter') + +test.beforeEach(async (ctx) => { + ctx.nr = {} + const { host, port, server } = await createOpenAIMockServer() + ctx.nr.server = server + ctx.nr.agent = helper.instrumentMockedAgent(config) + const { ChatPromptTemplate } = require('@langchain/core/prompts') + const { StringOutputParser } = require('@langchain/core/output_parsers') + const { ChatOpenAI } = require('@langchain/openai') + + ctx.nr.prompt = ChatPromptTemplate.fromMessages([['assistant', 'You are a {topic}.']]) + ctx.nr.model = new ChatOpenAI({ + openAIApiKey: 'fake-key', + maxRetries: 0, + configuration: { + baseURL: `http://${host}:${port}` + } + }) + ctx.nr.outputParser = new StringOutputParser() +}) + +test.afterEach(async (ctx) => { + ctx.nr?.server?.close() + helper.unloadAgent(ctx.nr.agent) + // bust the require-cache so it can re-instrument + removeModules(['@langchain/core', 'openai']) +}) + +test('should create langchain events for every invoke call', (t, end) => { + const { agent, prompt, outputParser, model } = t.nr + helper.runInTransaction(agent, async (tx) => { + const input = { topic: 'scientist' } + const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } + + const chain = prompt.pipe(model).pipe(outputParser) + await chain.invoke(input, options) + + const events = agent.customEventAggregator.events.toArray() + assert.equal(events.length, 6, 'should create 6 events') + + const langchainEvents = events.filter((event) => { + const [, chainEvent] = event + return chainEvent.vendor === 'langchain' + }) + + assert.equal(langchainEvents.length, 3, 'should create 3 langchain events') + + tx.end() + end() + }) +}) + +test('should increment tracking metric for each langchain chat prompt event', (t, end) => { + const { agent, prompt, outputParser, model } = t.nr + + helper.runInTransaction(agent, async (tx) => { + const input = { topic: 'scientist' } + const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } + + const chain = prompt.pipe(model).pipe(outputParser) + await chain.invoke(input, options) + + const metrics = agent.metrics.getOrCreateMetric( + `Supportability/Nodejs/ML/Langchain/${pkgVersion}` + ) + assert.equal(metrics.callCount > 0, true) + + tx.end() + end() + }) +}) + +test('should support custom attributes on the LLM events', (t, end) => { + const { agent, prompt, outputParser, model } = t.nr + const api = helper.getAgentApi() + helper.runInTransaction(agent, async (tx) => { + api.withLlmCustomAttributes({ 'llm.contextAttribute': 'someValue' }, async () => { + const input = { topic: 'scientist' } + const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } + + const chain = prompt.pipe(model).pipe(outputParser) + await chain.invoke(input, options) + const events = agent.customEventAggregator.events.toArray() + + const [[, message]] = events + assert.equal(message['llm.contextAttribute'], 'someValue') + + tx.end() + end() + }) + }) +}) + +test('should create langchain events for every invoke call on chat prompt + model + parser', (t, end) => { + const { agent, prompt, outputParser, model } = t.nr + helper.runInTransaction(agent, async (tx) => { + const input = { topic: 'scientist' } + const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } + + const chain = prompt.pipe(model).pipe(outputParser) + await chain.invoke(input, options) + + const events = agent.customEventAggregator.events.toArray() + + const langchainEvents = filterLangchainEvents(events) + const langChainMessageEvents = filterLangchainEventsByType( + langchainEvents, + 'LlmChatCompletionMessage' + ) + const langChainSummaryEvents = filterLangchainEventsByType( + langchainEvents, + 'LlmChatCompletionSummary' + ) + + assertLangChainChatCompletionSummary({ + tx, + chatSummary: langChainSummaryEvents[0] + }) + + assertLangChainChatCompletionMessages({ + tx, + chatMsgs: langChainMessageEvents, + chatSummary: langChainSummaryEvents[0][1] + }) + + tx.end() + end() + }) +}) + +test('should create langchain events for every invoke call on chat prompt + model', (t, end) => { + const { agent, prompt, model } = t.nr + + helper.runInTransaction(agent, async (tx) => { + const input = { topic: 'scientist' } + const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } + + const chain = prompt.pipe(model) + await chain.invoke(input, options) + + const events = agent.customEventAggregator.events.toArray() + + const langchainEvents = filterLangchainEvents(events) + const langChainMessageEvents = filterLangchainEventsByType( + langchainEvents, + 'LlmChatCompletionMessage' + ) + const langChainSummaryEvents = filterLangchainEventsByType( + langchainEvents, + 'LlmChatCompletionSummary' + ) + + assertLangChainChatCompletionSummary({ + tx, + chatSummary: langChainSummaryEvents[0] + }) + + assertLangChainChatCompletionMessages({ + tx, + chatMsgs: langChainMessageEvents, + chatSummary: langChainSummaryEvents[0][1] + }) + + tx.end() + end() + }) +}) + +test('should create langchain events for every invoke call with parser that returns an array as output', (t, end) => { + const { CommaSeparatedListOutputParser } = require('@langchain/core/output_parsers') + const { agent, prompt, model } = t.nr + + helper.runInTransaction(agent, async (tx) => { + const parser = new CommaSeparatedListOutputParser() + + const input = { topic: 'scientist' } + const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } + + const chain = prompt.pipe(model).pipe(parser) + await chain.invoke(input, options) + + const events = agent.customEventAggregator.events.toArray() + + const langchainEvents = filterLangchainEvents(events) + const langChainMessageEvents = filterLangchainEventsByType( + langchainEvents, + 'LlmChatCompletionMessage' + ) + const langChainSummaryEvents = filterLangchainEventsByType( + langchainEvents, + 'LlmChatCompletionSummary' + ) + + assertLangChainChatCompletionSummary({ + tx, + chatSummary: langChainSummaryEvents[0] + }) + + assertLangChainChatCompletionMessages({ + tx, + chatMsgs: langChainMessageEvents, + chatSummary: langChainSummaryEvents[0][1], + output: `["212 degrees Fahrenheit is equal to 100 degrees Celsius."]` + }) + + tx.end() + end() + }) +}) + +test('should add runId when a callback handler exists', (t, end) => { + const { BaseCallbackHandler } = require('@langchain/core/callbacks/base') + let runId + const cbHandler = BaseCallbackHandler.fromMethods({ + handleChainStart(...args) { + runId = args?.[2] + } + }) + + const { agent, prompt, outputParser, model } = t.nr + + helper.runInTransaction(agent, async (tx) => { + const input = { topic: 'scientist' } + const options = { + metadata: { key: 'value', hello: 'world' }, + callbacks: [cbHandler], + tags: ['tag1', 'tag2'] + } + + const chain = prompt.pipe(model).pipe(outputParser) + await chain.invoke(input, options) + + const events = agent.customEventAggregator.events.toArray() + + const langchainEvents = filterLangchainEvents(events) + assert.equal(langchainEvents[0][1].request_id, runId) + + tx.end() + end() + }) +}) + +test('should create langchain events for every invoke call on chat prompt + model + parser with callback', (t, end) => { + const { BaseCallbackHandler } = require('@langchain/core/callbacks/base') + const cbHandler = BaseCallbackHandler.fromMethods({ + handleChainStart() {} + }) + + const { agent, prompt, outputParser, model } = t.nr + + helper.runInTransaction(agent, async (tx) => { + const input = { topic: 'scientist' } + const options = { + metadata: { key: 'value', hello: 'world' }, + callbacks: [cbHandler], + tags: ['tag1', 'tag2'] + } + + const chain = prompt.pipe(model).pipe(outputParser) + await chain.invoke(input, options) + + const events = agent.customEventAggregator.events.toArray() + + const langchainEvents = filterLangchainEvents(events) + const langChainMessageEvents = filterLangchainEventsByType( + langchainEvents, + 'LlmChatCompletionMessage' + ) + const langChainSummaryEvents = filterLangchainEventsByType( + langchainEvents, + 'LlmChatCompletionSummary' + ) + assertLangChainChatCompletionSummary({ + tx, + chatSummary: langChainSummaryEvents[0], + withCallback: cbHandler + }) + + assertLangChainChatCompletionMessages({ + tx, + chatMsgs: langChainMessageEvents, + chatSummary: langChainSummaryEvents[0][1], + withCallback: cbHandler + }) + + tx.end() + end() + }) +}) + +test('should not create langchain events when not in a transaction', async (t) => { + const { agent, prompt, outputParser, model } = t.nr + + const input = { topic: 'scientist' } + const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } + + const chain = prompt.pipe(model).pipe(outputParser) + await chain.invoke(input, options) + + const events = agent.customEventAggregator.events.toArray() + assert.equal(events.length, 0, 'should not create langchain events') +}) + +test('should add llm attribute to transaction', (t, end) => { + const { agent, prompt, model } = t.nr + + const input = { topic: 'scientist' } + const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } + + helper.runInTransaction(agent, async (tx) => { + const chain = prompt.pipe(model) + await chain.invoke(input, options) + + const attributes = tx.trace.attributes.get(DESTINATIONS.TRANS_EVENT) + assert.equal(attributes.llm, true) + + tx.end() + end() + }) +}) + +test('should create span on successful runnables create', (t, end) => { + const { agent, prompt, model } = t.nr + + const input = { topic: 'scientist' } + const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } + + helper.runInTransaction(agent, async (tx) => { + const chain = prompt.pipe(model) + const result = await chain.invoke(input, options) + + assert.ok(result) + assertSegments(tx.trace.root, ['Llm/chain/Langchain/invoke'], { exact: false }) + + tx.end() + end() + }) +}) + +// testing JSON.stringify on request (input) during creation of LangChainCompletionMessage event +test('should use empty string for content property on completion message event when invalid input is used - circular reference', (t, end) => { + const { agent, prompt, outputParser, model } = t.nr + + helper.runInTransaction(agent, async (tx) => { + const input = { topic: 'scientist' } + input.myself = input + const options = { metadata: { key: 'value', hello: 'world' }, tags: ['tag1', 'tag2'] } + + const chain = prompt.pipe(model).pipe(outputParser) + await chain.invoke(input, options) + + const events = agent.customEventAggregator.events.toArray() + + const langchainEvents = filterLangchainEvents(events) + const langChainMessageEvents = filterLangchainEventsByType( + langchainEvents, + 'LlmChatCompletionMessage' + ) + + const msgEventEmptyContent = langChainMessageEvents.filter((event) => event[1].content === '') + + assert.equal(msgEventEmptyContent.length, 1, 'should have 1 event with empty content property') + + tx.end() + end() + }) +}) + +test('should create error events', (t, end) => { + const { ChatPromptTemplate } = require('@langchain/core/prompts') + const prompt = ChatPromptTemplate.fromMessages([['assistant', 'Invalid API key.']]) + const { agent, outputParser, model } = t.nr + + helper.runInTransaction(agent, async (tx) => { + const chain = prompt.pipe(model).pipe(outputParser) + + try { + await chain.invoke('') + } catch (error) { + assert.ok(error) + } + + // We should still get the same 3xLangChain and 3xLLM events as in the + // success case: + const events = agent.customEventAggregator.events.toArray() + assert.equal(events.length, 6, 'should create 6 events') + + const langchainEvents = events.filter((event) => { + const [, chainEvent] = event + return chainEvent.vendor === 'langchain' + }) + assert.equal(langchainEvents.length, 3, 'should create 3 langchain events') + const summary = langchainEvents.find((e) => e[0].type === 'LlmChatCompletionSummary')?.[1] + assert.equal(summary.error, true) + + // But, we should also get two error events: 1xLLM and 1xLangChain + const exceptions = tx.exceptions + for (const e of exceptions) { + const str = Object.prototype.toString.call(e.customAttributes) + assert.equal(str, '[object LlmErrorMessage]') + } + + tx.end() + end() + }) +}) diff --git a/test/versioned/langchain/tools.tap.js b/test/versioned/langchain/tools.tap.js deleted file mode 100644 index baf24f7343..0000000000 --- a/test/versioned/langchain/tools.tap.js +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright 2024 New Relic Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -'use strict' - -const tap = require('tap') -const helper = require('../../lib/agent_helper') -const { removeModules, removeMatchedModules } = require('../../lib/cache-buster') -// load the assertSegments assertion -require('../../lib/metrics_helper') -const { version: pkgVersion } = require('@langchain/core/package.json') -const config = { - ai_monitoring: { - enabled: true - } -} -const baseUrl = 'http://httpbin.org' -const { DESTINATIONS } = require('../../../lib/config/attribute-filter') - -tap.test('Langchain instrumentation - tools', (t) => { - t.beforeEach((t) => { - t.context.agent = helper.instrumentMockedAgent(config) - const TestTool = require('./helpers/custom-tool') - const tool = new TestTool({ - baseUrl - }) - t.context.tool = tool - t.context.input = 'langchain' - }) - - t.afterEach((t) => { - helper.unloadAgent(t.context.agent) - // bust the require-cache so it can re-instrument - removeModules(['@langchain/core']) - removeMatchedModules(/helpers\/custom-tool\.js$/) - }) - - t.test('should create span on successful tools create', (t) => { - const { agent, tool, input } = t.context - helper.runInTransaction(agent, async (tx) => { - const result = await tool.call(input) - t.ok(result) - t.assertSegments(tx.trace.root, ['Llm/tool/Langchain/node-agent-test-tool'], { exact: false }) - tx.end() - t.end() - }) - }) - - t.test('should increment tracking metric for each tool event', (t) => { - const { tool, agent, input } = t.context - helper.runInTransaction(agent, async (tx) => { - await tool.call(input) - - const metrics = agent.metrics.getOrCreateMetric( - `Supportability/Nodejs/ML/Langchain/${pkgVersion}` - ) - t.equal(metrics.callCount > 0, true) - - tx.end() - t.end() - }) - }) - - t.test('should create LlmTool event for every tool.call', (t) => { - const { agent, tool, input } = t.context - helper.runInTransaction(agent, async (tx) => { - tool.metadata = { key: 'instance-value', hello: 'world' } - tool.tags = ['tag1', 'tag2'] - await tool.call(input, { metadata: { key: 'value' }, tags: ['tag2', 'tag3'] }) - const events = agent.customEventAggregator.events.toArray() - t.equal(events.length, 1, 'should create a LlmTool event') - const [[{ type }, toolEvent]] = events - t.equal(type, 'LlmTool') - t.match(toolEvent, { - 'id': /[a-f0-9]{36}/, - 'appName': 'New Relic for Node.js tests', - 'span_id': tx.trace.root.children[0].id, - 'trace_id': tx.traceId, - 'ingest_source': 'Node', - 'vendor': 'langchain', - 'metadata.key': 'value', - 'metadata.hello': 'world', - 'tags': 'tag1,tag2,tag3', - input, - 'output': tool.fakeData[input], - 'name': tool.name, - 'description': tool.description, - 'duration': tx.trace.root.children[0].getDurationInMillis(), - 'run_id': undefined - }) - tx.end() - t.end() - }) - }) - - t.test('should add runId when a callback handler exists', (t) => { - const { BaseCallbackHandler } = require('@langchain/core/callbacks/base') - let runId - const cbHandler = BaseCallbackHandler.fromMethods({ - handleToolStart(...args) { - runId = args?.[2] - } - }) - - const { agent, tool, input } = t.context - tool.callbacks = [cbHandler] - helper.runInTransaction(agent, async (tx) => { - await tool.call(input) - const events = agent.customEventAggregator.events.toArray() - t.equal(events.length, 1, 'should create a LlmTool event') - const [[, toolEvent]] = events - - t.equal(toolEvent.run_id, runId) - tx.end() - t.end() - }) - }) - - t.test('should not create llm tool events when not in a transaction', async (t) => { - const { tool, agent, input } = t.context - await tool.call(input) - - const events = agent.customEventAggregator.events.toArray() - t.equal(events.length, 0, 'should not create llm events') - }) - - t.test('should add llm attribute to transaction', (t) => { - const { tool, agent, input } = t.context - helper.runInTransaction(agent, async (tx) => { - await tool.call(input) - - const attributes = tx.trace.attributes.get(DESTINATIONS.TRANS_EVENT) - t.equal(attributes.llm, true) - - tx.end() - t.end() - }) - }) - - t.test('should capture error events', (t) => { - const { agent, tool } = t.context - helper.runInTransaction(agent, async (tx) => { - try { - await tool.call('bad input') - } catch (error) { - t.ok(error) - } - - const events = agent.customEventAggregator.events.toArray() - t.equal(events.length, 1) - const toolEvent = events.find((e) => e[0].type === 'LlmTool')?.[1] - t.equal(toolEvent.error, true) - - const exceptions = tx.exceptions - t.equal(exceptions.length, 1) - const str = Object.prototype.toString.call(exceptions[0].customAttributes) - t.equal(str, '[object LlmErrorMessage]') - t.equal(exceptions[0].customAttributes.tool_id, toolEvent.id) - - tx.end() - t.end() - }) - }) - - t.end() -}) diff --git a/test/versioned/langchain/tools.test.js b/test/versioned/langchain/tools.test.js new file mode 100644 index 0000000000..9b4654c7de --- /dev/null +++ b/test/versioned/langchain/tools.test.js @@ -0,0 +1,167 @@ +/* + * Copyright 2024 New Relic Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +'use strict' + +const test = require('node:test') +const assert = require('node:assert') + +const { removeModules, removeMatchedModules } = require('../../lib/cache-buster') +const { assertSegments, match } = require('../../lib/custom-assertions') +const { version: pkgVersion } = require('@langchain/core/package.json') +const helper = require('../../lib/agent_helper') + +const baseUrl = 'http://httpbin.org' +const config = { + ai_monitoring: { + enabled: true + } +} +const { DESTINATIONS } = require('../../../lib/config/attribute-filter') + +test.beforeEach((ctx) => { + ctx.nr = {} + ctx.nr.agent = helper.instrumentMockedAgent(config) + const TestTool = require('./helpers/custom-tool') + const tool = new TestTool({ + baseUrl + }) + ctx.nr.tool = tool + ctx.nr.input = 'langchain' +}) + +test.afterEach((ctx) => { + helper.unloadAgent(ctx.nr.agent) + // bust the require-cache so it can re-instrument + removeModules(['@langchain/core']) + removeMatchedModules(/helpers\/custom-tool\.js$/) +}) + +test('should create span on successful tools create', (t, end) => { + const { agent, tool, input } = t.nr + helper.runInTransaction(agent, async (tx) => { + const result = await tool.call(input) + assert.ok(result) + assertSegments(tx.trace.root, ['Llm/tool/Langchain/node-agent-test-tool'], { exact: false }) + tx.end() + end() + }) +}) + +test('should increment tracking metric for each tool event', (t, end) => { + const { tool, agent, input } = t.nr + helper.runInTransaction(agent, async (tx) => { + await tool.call(input) + + const metrics = agent.metrics.getOrCreateMetric( + `Supportability/Nodejs/ML/Langchain/${pkgVersion}` + ) + assert.equal(metrics.callCount > 0, true) + + tx.end() + end() + }) +}) + +test('should create LlmTool event for every tool.call', (t, end) => { + const { agent, tool, input } = t.nr + helper.runInTransaction(agent, async (tx) => { + tool.metadata = { key: 'instance-value', hello: 'world' } + tool.tags = ['tag1', 'tag2'] + await tool.call(input, { metadata: { key: 'value' }, tags: ['tag2', 'tag3'] }) + const events = agent.customEventAggregator.events.toArray() + assert.equal(events.length, 1, 'should create a LlmTool event') + const [[{ type }, toolEvent]] = events + assert.equal(type, 'LlmTool') + match(toolEvent, { + 'id': /[a-f0-9]{36}/, + 'appName': 'New Relic for Node.js tests', + 'span_id': tx.trace.root.children[0].id, + 'trace_id': tx.traceId, + 'ingest_source': 'Node', + 'vendor': 'langchain', + 'metadata.key': 'value', + 'metadata.hello': 'world', + 'tags': 'tag1,tag2,tag3', + input, + 'output': tool.fakeData[input], + 'name': tool.name, + 'description': tool.description, + 'duration': tx.trace.root.children[0].getDurationInMillis(), + 'run_id': undefined + }) + tx.end() + end() + }) +}) + +test('should add runId when a callback handler exists', (t, end) => { + const { BaseCallbackHandler } = require('@langchain/core/callbacks/base') + let runId + const cbHandler = BaseCallbackHandler.fromMethods({ + handleToolStart(...args) { + runId = args?.[2] + } + }) + + const { agent, tool, input } = t.nr + tool.callbacks = [cbHandler] + helper.runInTransaction(agent, async (tx) => { + await tool.call(input) + const events = agent.customEventAggregator.events.toArray() + assert.equal(events.length, 1, 'should create a LlmTool event') + const [[, toolEvent]] = events + + assert.equal(toolEvent.run_id, runId) + tx.end() + end() + }) +}) + +test('should not create llm tool events when not in a transaction', async (t) => { + const { tool, agent, input } = t.nr + await tool.call(input) + + const events = agent.customEventAggregator.events.toArray() + assert.equal(events.length, 0, 'should not create llm events') +}) + +test('should add llm attribute to transaction', (t, end) => { + const { tool, agent, input } = t.nr + helper.runInTransaction(agent, async (tx) => { + await tool.call(input) + + const attributes = tx.trace.attributes.get(DESTINATIONS.TRANS_EVENT) + assert.equal(attributes.llm, true) + + tx.end() + end() + }) +}) + +test('should capture error events', (t, end) => { + const { agent, tool } = t.nr + helper.runInTransaction(agent, async (tx) => { + try { + await tool.call('bad input') + } catch (error) { + assert.ok(error) + } + + const events = agent.customEventAggregator.events.toArray() + assert.equal(events.length, 1) + const toolEvent = events.find((e) => e[0].type === 'LlmTool')?.[1] + assert.equal(toolEvent.error, true) + + const exceptions = tx.exceptions + assert.equal(exceptions.length, 1) + const str = Object.prototype.toString.call(exceptions[0].customAttributes) + assert.equal(str, '[object LlmErrorMessage]') + assert.equal(exceptions[0].customAttributes.tool_id, toolEvent.id) + + tx.end() + end() + }) +}) diff --git a/test/versioned/langchain/vectorstore.tap.js b/test/versioned/langchain/vectorstore.tap.js deleted file mode 100644 index 9d0a3520fb..0000000000 --- a/test/versioned/langchain/vectorstore.tap.js +++ /dev/null @@ -1,256 +0,0 @@ -/* - * Copyright 2024 New Relic Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -'use strict' - -const tap = require('tap') -const helper = require('../../lib/agent_helper') -const { removeModules } = require('../../lib/cache-buster') -// load the assertSegments assertion -require('../../lib/metrics_helper') -const fs = require('fs') -let pkgVersion -try { - ;({ version: pkgVersion } = JSON.parse( - fs.readFileSync( - `${__dirname}/node_modules/@langchain/community/node_modules/@langchain/core/package.json` - ) - )) -} catch { - ;({ version: pkgVersion } = require('@langchain/core/package.json')) -} -const createOpenAIMockServer = require('../openai/mock-server') -const { filterLangchainEvents, filterLangchainEventsByType } = require('./common') -const { DESTINATIONS } = require('../../../lib/config/attribute-filter') -const params = require('../../lib/params') -const { Document } = require('@langchain/core/documents') - -const config = { - ai_monitoring: { - enabled: true - } -} - -tap.test('Langchain instrumentation - vectorstore', (t) => { - t.autoend() - - t.beforeEach(async (t) => { - const { host, port, server } = await createOpenAIMockServer() - t.context.server = server - t.context.agent = helper.instrumentMockedAgent(config) - const { OpenAIEmbeddings } = require('@langchain/openai') - - const { Client } = require('@elastic/elasticsearch') - const clientArgs = { - client: new Client({ - node: `http://${params.elastic_host}:${params.elastic_port}` - }) - } - const { ElasticVectorSearch } = require('@langchain/community/vectorstores/elasticsearch') - - t.context.embedding = new OpenAIEmbeddings({ - openAIApiKey: 'fake-key', - configuration: { - baseURL: `http://${host}:${port}` - } - }) - const docs = [ - new Document({ - metadata: { id: '2' }, - pageContent: 'This is an embedding test.' - }) - ] - const vectorStore = new ElasticVectorSearch(t.context.embedding, clientArgs) - await vectorStore.deleteIfExists() - await vectorStore.addDocuments(docs) - t.context.vs = vectorStore - }) - - t.afterEach(async (t) => { - t.context?.server?.close() - helper.unloadAgent(t.context.agent) - // bust the require-cache so it can re-instrument - removeModules(['@langchain/core', 'openai', '@elastic', '@langchain/community']) - }) - - t.test('should create vectorstore events for every similarity search call', (t) => { - const { agent, vs } = t.context - - helper.runInNamedTransaction(agent, async (tx) => { - await vs.similaritySearch('This is an embedding test.', 1) - - const events = agent.customEventAggregator.events.toArray() - t.equal(events.length, 3, 'should create 3 events') - - const langchainEvents = events.filter((event) => { - const [, chainEvent] = event - return chainEvent.vendor === 'langchain' - }) - - t.equal(langchainEvents.length, 2, 'should create 2 langchain events') - - tx.end() - t.end() - }) - }) - - t.test('should create span on successful vectorstore create', (t) => { - const { agent, vs } = t.context - helper.runInTransaction(agent, async (tx) => { - const result = await vs.similaritySearch('This is an embedding test.', 1) - t.ok(result) - t.assertSegments(tx.trace.root, ['Llm/vectorstore/Langchain/similaritySearch'], { - exact: false - }) - tx.end() - t.end() - }) - }) - - t.test('should increment tracking metric for each langchain vectorstore event', (t) => { - const { agent, vs } = t.context - - helper.runInTransaction(agent, async (tx) => { - await vs.similaritySearch('This is an embedding test.', 1) - - const metrics = agent.metrics.getOrCreateMetric( - `Supportability/Nodejs/ML/Langchain/${pkgVersion}` - ) - t.equal(metrics.callCount > 0, true) - - tx.end() - t.end() - }) - }) - - t.test( - 'should create vectorstore events for every similarity search call with embeddings', - (t) => { - const { agent, vs } = t.context - - helper.runInNamedTransaction(agent, async (tx) => { - await vs.similaritySearch('This is an embedding test.', 1) - - const events = agent.customEventAggregator.events.toArray() - const langchainEvents = filterLangchainEvents(events) - - const vectorSearchResultEvents = filterLangchainEventsByType( - langchainEvents, - 'LlmVectorSearchResult' - ) - - const vectorSearchEvents = filterLangchainEventsByType(langchainEvents, 'LlmVectorSearch') - - t.langchainVectorSearch({ - tx, - vectorSearch: vectorSearchEvents[0], - responseDocumentSize: 1 - }) - t.langchainVectorSearchResult({ - tx, - vectorSearchResult: vectorSearchResultEvents, - vectorSearchId: vectorSearchEvents[0][1].id - }) - - tx.end() - t.end() - }) - } - ) - - t.test( - 'should create only vectorstore search event for similarity search call with embeddings and invalid metadata filter', - (t) => { - const { agent, vs } = t.context - - helper.runInNamedTransaction(agent, async (tx) => { - // search for documents with invalid filter - await vs.similaritySearch('This is an embedding test.', 1, { - a: 'some filter' - }) - - const events = agent.customEventAggregator.events.toArray() - const langchainEvents = filterLangchainEvents(events) - - const vectorSearchResultEvents = filterLangchainEventsByType( - langchainEvents, - 'LlmVectorSearchResult' - ) - - const vectorSearchEvents = filterLangchainEventsByType(langchainEvents, 'LlmVectorSearch') - - // there are no documents in vector store with that filter - t.equal(vectorSearchResultEvents.length, 0, 'should have 0 events') - t.langchainVectorSearch({ - tx, - vectorSearch: vectorSearchEvents[0], - responseDocumentSize: 0 - }) - - tx.end() - t.end() - }) - } - ) - - t.test('should not create vectorstore events when not in a transaction', async (t) => { - const { agent, vs } = t.context - - await vs.similaritySearch('This is an embedding test.', 1) - - const events = agent.customEventAggregator.events.toArray() - t.equal(events.length, 0, 'should not create vectorstore events') - t.end() - }) - - t.test('should add llm attribute to transaction', (t) => { - const { agent, vs } = t.context - - helper.runInTransaction(agent, async (tx) => { - await vs.similaritySearch('This is an embedding test.', 1) - - const attributes = tx.trace.attributes.get(DESTINATIONS.TRANS_EVENT) - t.equal(attributes.llm, true) - - tx.end() - t.end() - }) - }) - - t.test('should create error events', (t) => { - const { agent, vs } = t.context - - helper.runInNamedTransaction(agent, async (tx) => { - try { - await vs.similaritySearch('Embedding not allowed.', 1) - } catch (error) { - t.ok(error) - } - - const events = agent.customEventAggregator.events.toArray() - // Only LlmEmbedding and LlmVectorSearch events will be created - // LangChainVectorSearchResult event won't be created since there was an error - t.equal(events.length, 2, 'should create 2 events') - - const langchainEvents = events.filter((event) => { - const [, chainEvent] = event - return chainEvent.vendor === 'langchain' - }) - - t.equal(langchainEvents.length, 1, 'should create 1 langchain vectorsearch event') - t.equal(langchainEvents[0][1].error, true) - - // But, we should also get two error events: 1xLLM and 1xLangChain - const exceptions = tx.exceptions - for (const e of exceptions) { - const str = Object.prototype.toString.call(e.customAttributes) - t.equal(str, '[object LlmErrorMessage]') - } - - tx.end() - t.end() - }) - }) -}) diff --git a/test/versioned/langchain/vectorstore.test.js b/test/versioned/langchain/vectorstore.test.js new file mode 100644 index 0000000000..57d3c71799 --- /dev/null +++ b/test/versioned/langchain/vectorstore.test.js @@ -0,0 +1,242 @@ +/* + * Copyright 2024 New Relic Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +'use strict' + +const test = require('node:test') +const assert = require('node:assert') + +const { removeModules } = require('../../lib/cache-buster') +const { assertSegments } = require('../../lib/custom-assertions') +const { + assertLangChainVectorSearch, + assertLangChainVectorSearchResult, + filterLangchainEvents, + filterLangchainEventsByType +} = require('./common') +const { version: pkgVersion } = require('@langchain/core/package.json') +const { Document } = require('@langchain/core/documents') +const createOpenAIMockServer = require('../openai/mock-server') +const params = require('../../lib/params') +const helper = require('../../lib/agent_helper') + +const config = { + ai_monitoring: { + enabled: true + } +} +const { DESTINATIONS } = require('../../../lib/config/attribute-filter') + +test.beforeEach(async (ctx) => { + ctx.nr = {} + const { host, port, server } = await createOpenAIMockServer() + ctx.nr.server = server + ctx.nr.agent = helper.instrumentMockedAgent(config) + const { OpenAIEmbeddings } = require('@langchain/openai') + + const { Client } = require('@elastic/elasticsearch') + const clientArgs = { + client: new Client({ + node: `http://${params.elastic_host}:${params.elastic_port}` + }) + } + const { ElasticVectorSearch } = require('@langchain/community/vectorstores/elasticsearch') + + ctx.nr.embedding = new OpenAIEmbeddings({ + openAIApiKey: 'fake-key', + configuration: { + baseURL: `http://${host}:${port}` + } + }) + const docs = [ + new Document({ + metadata: { id: '2' }, + pageContent: 'This is an embedding test.' + }) + ] + const vectorStore = new ElasticVectorSearch(ctx.nr.embedding, clientArgs) + await vectorStore.deleteIfExists() + await vectorStore.addDocuments(docs) + ctx.nr.vs = vectorStore +}) + +test.afterEach(async (ctx) => { + ctx.nr?.server?.close() + helper.unloadAgent(ctx.nr.agent) + // bust the require-cache so it can re-instrument + removeModules(['@langchain/core', 'openai', '@elastic', '@langchain/community']) +}) + +test('should create vectorstore events for every similarity search call', (t, end) => { + const { agent, vs } = t.nr + + helper.runInNamedTransaction(agent, async (tx) => { + await vs.similaritySearch('This is an embedding test.', 1) + + const events = agent.customEventAggregator.events.toArray() + assert.equal(events.length, 3, 'should create 3 events') + + const langchainEvents = events.filter((event) => { + const [, chainEvent] = event + return chainEvent.vendor === 'langchain' + }) + + assert.equal(langchainEvents.length, 2, 'should create 2 langchain events') + + tx.end() + end() + }) +}) + +test('should create span on successful vectorstore create', (t, end) => { + const { agent, vs } = t.nr + helper.runInTransaction(agent, async (tx) => { + const result = await vs.similaritySearch('This is an embedding test.', 1) + assert.ok(result) + assertSegments(tx.trace.root, ['Llm/vectorstore/Langchain/similaritySearch'], { + exact: false + }) + tx.end() + end() + }) +}) + +test('should increment tracking metric for each langchain vectorstore event', (t, end) => { + const { agent, vs } = t.nr + + helper.runInTransaction(agent, async (tx) => { + await vs.similaritySearch('This is an embedding test.', 1) + + const metrics = agent.metrics.getOrCreateMetric( + `Supportability/Nodejs/ML/Langchain/${pkgVersion}` + ) + assert.equal(metrics.callCount > 0, true) + + tx.end() + end() + }) +}) + +test('should create vectorstore events for every similarity search call with embeddings', (t, end) => { + const { agent, vs } = t.nr + + helper.runInNamedTransaction(agent, async (tx) => { + await vs.similaritySearch('This is an embedding test.', 1) + + const events = agent.customEventAggregator.events.toArray() + const langchainEvents = filterLangchainEvents(events) + + const vectorSearchResultEvents = filterLangchainEventsByType( + langchainEvents, + 'LlmVectorSearchResult' + ) + + const vectorSearchEvents = filterLangchainEventsByType(langchainEvents, 'LlmVectorSearch') + + assertLangChainVectorSearch({ + tx, + vectorSearch: vectorSearchEvents[0], + responseDocumentSize: 1 + }) + assertLangChainVectorSearchResult({ + tx, + vectorSearchResult: vectorSearchResultEvents, + vectorSearchId: vectorSearchEvents[0][1].id + }) + + tx.end() + end() + }) +}) + +test('should create only vectorstore search event for similarity search call with embeddings and invalid metadata filter', (t, end) => { + const { agent, vs } = t.nr + + helper.runInNamedTransaction(agent, async (tx) => { + // search for documents with invalid filter + await vs.similaritySearch('This is an embedding test.', 1, { + a: 'some filter' + }) + + const events = agent.customEventAggregator.events.toArray() + const langchainEvents = filterLangchainEvents(events) + + const vectorSearchResultEvents = filterLangchainEventsByType( + langchainEvents, + 'LlmVectorSearchResult' + ) + + const vectorSearchEvents = filterLangchainEventsByType(langchainEvents, 'LlmVectorSearch') + + // there are no documents in vector store with that filter + assert.equal(vectorSearchResultEvents.length, 0, 'should have 0 events') + assertLangChainVectorSearch({ + tx, + vectorSearch: vectorSearchEvents[0], + responseDocumentSize: 0 + }) + + tx.end() + end() + }) +}) + +test('should not create vectorstore events when not in a transaction', async (t) => { + const { agent, vs } = t.nr + + await vs.similaritySearch('This is an embedding test.', 1) + + const events = agent.customEventAggregator.events.toArray() + assert.equal(events.length, 0, 'should not create vectorstore events') +}) + +test('should add llm attribute to transaction', (t, end) => { + const { agent, vs } = t.nr + + helper.runInTransaction(agent, async (tx) => { + await vs.similaritySearch('This is an embedding test.', 1) + + const attributes = tx.trace.attributes.get(DESTINATIONS.TRANS_EVENT) + assert.equal(attributes.llm, true) + + tx.end() + end() + }) +}) + +test('should create error events', (t, end) => { + const { agent, vs } = t.nr + + helper.runInNamedTransaction(agent, async (tx) => { + try { + await vs.similaritySearch('Embedding not allowed.', 1) + } catch (error) { + assert.ok(error) + } + + const events = agent.customEventAggregator.events.toArray() + // Only LlmEmbedding and LlmVectorSearch events will be created + // LangChainVectorSearchResult event won't be created since there was an error + assert.equal(events.length, 2, 'should create 2 events') + + const langchainEvents = events.filter((event) => { + const [, chainEvent] = event + return chainEvent.vendor === 'langchain' + }) + + assert.equal(langchainEvents.length, 1, 'should create 1 langchain vectorsearch event') + assert.equal(langchainEvents[0][1].error, true) + + // But, we should also get two error events: 1xLLM and 1xLangChain + const exceptions = tx.exceptions + for (const e of exceptions) { + const str = Object.prototype.toString.call(e.customAttributes) + assert.equal(str, '[object LlmErrorMessage]') + } + + tx.end() + end() + }) +}) diff --git a/test/versioned/openai/chat-completions.tap.js b/test/versioned/openai/chat-completions.tap.js deleted file mode 100644 index 322691bebb..0000000000 --- a/test/versioned/openai/chat-completions.tap.js +++ /dev/null @@ -1,511 +0,0 @@ -/* - * Copyright 2023 New Relic Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -/* - * Copyright 2023 New Relic Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -'use strict' - -const tap = require('tap') -const helper = require('../../lib/agent_helper') -// load the assertSegments assertion -require('../../lib/metrics_helper') -const { - AI: { OPENAI } -} = require('../../../lib/metrics/names') -const responses = require('./mock-responses') -const { beforeHook, afterEachHook, afterHook } = require('./common') -const semver = require('semver') -const fs = require('fs') -// have to read and not require because openai does not export the package.json -const { version: pkgVersion } = JSON.parse( - fs.readFileSync(`${__dirname}/node_modules/openai/package.json`) -) -const { DESTINATIONS } = require('../../../lib/config/attribute-filter') -const TRACKING_METRIC = `Supportability/Nodejs/ML/OpenAI/${pkgVersion}` - -tap.test('OpenAI instrumentation - chat completions', (t) => { - t.autoend() - - t.before(beforeHook.bind(null, t)) - - t.afterEach(afterEachHook.bind(null, t)) - - t.teardown(afterHook.bind(null, t)) - - t.test('should create span on successful chat completion create', (test) => { - const { client, agent, host, port } = t.context - helper.runInTransaction(agent, async (tx) => { - const results = await client.chat.completions.create({ - messages: [{ role: 'user', content: 'You are a mathematician.' }] - }) - - test.notOk(results.headers, 'should remove response headers from user result') - test.equal(results.choices[0].message.content, '1 plus 2 is 3.') - - test.assertSegments( - tx.trace.root, - [OPENAI.COMPLETION, [`External/${host}:${port}/chat/completions`]], - { exact: false } - ) - tx.end() - test.end() - }) - }) - - t.test('should increment tracking metric for each chat completion event', (test) => { - const { client, agent } = t.context - helper.runInTransaction(agent, async (tx) => { - await client.chat.completions.create({ - messages: [{ role: 'user', content: 'You are a mathematician.' }] - }) - - const metrics = agent.metrics.getOrCreateMetric(TRACKING_METRIC) - t.equal(metrics.callCount > 0, true) - - tx.end() - test.end() - }) - }) - - t.test('should create chat completion message and summary for every message sent', (test) => { - const { client, agent } = t.context - helper.runInTransaction(agent, async (tx) => { - const model = 'gpt-3.5-turbo-0613' - const content = 'You are a mathematician.' - await client.chat.completions.create({ - max_tokens: 100, - temperature: 0.5, - model, - messages: [ - { role: 'user', content }, - { role: 'user', content: 'What does 1 plus 1 equal?' } - ] - }) - - const events = agent.customEventAggregator.events.toArray() - test.equal(events.length, 4, 'should create a chat completion message and summary event') - const chatMsgs = events.filter(([{ type }]) => type === 'LlmChatCompletionMessage') - test.llmMessages({ - tx, - chatMsgs, - model, - id: 'chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTeat', - resContent: '1 plus 2 is 3.', - reqContent: content - }) - - const chatSummary = events.filter(([{ type }]) => type === 'LlmChatCompletionSummary')[0] - test.llmSummary({ tx, model, chatSummary, tokenUsage: true }) - tx.end() - test.end() - }) - }) - - if (semver.gte(pkgVersion, '4.12.2')) { - t.test('should create span on successful chat completion stream create', (test) => { - const { client, agent, host, port } = t.context - helper.runInTransaction(agent, async (tx) => { - const content = 'Streamed response' - const stream = await client.chat.completions.create({ - stream: true, - messages: [{ role: 'user', content }] - }) - - let chunk = {} - let res = '' - for await (chunk of stream) { - res += chunk.choices[0]?.delta?.content - } - test.notOk(chunk.headers, 'should remove response headers from user result') - test.equal(chunk.choices[0].message.role, 'assistant') - const expectedRes = responses.get(content) - test.equal(chunk.choices[0].message.content, expectedRes.streamData) - test.equal(chunk.choices[0].message.content, res) - - test.assertSegments( - tx.trace.root, - [OPENAI.COMPLETION, [`External/${host}:${port}/chat/completions`]], - { exact: false } - ) - tx.end() - test.end() - }) - }) - - t.test( - 'should create chat completion message and summary for every message sent in stream', - (test) => { - const { client, agent } = t.context - helper.runInTransaction(agent, async (tx) => { - const content = 'Streamed response' - const model = 'gpt-4' - const stream = await client.chat.completions.create({ - max_tokens: 100, - temperature: 0.5, - model, - messages: [ - { role: 'user', content }, - { role: 'user', content: 'What does 1 plus 1 equal?' } - ], - stream: true - }) - - let res = '' - - let i = 0 - for await (const chunk of stream) { - res += chunk.choices[0]?.delta?.content - - // I tried to doing stream.controller.abort like their docs say - // but this didn't break - if (i === 10) { - break - } - i++ - } - - const events = agent.customEventAggregator.events.toArray() - test.equal(events.length, 4, 'should create a chat completion message and summary event') - const chatMsgs = events.filter(([{ type }]) => type === 'LlmChatCompletionMessage') - test.llmMessages({ - tx, - chatMsgs, - id: 'chatcmpl-8MzOfSMbLxEy70lYAolSwdCzfguQZ', - model, - resContent: res, - reqContent: content - }) - - const chatSummary = events.filter(([{ type }]) => type === 'LlmChatCompletionSummary')[0] - test.llmSummary({ tx, model, chatSummary }) - tx.end() - test.end() - }) - } - ) - - t.test('should call the tokenCountCallback in streaming', (test) => { - const { client, agent } = t.context - const promptContent = 'Streamed response' - const promptContent2 = 'What does 1 plus 1 equal?' - let res = '' - const expectedModel = 'gpt-4' - const api = helper.getAgentApi() - function cb(model, content) { - t.equal(model, expectedModel) - if (content === promptContent || content === promptContent2) { - return 53 - } else if (content === res) { - return 11 - } - } - api.setLlmTokenCountCallback(cb) - test.teardown(() => { - delete agent.llm.tokenCountCallback - }) - helper.runInTransaction(agent, async (tx) => { - const stream = await client.chat.completions.create({ - max_tokens: 100, - temperature: 0.5, - model: expectedModel, - messages: [ - { role: 'user', content: promptContent }, - { role: 'user', content: promptContent2 } - ], - stream: true - }) - - for await (const chunk of stream) { - res += chunk.choices[0]?.delta?.content - } - - const events = agent.customEventAggregator.events.toArray() - const chatMsgs = events.filter(([{ type }]) => type === 'LlmChatCompletionMessage') - test.llmMessages({ - tokenUsage: true, - tx, - chatMsgs, - id: 'chatcmpl-8MzOfSMbLxEy70lYAolSwdCzfguQZ', - model: expectedModel, - resContent: res, - reqContent: promptContent - }) - - tx.end() - test.end() - }) - }) - - t.test('handles error in stream', (test) => { - const { client, agent } = t.context - helper.runInTransaction(agent, async (tx) => { - const content = 'bad stream' - const model = 'gpt-4' - const stream = await client.chat.completions.create({ - max_tokens: 100, - temperature: 0.5, - model, - messages: [ - { role: 'user', content }, - { role: 'user', content: 'What does 1 plus 1 equal?' } - ], - stream: true - }) - - let res = '' - - try { - for await (const chunk of stream) { - res += chunk.choices[0]?.delta?.content - } - } catch (err) { - test.ok(res) - test.ok(err.message, 'exceeded count') - const events = agent.customEventAggregator.events.toArray() - test.equal(events.length, 4) - const chatSummary = events.filter(([{ type }]) => type === 'LlmChatCompletionSummary')[0] - test.llmSummary({ tx, model, chatSummary, error: true }) - test.equal(tx.exceptions.length, 1) - // only asserting message and completion_id as the rest of the attrs - // are asserted in other tests - test.match(tx.exceptions[0], { - customAttributes: { - 'error.message': 'Premature close', - 'completion_id': /\w{32}/ - } - }) - tx.end() - test.end() - } - }) - }) - - t.test('should not create llm events when ai_monitoring.streaming.enabled is false', (test) => { - const { client, agent } = t.context - agent.config.ai_monitoring.streaming.enabled = false - helper.runInTransaction(agent, async (tx) => { - const content = 'Streamed response' - const model = 'gpt-4' - const stream = await client.chat.completions.create({ - max_tokens: 100, - temperature: 0.5, - model, - messages: [{ role: 'user', content }], - stream: true - }) - - let res = '' - let chunk = {} - - for await (chunk of stream) { - res += chunk.choices[0]?.delta?.content - } - const expectedRes = responses.get(content) - test.equal(res, expectedRes.streamData) - - const events = agent.customEventAggregator.events.toArray() - test.equal(events.length, 0, 'should not llm events when streaming is disabled') - const metrics = agent.metrics.getOrCreateMetric(TRACKING_METRIC) - test.equal(metrics.callCount > 0, true) - const attributes = tx.trace.attributes.get(DESTINATIONS.TRANS_EVENT) - test.equal(attributes.llm, true) - const streamingDisabled = agent.metrics.getOrCreateMetric( - 'Supportability/Nodejs/ML/Streaming/Disabled' - ) - test.equal(streamingDisabled.callCount > 0, true) - tx.end() - test.end() - }) - }) - } else { - t.test('should not instrument streams when openai < 4.12.2', (test) => { - const { client, agent, host, port } = t.context - helper.runInTransaction(agent, async (tx) => { - const content = 'Streamed response' - const stream = await client.chat.completions.create({ - stream: true, - messages: [{ role: 'user', content }] - }) - - let chunk = {} - let res = '' - for await (chunk of stream) { - res += chunk.choices[0]?.delta?.content - } - - test.ok(res) - const events = agent.customEventAggregator.events.toArray() - test.equal(events.length, 0) - // we will still record the external segment but not the chat completion - test.assertSegments(tx.trace.root, [ - 'timers.setTimeout', - `External/${host}:${port}/chat/completions` - ]) - tx.end() - test.end() - }) - }) - } - - t.test('should not create llm events when not in a transaction', async (test) => { - const { client, agent } = t.context - await client.chat.completions.create({ - messages: [{ role: 'user', content: 'You are a mathematician.' }] - }) - - const events = agent.customEventAggregator.events.toArray() - test.equal(events.length, 0, 'should not create llm events') - }) - - t.test('auth errors should be tracked', (test) => { - const { client, agent } = t.context - helper.runInTransaction(agent, async (tx) => { - try { - await client.chat.completions.create({ - messages: [{ role: 'user', content: 'Invalid API key.' }] - }) - } catch {} - - test.equal(tx.exceptions.length, 1) - t.match(tx.exceptions[0], { - error: { - status: 401, - code: 'invalid_api_key', - param: 'null' - }, - customAttributes: { - 'http.statusCode': 401, - 'error.message': /Incorrect API key provided:/, - 'error.code': 'invalid_api_key', - 'error.param': 'null', - 'completion_id': /[\w\d]{32}/ - }, - agentAttributes: { - spanId: /[\w\d]+/ - } - }) - - const summary = agent.customEventAggregator.events.toArray().find((e) => { - return e[0].type === 'LlmChatCompletionSummary' - }) - test.ok(summary) - test.equal(summary[1].error, true) - - tx.end() - test.end() - }) - }) - - t.test('invalid payload errors should be tracked', (test) => { - const { client, agent } = t.context - helper.runInTransaction(agent, async (tx) => { - try { - await client.chat.completions.create({ - messages: [{ role: 'bad-role', content: 'Invalid role.' }] - }) - } catch {} - - test.equal(tx.exceptions.length, 1) - test.match(tx.exceptions[0], { - error: { - status: 400, - code: null, - param: null - }, - customAttributes: { - 'http.statusCode': 400, - 'error.message': /'bad-role' is not one of/, - 'error.code': null, - 'error.param': null, - 'completion_id': /\w{32}/ - }, - agentAttributes: { - spanId: /\w+/ - } - }) - - tx.end() - test.end() - }) - }) - - t.test('should add llm attribute to transaction', (test) => { - const { client, agent } = t.context - helper.runInTransaction(agent, async (tx) => { - await client.chat.completions.create({ - messages: [{ role: 'user', content: 'You are a mathematician.' }] - }) - - const attributes = tx.trace.attributes.get(DESTINATIONS.TRANS_EVENT) - t.equal(attributes.llm, true) - - tx.end() - test.end() - }) - }) - - t.test('should record LLM custom events with attributes', (test) => { - const { client, agent } = t.context - const api = helper.getAgentApi() - - helper.runInTransaction(agent, () => { - api.withLlmCustomAttributes({ 'llm.shared': true, 'llm.path': 'root/' }, async () => { - await api.withLlmCustomAttributes( - { 'llm.path': 'root/branch1', 'llm.attr1': true }, - async () => { - agent.config.ai_monitoring.streaming.enabled = true - const model = 'gpt-3.5-turbo-0613' - const content = 'You are a mathematician.' - await client.chat.completions.create({ - max_tokens: 100, - temperature: 0.5, - model, - messages: [ - { role: 'user', content }, - { role: 'user', content: 'What does 1 plus 1 equal?' } - ] - }) - } - ) - - await api.withLlmCustomAttributes( - { 'llm.path': 'root/branch2', 'llm.attr2': true }, - async () => { - agent.config.ai_monitoring.streaming.enabled = true - const model = 'gpt-3.5-turbo-0613' - const content = 'You are a mathematician.' - await client.chat.completions.create({ - max_tokens: 100, - temperature: 0.5, - model, - messages: [ - { role: 'user', content }, - { role: 'user', content: 'What does 1 plus 2 equal?' } - ] - }) - } - ) - - const events = agent.customEventAggregator.events.toArray().map((event) => event[1]) - - events.forEach((event) => { - t.ok(event['llm.shared']) - if (event['llm.path'] === 'root/branch1') { - t.ok(event['llm.attr1']) - t.notOk(event['llm.attr2']) - } else { - t.ok(event['llm.attr2']) - t.notOk(event['llm.attr1']) - } - }) - - test.end() - }) - }) - }) -}) diff --git a/test/versioned/openai/chat-completions.test.js b/test/versioned/openai/chat-completions.test.js new file mode 100644 index 0000000000..e9bd94043a --- /dev/null +++ b/test/versioned/openai/chat-completions.test.js @@ -0,0 +1,529 @@ +/* + * Copyright 2024 New Relic Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +'use strict' + +const test = require('node:test') +const assert = require('node:assert') +const fs = require('node:fs') +const semver = require('semver') + +const { removeModules } = require('../../lib/cache-buster') +const { assertSegments, match } = require('../../lib/custom-assertions') +const { assertChatCompletionMessages, assertChatCompletionSummary } = require('./common') +const createOpenAIMockServer = require('./mock-server') +const helper = require('../../lib/agent_helper') + +const { + AI: { OPENAI } +} = require('../../../lib/metrics/names') +// have to read and not require because openai does not export the package.json +const { version: pkgVersion } = JSON.parse( + fs.readFileSync(`${__dirname}/node_modules/openai/package.json`) +) +const { DESTINATIONS } = require('../../../lib/config/attribute-filter') +const responses = require('./mock-responses') +const TRACKING_METRIC = `Supportability/Nodejs/ML/OpenAI/${pkgVersion}` + +test.beforeEach(async (ctx) => { + ctx.nr = {} + const { host, port, server } = await createOpenAIMockServer() + ctx.nr.host = host + ctx.nr.port = port + ctx.nr.server = server + ctx.nr.agent = helper.instrumentMockedAgent({ + ai_monitoring: { + enabled: true + }, + streaming: { + enabled: true + } + }) + const OpenAI = require('openai') + ctx.nr.client = new OpenAI({ + apiKey: 'fake-versioned-test-key', + baseURL: `http://${host}:${port}` + }) +}) + +test.afterEach((ctx) => { + helper.unloadAgent(ctx.nr.agent) + ctx.nr.server?.close() + removeModules('openai') +}) + +test('should create span on successful chat completion create', (t, end) => { + const { client, agent, host, port } = t.nr + helper.runInTransaction(agent, async (tx) => { + const results = await client.chat.completions.create({ + messages: [{ role: 'user', content: 'You are a mathematician.' }] + }) + + assert.equal(results.headers, undefined, 'should remove response headers from user result') + assert.equal(results.choices[0].message.content, '1 plus 2 is 3.') + + assertSegments( + tx.trace.root, + [OPENAI.COMPLETION, [`External/${host}:${port}/chat/completions`]], + { exact: false } + ) + + tx.end() + end() + }) +}) + +test('should increment tracking metric for each chat completion event', (t, end) => { + const { client, agent } = t.nr + helper.runInTransaction(agent, async (tx) => { + await client.chat.completions.create({ + messages: [{ role: 'user', content: 'You are a mathematician.' }] + }) + + const metrics = agent.metrics.getOrCreateMetric(TRACKING_METRIC) + assert.equal(metrics.callCount > 0, true) + + tx.end() + end() + }) +}) + +test('should create chat completion message and summary for every message sent', (t, end) => { + const { client, agent } = t.nr + helper.runInTransaction(agent, async (tx) => { + const model = 'gpt-3.5-turbo-0613' + const content = 'You are a mathematician.' + await client.chat.completions.create({ + max_tokens: 100, + temperature: 0.5, + model, + messages: [ + { role: 'user', content }, + { role: 'user', content: 'What does 1 plus 1 equal?' } + ] + }) + + const events = agent.customEventAggregator.events.toArray() + assert.equal(events.length, 4, 'should create a chat completion message and summary event') + const chatMsgs = events.filter(([{ type }]) => type === 'LlmChatCompletionMessage') + assertChatCompletionMessages({ + tx, + chatMsgs, + model, + id: 'chatcmpl-87sb95K4EF2nuJRcTs43Tm9ntTeat', + resContent: '1 plus 2 is 3.', + reqContent: content + }) + + const chatSummary = events.filter(([{ type }]) => type === 'LlmChatCompletionSummary')[0] + assertChatCompletionSummary({ tx, model, chatSummary, tokenUsage: true }) + + tx.end() + end() + }) +}) + +if (semver.gte(pkgVersion, '4.12.2')) { + test('should create span on successful chat completion stream create', (t, end) => { + const { client, agent, host, port } = t.nr + helper.runInTransaction(agent, async (tx) => { + const content = 'Streamed response' + const stream = await client.chat.completions.create({ + stream: true, + messages: [{ role: 'user', content }] + }) + + let chunk = {} + let res = '' + for await (chunk of stream) { + res += chunk.choices[0]?.delta?.content + } + assert.equal(chunk.headers, undefined, 'should remove response headers from user result') + assert.equal(chunk.choices[0].message.role, 'assistant') + const expectedRes = responses.get(content) + assert.equal(chunk.choices[0].message.content, expectedRes.streamData) + assert.equal(chunk.choices[0].message.content, res) + + assertSegments( + tx.trace.root, + [OPENAI.COMPLETION, [`External/${host}:${port}/chat/completions`]], + { exact: false } + ) + + tx.end() + end() + }) + }) + + test('should create chat completion message and summary for every message sent in stream', (t, end) => { + const { client, agent } = t.nr + helper.runInTransaction(agent, async (tx) => { + const content = 'Streamed response' + const model = 'gpt-4' + const stream = await client.chat.completions.create({ + max_tokens: 100, + temperature: 0.5, + model, + messages: [ + { role: 'user', content }, + { role: 'user', content: 'What does 1 plus 1 equal?' } + ], + stream: true + }) + + let res = '' + + let i = 0 + for await (const chunk of stream) { + res += chunk.choices[0]?.delta?.content + + // I tried to doing stream.controller.abort like their docs say + // but this didn't break + if (i === 10) { + break + } + i++ + } + + const events = agent.customEventAggregator.events.toArray() + assert.equal(events.length, 4, 'should create a chat completion message and summary event') + const chatMsgs = events.filter(([{ type }]) => type === 'LlmChatCompletionMessage') + assertChatCompletionMessages({ + tx, + chatMsgs, + id: 'chatcmpl-8MzOfSMbLxEy70lYAolSwdCzfguQZ', + model, + resContent: res, + reqContent: content + }) + + const chatSummary = events.filter(([{ type }]) => type === 'LlmChatCompletionSummary')[0] + assertChatCompletionSummary({ tx, model, chatSummary }) + + tx.end() + end() + }) + }) + + test('should call the tokenCountCallback in streaming', (t, end) => { + const { client, agent } = t.nr + const promptContent = 'Streamed response' + const promptContent2 = 'What does 1 plus 1 equal?' + let res = '' + const expectedModel = 'gpt-4' + const api = helper.getAgentApi() + function cb(model, content) { + assert.equal(model, expectedModel) + if (content === promptContent || content === promptContent2) { + return 53 + } else if (content === res) { + return 11 + } + } + api.setLlmTokenCountCallback(cb) + + helper.runInTransaction(agent, async (tx) => { + const stream = await client.chat.completions.create({ + max_tokens: 100, + temperature: 0.5, + model: expectedModel, + messages: [ + { role: 'user', content: promptContent }, + { role: 'user', content: promptContent2 } + ], + stream: true + }) + + for await (const chunk of stream) { + res += chunk.choices[0]?.delta?.content + } + + const events = agent.customEventAggregator.events.toArray() + const chatMsgs = events.filter(([{ type }]) => type === 'LlmChatCompletionMessage') + assertChatCompletionMessages({ + tokenUsage: true, + tx, + chatMsgs, + id: 'chatcmpl-8MzOfSMbLxEy70lYAolSwdCzfguQZ', + model: expectedModel, + resContent: res, + reqContent: promptContent + }) + + tx.end() + end() + }) + }) + + test('handles error in stream', (t, end) => { + const { client, agent } = t.nr + helper.runInTransaction(agent, async (tx) => { + const content = 'bad stream' + const model = 'gpt-4' + const stream = await client.chat.completions.create({ + max_tokens: 100, + temperature: 0.5, + model, + messages: [ + { role: 'user', content }, + { role: 'user', content: 'What does 1 plus 1 equal?' } + ], + stream: true + }) + + let res = '' + + try { + for await (const chunk of stream) { + res += chunk.choices[0]?.delta?.content + } + } catch (err) { + assert.ok(res) + assert.ok(err.message, 'exceeded count') + const events = agent.customEventAggregator.events.toArray() + assert.equal(events.length, 4) + const chatSummary = events.filter(([{ type }]) => type === 'LlmChatCompletionSummary')[0] + assertChatCompletionSummary({ tx, model, chatSummary, error: true }) + assert.equal(tx.exceptions.length, 1) + // only asserting message and completion_id as the rest of the attrs + // are asserted in other tests + match(tx.exceptions[0], { + customAttributes: { + 'error.message': 'Premature close', + 'completion_id': /\w{32}/ + } + }) + + tx.end() + end() + } + }) + }) + + test('should not create llm events when ai_monitoring.streaming.enabled is false', (t, end) => { + const { client, agent } = t.nr + agent.config.ai_monitoring.streaming.enabled = false + helper.runInTransaction(agent, async (tx) => { + const content = 'Streamed response' + const model = 'gpt-4' + const stream = await client.chat.completions.create({ + max_tokens: 100, + temperature: 0.5, + model, + messages: [{ role: 'user', content }], + stream: true + }) + + let res = '' + let chunk = {} + + for await (chunk of stream) { + res += chunk.choices[0]?.delta?.content + } + const expectedRes = responses.get(content) + assert.equal(res, expectedRes.streamData) + + const events = agent.customEventAggregator.events.toArray() + assert.equal(events.length, 0, 'should not llm events when streaming is disabled') + const metrics = agent.metrics.getOrCreateMetric(TRACKING_METRIC) + assert.equal(metrics.callCount > 0, true) + const attributes = tx.trace.attributes.get(DESTINATIONS.TRANS_EVENT) + assert.equal(attributes.llm, true) + const streamingDisabled = agent.metrics.getOrCreateMetric( + 'Supportability/Nodejs/ML/Streaming/Disabled' + ) + assert.equal(streamingDisabled.callCount > 0, true) + + tx.end() + end() + }) + }) +} else { + test('should not instrument streams when openai < 4.12.2', (t, end) => { + const { client, agent, host, port } = t.nr + helper.runInTransaction(agent, async (tx) => { + const content = 'Streamed response' + const stream = await client.chat.completions.create({ + stream: true, + messages: [{ role: 'user', content }] + }) + + let chunk = {} + let res = '' + for await (chunk of stream) { + res += chunk.choices[0]?.delta?.content + } + + assert.ok(res) + const events = agent.customEventAggregator.events.toArray() + assert.equal(events.length, 0) + // we will still record the external segment but not the chat completion + assertSegments(tx.trace.root, [ + 'timers.setTimeout', + `External/${host}:${port}/chat/completions` + ]) + + tx.end() + end() + }) + }) +} + +test('should not create llm events when not in a transaction', async (t) => { + const { client, agent } = t.nr + await client.chat.completions.create({ + messages: [{ role: 'user', content: 'You are a mathematician.' }] + }) + + const events = agent.customEventAggregator.events.toArray() + assert.equal(events.length, 0, 'should not create llm events') +}) + +test('auth errors should be tracked', (t, end) => { + const { client, agent } = t.nr + helper.runInTransaction(agent, async (tx) => { + try { + await client.chat.completions.create({ + messages: [{ role: 'user', content: 'Invalid API key.' }] + }) + } catch {} + + assert.equal(tx.exceptions.length, 1) + match(tx.exceptions[0], { + error: { + status: 401, + code: 'invalid_api_key', + param: 'null' + }, + customAttributes: { + 'http.statusCode': 401, + 'error.message': /Incorrect API key provided:/, + 'error.code': 'invalid_api_key', + 'error.param': 'null', + 'completion_id': /[\w\d]{32}/ + }, + agentAttributes: { + spanId: /[\w\d]+/ + } + }) + + const summary = agent.customEventAggregator.events.toArray().find((e) => { + return e[0].type === 'LlmChatCompletionSummary' + }) + assert.ok(summary) + assert.equal(summary[1].error, true) + + tx.end() + end() + }) +}) + +test('invalid payload errors should be tracked', (t, end) => { + const { client, agent } = t.nr + helper.runInTransaction(agent, async (tx) => { + try { + await client.chat.completions.create({ + messages: [{ role: 'bad-role', content: 'Invalid role.' }] + }) + } catch {} + + assert.equal(tx.exceptions.length, 1) + match(tx.exceptions[0], { + error: { + status: 400, + code: null, + param: null + }, + customAttributes: { + 'http.statusCode': 400, + 'error.message': /'bad-role' is not one of/, + 'error.code': null, + 'error.param': null, + 'completion_id': /\w{32}/ + }, + agentAttributes: { + spanId: /\w+/ + } + }) + + tx.end() + end() + }) +}) + +test('should add llm attribute to transaction', (t, end) => { + const { client, agent } = t.nr + helper.runInTransaction(agent, async (tx) => { + await client.chat.completions.create({ + messages: [{ role: 'user', content: 'You are a mathematician.' }] + }) + + const attributes = tx.trace.attributes.get(DESTINATIONS.TRANS_EVENT) + assert.equal(attributes.llm, true) + + tx.end() + end() + }) +}) + +test('should record LLM custom events with attributes', (t, end) => { + const { client, agent } = t.nr + const api = helper.getAgentApi() + + helper.runInTransaction(agent, () => { + api.withLlmCustomAttributes({ 'llm.shared': true, 'llm.path': 'root/' }, async () => { + await api.withLlmCustomAttributes( + { 'llm.path': 'root/branch1', 'llm.attr1': true }, + async () => { + agent.config.ai_monitoring.streaming.enabled = true + const model = 'gpt-3.5-turbo-0613' + const content = 'You are a mathematician.' + await client.chat.completions.create({ + max_tokens: 100, + temperature: 0.5, + model, + messages: [ + { role: 'user', content }, + { role: 'user', content: 'What does 1 plus 1 equal?' } + ] + }) + } + ) + + await api.withLlmCustomAttributes( + { 'llm.path': 'root/branch2', 'llm.attr2': true }, + async () => { + agent.config.ai_monitoring.streaming.enabled = true + const model = 'gpt-3.5-turbo-0613' + const content = 'You are a mathematician.' + await client.chat.completions.create({ + max_tokens: 100, + temperature: 0.5, + model, + messages: [ + { role: 'user', content }, + { role: 'user', content: 'What does 1 plus 2 equal?' } + ] + }) + } + ) + + const events = agent.customEventAggregator.events.toArray().map((event) => event[1]) + + events.forEach((event) => { + assert.ok(event['llm.shared']) + if (event['llm.path'] === 'root/branch1') { + assert.ok(event['llm.attr1']) + assert.equal(event['llm.attr2'], undefined) + } else { + assert.ok(event['llm.attr2']) + assert.equal(event['llm.attr1'], undefined) + } + }) + + end() + }) + }) +}) diff --git a/test/versioned/openai/common.js b/test/versioned/openai/common.js index 935b4e64f5..7e22eaaaac 100644 --- a/test/versioned/openai/common.js +++ b/test/versioned/openai/common.js @@ -4,50 +4,18 @@ */ 'use strict' -const tap = require('tap') -const common = module.exports -const createOpenAIMockServer = require('./mock-server') -const helper = require('../../lib/agent_helper') -const config = { - ai_monitoring: { - enabled: true - }, - streaming: { - enabled: true - } -} -common.beforeHook = async function beforeHook(t) { - const { host, port, server } = await createOpenAIMockServer() - t.context.host = host - t.context.port = port - t.context.server = server - t.context.agent = helper.instrumentMockedAgent(config) - const OpenAI = require('openai') - t.context.client = new OpenAI({ - apiKey: 'fake-versioned-test-key', - baseURL: `http://${host}:${port}` - }) +module.exports = { + assertChatCompletionMessages, + assertChatCompletionSummary } -common.afterEachHook = function afterEachHook(t) { - t.context.agent.customEventAggregator.clear() -} +const { match } = require('../../lib/custom-assertions') -common.afterHook = function afterHook(t) { - t.context?.server?.close() - t.context.agent && helper.unloadAgent(t.context.agent) -} - -function assertChatCompletionMessages({ - tx, - chatMsgs, - id, - model, - reqContent, - resContent, - tokenUsage -}) { +function assertChatCompletionMessages( + { tx, chatMsgs, id, model, reqContent, resContent, tokenUsage }, + { assert = require('node:assert') } = {} +) { const baseMsg = { 'appName': 'New Relic for Node.js tests', 'request_id': '49dbbffbd3c3f4612aa48def69059aad', @@ -88,12 +56,15 @@ function assertChatCompletionMessages({ } } - this.equal(msg[0].type, 'LlmChatCompletionMessage') - this.match(msg[1], expectedChatMsg, 'should match chat completion message') + assert.equal(msg[0].type, 'LlmChatCompletionMessage') + match(msg[1], expectedChatMsg, { assert }) }) } -function assertChatCompletionSummary({ tx, model, chatSummary, error = false }) { +function assertChatCompletionSummary( + { tx, model, chatSummary, error = false }, + { assert = require('node:assert') } = {} +) { const expectedChatSummary = { 'id': /[a-f0-9]{36}/, 'appName': 'New Relic for Node.js tests', @@ -117,9 +88,6 @@ function assertChatCompletionSummary({ tx, model, chatSummary, error = false }) 'error': error } - this.equal(chatSummary[0].type, 'LlmChatCompletionSummary') - this.match(chatSummary[1], expectedChatSummary, 'should match chat summary message') + assert.equal(chatSummary[0].type, 'LlmChatCompletionSummary') + match(chatSummary[1], expectedChatSummary, { assert }) } - -tap.Test.prototype.addAssert('llmMessages', 1, assertChatCompletionMessages) -tap.Test.prototype.addAssert('llmSummary', 1, assertChatCompletionSummary) diff --git a/test/versioned/openai/embeddings.tap.js b/test/versioned/openai/embeddings.tap.js deleted file mode 100644 index e5e7c05eeb..0000000000 --- a/test/versioned/openai/embeddings.tap.js +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright 2023 New Relic Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -/* - * Copyright 2023 New Relic Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -'use strict' - -const tap = require('tap') -const helper = require('../../lib/agent_helper') -require('../../lib/metrics_helper') -const { beforeHook, afterEachHook, afterHook } = require('./common') -const { - AI: { OPENAI } -} = require('../../../lib/metrics/names') - -const fs = require('fs') -// have to read and not require because openai does not export the package.json -const { version: pkgVersion } = JSON.parse( - fs.readFileSync(`${__dirname}/node_modules/openai/package.json`) -) -const { DESTINATIONS } = require('../../../lib/config/attribute-filter') - -tap.test('OpenAI instrumentation - embedding', (t) => { - t.autoend() - - t.before(beforeHook.bind(null, t)) - - t.afterEach(afterEachHook.bind(null, t)) - - t.teardown(afterHook.bind(null, t)) - - t.test('should create span on successful embedding create', (test) => { - const { client, agent, host, port } = t.context - helper.runInTransaction(agent, async (tx) => { - const results = await client.embeddings.create({ - input: 'This is an embedding test.', - model: 'text-embedding-ada-002' - }) - - test.notOk(results.headers, 'should remove response headers from user result') - test.equal(results.model, 'text-embedding-ada-002-v2') - - test.assertSegments( - tx.trace.root, - [OPENAI.EMBEDDING, [`External/${host}:${port}/embeddings`]], - { - exact: false - } - ) - tx.end() - test.end() - }) - }) - - t.test('should increment tracking metric for each embedding event', (test) => { - const { client, agent } = t.context - helper.runInTransaction(agent, async (tx) => { - await client.embeddings.create({ - input: 'This is an embedding test.', - model: 'text-embedding-ada-002' - }) - - const metrics = agent.metrics.getOrCreateMetric( - `Supportability/Nodejs/ML/OpenAI/${pkgVersion}` - ) - test.equal(metrics.callCount > 0, true) - - tx.end() - test.end() - }) - }) - - t.test('should create an embedding message', (test) => { - const { client, agent } = t.context - helper.runInTransaction(agent, async (tx) => { - await client.embeddings.create({ - input: 'This is an embedding test.', - model: 'text-embedding-ada-002' - }) - const events = agent.customEventAggregator.events.toArray() - test.equal(events.length, 1, 'should create a chat completion message and summary event') - const [embedding] = events - const expectedEmbedding = { - 'id': /[a-f0-9]{36}/, - 'appName': 'New Relic for Node.js tests', - 'request_id': 'c70828b2293314366a76a2b1dcb20688', - 'trace_id': tx.traceId, - 'span_id': tx.trace.root.children[0].id, - 'response.model': 'text-embedding-ada-002-v2', - 'vendor': 'openai', - 'ingest_source': 'Node', - 'request.model': 'text-embedding-ada-002', - 'duration': tx.trace.root.children[0].getDurationInMillis(), - 'response.organization': 'new-relic-nkmd8b', - 'token_count': undefined, - 'response.headers.llmVersion': '2020-10-01', - 'response.headers.ratelimitLimitRequests': '200', - 'response.headers.ratelimitLimitTokens': '150000', - 'response.headers.ratelimitResetTokens': '2ms', - 'response.headers.ratelimitRemainingTokens': '149994', - 'response.headers.ratelimitRemainingRequests': '197', - 'input': 'This is an embedding test.', - 'error': false - } - - test.equal(embedding[0].type, 'LlmEmbedding') - test.match(embedding[1], expectedEmbedding, 'should match embedding message') - tx.end() - test.end() - }) - }) - - t.test('embedding invalid payload errors should be tracked', (test) => { - const { client, agent } = t.context - helper.runInTransaction(agent, async (tx) => { - try { - await client.embeddings.create({ - model: 'gpt-4', - input: 'Embedding not allowed.' - }) - } catch {} - - test.equal(tx.exceptions.length, 1) - test.match(tx.exceptions[0], { - error: { - status: 403, - code: null, - param: null - }, - customAttributes: { - 'http.statusCode': 403, - 'error.message': 'You are not allowed to generate embeddings from this model', - 'error.code': null, - 'error.param': null, - 'completion_id': undefined, - 'embedding_id': /\w{32}/ - }, - agentAttributes: { - spanId: /\w+/ - } - }) - - const embedding = agent.customEventAggregator.events.toArray().slice(0, 1)[0][1] - test.equal(embedding.error, true) - - tx.end() - test.end() - }) - }) - - t.test('should add llm attribute to transaction', (test) => { - const { client, agent } = t.context - helper.runInTransaction(agent, async (tx) => { - await client.embeddings.create({ - input: 'This is an embedding test.', - model: 'text-embedding-ada-002' - }) - - const attributes = tx.trace.attributes.get(DESTINATIONS.TRANS_EVENT) - t.equal(attributes.llm, true) - - tx.end() - test.end() - }) - }) -}) diff --git a/test/versioned/openai/embeddings.test.js b/test/versioned/openai/embeddings.test.js new file mode 100644 index 0000000000..48fbe652ef --- /dev/null +++ b/test/versioned/openai/embeddings.test.js @@ -0,0 +1,182 @@ +/* + * Copyright 2024 New Relic Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +'use strict' + +const test = require('node:test') +const assert = require('node:assert') +const fs = require('node:fs') + +const { removeModules } = require('../../lib/cache-buster') +const { assertSegments, match } = require('../../lib/custom-assertions') +const createOpenAIMockServer = require('./mock-server') +const helper = require('../../lib/agent_helper') + +const { + AI: { OPENAI } +} = require('../../../lib/metrics/names') +// have to read and not require because openai does not export the package.json +const { version: pkgVersion } = JSON.parse( + fs.readFileSync(`${__dirname}/node_modules/openai/package.json`) +) +const { DESTINATIONS } = require('../../../lib/config/attribute-filter') + +test.beforeEach(async (ctx) => { + ctx.nr = {} + const { host, port, server } = await createOpenAIMockServer() + ctx.nr.host = host + ctx.nr.port = port + ctx.nr.server = server + ctx.nr.agent = helper.instrumentMockedAgent({ + ai_monitoring: { + enabled: true + }, + streaming: { + enabled: true + } + }) + const OpenAI = require('openai') + ctx.nr.client = new OpenAI({ + apiKey: 'fake-versioned-test-key', + baseURL: `http://${host}:${port}` + }) +}) + +test.afterEach((ctx) => { + helper.unloadAgent(ctx.nr.agent) + ctx.nr.server?.close() + removeModules('openai') +}) + +test('should create span on successful embedding create', (t, end) => { + const { client, agent, host, port } = t.nr + helper.runInTransaction(agent, async (tx) => { + const results = await client.embeddings.create({ + input: 'This is an embedding test.', + model: 'text-embedding-ada-002' + }) + + assert.equal(results.headers, undefined, 'should remove response headers from user result') + assert.equal(results.model, 'text-embedding-ada-002-v2') + + assertSegments(tx.trace.root, [OPENAI.EMBEDDING, [`External/${host}:${port}/embeddings`]], { + exact: false + }) + + tx.end() + end() + }) +}) + +test('should increment tracking metric for each embedding event', (t, end) => { + const { client, agent } = t.nr + helper.runInTransaction(agent, async (tx) => { + await client.embeddings.create({ + input: 'This is an embedding test.', + model: 'text-embedding-ada-002' + }) + + const metrics = agent.metrics.getOrCreateMetric(`Supportability/Nodejs/ML/OpenAI/${pkgVersion}`) + assert.equal(metrics.callCount > 0, true) + + tx.end() + end() + }) +}) + +test('should create an embedding message', (t, end) => { + const { client, agent } = t.nr + helper.runInTransaction(agent, async (tx) => { + await client.embeddings.create({ + input: 'This is an embedding test.', + model: 'text-embedding-ada-002' + }) + const events = agent.customEventAggregator.events.toArray() + assert.equal(events.length, 1, 'should create a chat completion message and summary event') + const [embedding] = events + const expectedEmbedding = { + 'id': /[a-f0-9]{36}/, + 'appName': 'New Relic for Node.js tests', + 'request_id': 'c70828b2293314366a76a2b1dcb20688', + 'trace_id': tx.traceId, + 'span_id': tx.trace.root.children[0].id, + 'response.model': 'text-embedding-ada-002-v2', + 'vendor': 'openai', + 'ingest_source': 'Node', + 'request.model': 'text-embedding-ada-002', + 'duration': tx.trace.root.children[0].getDurationInMillis(), + 'response.organization': 'new-relic-nkmd8b', + 'token_count': undefined, + 'response.headers.llmVersion': '2020-10-01', + 'response.headers.ratelimitLimitRequests': '200', + 'response.headers.ratelimitLimitTokens': '150000', + 'response.headers.ratelimitResetTokens': '2ms', + 'response.headers.ratelimitRemainingTokens': '149994', + 'response.headers.ratelimitRemainingRequests': '197', + 'input': 'This is an embedding test.', + 'error': false + } + + assert.equal(embedding[0].type, 'LlmEmbedding') + match(embedding[1], expectedEmbedding, 'should match embedding message') + + tx.end() + end() + }) +}) + +test('embedding invalid payload errors should be tracked', (t, end) => { + const { client, agent } = t.nr + helper.runInTransaction(agent, async (tx) => { + try { + await client.embeddings.create({ + model: 'gpt-4', + input: 'Embedding not allowed.' + }) + } catch {} + + assert.equal(tx.exceptions.length, 1) + match(tx.exceptions[0], { + error: { + status: 403, + code: null, + param: null + }, + customAttributes: { + 'http.statusCode': 403, + 'error.message': '403 You are not allowed to generate embeddings from this model', + 'error.code': null, + 'error.param': null, + 'completion_id': undefined, + 'embedding_id': /\w{32}/ + }, + agentAttributes: { + spanId: /\w+/ + } + }) + + const embedding = agent.customEventAggregator.events.toArray().slice(0, 1)[0][1] + assert.equal(embedding.error, true) + + tx.end() + end() + }) +}) + +test('should add llm attribute to transaction', (t, end) => { + const { client, agent } = t.nr + helper.runInTransaction(agent, async (tx) => { + await client.embeddings.create({ + input: 'This is an embedding test.', + model: 'text-embedding-ada-002' + }) + + const attributes = tx.trace.attributes.get(DESTINATIONS.TRANS_EVENT) + assert.equal(attributes.llm, true) + + tx.end() + end() + }) +}) diff --git a/test/versioned/openai/feedback-messages.tap.js b/test/versioned/openai/feedback-messages.tap.js deleted file mode 100644 index 9ae94cf1c5..0000000000 --- a/test/versioned/openai/feedback-messages.tap.js +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2023 New Relic Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -/* - * Copyright 2023 New Relic Corporation. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -'use strict' - -const tap = require('tap') -const helper = require('../../lib/agent_helper') -const { beforeHook, afterEachHook, afterHook } = require('./common') - -tap.test('OpenAI instrumentation - feedback messages', (t) => { - t.autoend() - - t.before(beforeHook.bind(null, t)) - - t.afterEach(afterEachHook.bind(null, t)) - - t.teardown(afterHook.bind(null, t)) - - t.test('can send feedback events', (test) => { - const { client, agent } = t.context - const api = helper.getAgentApi() - helper.runInTransaction(agent, async (tx) => { - await client.chat.completions.create({ - messages: [{ role: 'user', content: 'You are a mathematician.' }] - }) - const { traceId } = api.getTraceMetadata() - - api.recordLlmFeedbackEvent({ - traceId, - category: 'test-event', - rating: '5 star', - message: 'You are a mathematician.', - metadata: { foo: 'foo' } - }) - - const recordedEvents = agent.customEventAggregator.getEvents() - test.equal( - true, - recordedEvents.some((ele) => { - const [info, data] = ele - if (info.type !== 'LlmFeedbackMessage') { - return false - } - return test.match(data, { - id: /[\w\d]{32}/, - trace_id: traceId, - category: 'test-event', - rating: '5 star', - message: 'You are a mathematician.', - ingest_source: 'Node', - foo: 'foo' - }) - }) - ) - tx.end() - test.end() - }) - }) -}) diff --git a/test/versioned/openai/feedback-messages.test.js b/test/versioned/openai/feedback-messages.test.js new file mode 100644 index 0000000000..db7561d1c2 --- /dev/null +++ b/test/versioned/openai/feedback-messages.test.js @@ -0,0 +1,86 @@ +/* + * Copyright 2024 New Relic Corporation. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +'use strict' + +const test = require('node:test') +const assert = require('node:assert') + +const { removeModules } = require('../../lib/cache-buster') +const { match } = require('../../lib/custom-assertions') +const createOpenAIMockServer = require('./mock-server') +const helper = require('../../lib/agent_helper') + +test.beforeEach(async (ctx) => { + ctx.nr = {} + const { host, port, server } = await createOpenAIMockServer() + ctx.nr.host = host + ctx.nr.port = port + ctx.nr.server = server + ctx.nr.agent = helper.instrumentMockedAgent({ + ai_monitoring: { + enabled: true + }, + streaming: { + enabled: true + } + }) + const OpenAI = require('openai') + ctx.nr.client = new OpenAI({ + apiKey: 'fake-versioned-test-key', + baseURL: `http://${host}:${port}` + }) +}) + +test.afterEach((ctx) => { + helper.unloadAgent(ctx.nr.agent) + ctx.nr.server?.close() + removeModules('openai') +}) + +test('can send feedback events', (t, end) => { + const { client, agent } = t.nr + const api = helper.getAgentApi() + helper.runInTransaction(agent, async (tx) => { + await client.chat.completions.create({ + messages: [{ role: 'user', content: 'You are a mathematician.' }] + }) + const { traceId } = api.getTraceMetadata() + + api.recordLlmFeedbackEvent({ + traceId, + category: 'test-event', + rating: '5 star', + message: 'You are a mathematician.', + metadata: { foo: 'foo' } + }) + + const recordedEvents = agent.customEventAggregator.getEvents() + const hasMatchingEvents = recordedEvents.some((ele) => { + const [info, data] = ele + if (info.type !== 'LlmFeedbackMessage') { + return false + } + try { + match(data, { + id: /[\w\d]{32}/, + trace_id: traceId, + category: 'test-event', + rating: '5 star', + message: 'You are a mathematician.', + ingest_source: 'Node', + foo: 'foo' + }) + } catch { + return false + } + return true + }) + assert.equal(hasMatchingEvents, true) + + tx.end() + end() + }) +}) diff --git a/test/versioned/openai/package.json b/test/versioned/openai/package.json index 754fc3a6fb..d11d5257fa 100644 --- a/test/versioned/openai/package.json +++ b/test/versioned/openai/package.json @@ -15,9 +15,9 @@ "openai": ">=4.0.0" }, "files": [ - "chat-completions.tap.js", - "embeddings.tap.js", - "feedback-messages.tap.js" + "chat-completions.test.js", + "embeddings.test.js", + "feedback-messages.test.js" ] } ]