Skip to content

Commit

Permalink
Merge pull request #71 from superglue-ai/glu-217-understand-number-of…
Browse files Browse the repository at this point in the history
…-different-target-systems

Glu 217 understand number of different target systems
  • Loading branch information
stefanfaistenauer authored Mar 6, 2025
2 parents 24df9c0 + b1e675f commit 4774620
Show file tree
Hide file tree
Showing 5 changed files with 251 additions and 28 deletions.
2 changes: 2 additions & 0 deletions .cursor/rules/next-js.mdc
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,5 @@ if (retries > 3) {...}
const MAX_RETRIES = 3
if (retries > MAX_RETRIES) {...}
```

always be mindful of deleting tests and explicitly let the user know when you do so (ask for consent!).
6 changes: 0 additions & 6 deletions packages/core/graphql/resolvers/call.ts
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,6 @@ export const callResolver = async (
completedAt: new Date(),
};
context.datastore.createRun(result, context.orgId);

return {...result, data: transformedResponse.data};
} catch (error) {
const maskedError = maskCredentials(error.message, credentials);
Expand All @@ -129,11 +128,6 @@ export const callResolver = async (
startedAt,
completedAt: new Date(),
};
telemetryClient?.captureException(maskedError, context.orgId, {
preparedEndpoint: preparedEndpoint,
messages: messages,
result: result
});
context.datastore.createRun(result, context.orgId);
return result;
}
Expand Down
21 changes: 2 additions & 19 deletions packages/core/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { LocalKeyManager } from './auth/localKeyManager.js';
import { SupabaseKeyManager } from './auth/supabaseKeyManager.js';
import { createDataStore } from './datastore/datastore.js';
import { resolvers, typeDefs } from './graphql/graphql.js';
import { handleQueryError, telemetryClient, telemetryMiddleware } from './utils/telemetry.js';
import { createTelemetryPlugin, telemetryMiddleware } from './utils/telemetry.js';

// Constants
const PORT = process.env.GRAPHQL_PORT || 3000;
Expand Down Expand Up @@ -42,24 +42,7 @@ const apolloConfig = {
embed: true,
document: DEFAULT_QUERY
}),
// Telemetry Plugin
{
requestDidStart: async () => ({
willSendResponse: async (requestContext) => {
const errors = requestContext.errors ||
requestContext?.response?.body?.singleResult?.errors ||
Object.values(requestContext?.response?.body?.singleResult?.data || {}).map((d: any) => d.error).filter(Boolean);

if(errors && errors.length > 0) {
console.error(errors);
}
if (errors && errors.length > 0 && telemetryClient) {
const orgId = requestContext.contextValue.orgId;
handleQueryError(errors, requestContext.request.query, orgId);
}
}
})
}
createTelemetryPlugin()
],
};

Expand Down
168 changes: 168 additions & 0 deletions packages/core/utils/telemetry.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
import { PostHog } from 'posthog-node';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { resolvers } from '../graphql/graphql.js';
import * as telemetryModule from './telemetry.js';

// Mock PostHog
vi.mock('posthog-node', async () => {
return {
PostHog: vi.fn().mockImplementation(() => ({
capture: vi.fn(),
captureException: vi.fn()
}))
};
});

describe('Telemetry Utils', () => {
beforeEach(() => {
vi.clearAllMocks();
});

afterEach(() => {
vi.resetAllMocks();
});

describe('telemetry environment variables', () => {
it('disables telemetry when DISABLE_TELEMETRY is set to true', async () => {
// Mock the environment variables
const originalEnv = process.env.DISABLE_TELEMETRY;
vi.stubEnv('DISABLE_TELEMETRY', 'true');

// Mock the initialization of telemetryClient
const mockPostHog = vi.mocked(PostHog);
mockPostHog.mockClear();

// Create a new instance of telemetry utilities to test the disabled state
// We can do this by re-executing the logic that initializes telemetryClient
const isTelemetryDisabled = process.env.DISABLE_TELEMETRY === "true";
const isDebug = process.env.DEBUG === "true";
const telemetryClient = !isTelemetryDisabled && !isDebug ?
new PostHog('test-key', { host: 'test-host', enableExceptionAutocapture: true }) : null;

// Verify telemetry is disabled
expect(isTelemetryDisabled).toBe(true);
expect(telemetryClient).toBeNull();

// Test the middleware with telemetry disabled
const mockReq = {
body: {
query: `
mutation {
call(input: { id: "123" }, payload: {}) {
id
success
}
}
`
},
orgId: 'test-org'
};
const mockRes = {};
const mockNext = vi.fn();

// Use the real middleware function with our test request
telemetryModule.telemetryMiddleware(mockReq, mockRes, mockNext);

// Expect next to be called without error
expect(mockNext).toHaveBeenCalled();
expect(mockPostHog).not.toHaveBeenCalled();

// Mock console.error to prevent actual error logging during tests
const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});

try {
// Test the telemetry plugin with telemetry disabled
const plugin = telemetryModule.createTelemetryPlugin();
const requestHandler = await plugin.requestDidStart();

// Create a more complete mock of requestContext based on the checkIfSelfHosted requirements
const mockRequestContext = {
contextValue: {
orgId: 'test-org',
datastore: {
constructor: { name: 'MockDataStore' },
storage: {
tenant: {
email: 'test@example.com',
emailEntrySkipped: false
}
}
}
},
request: {
query: 'query { test }'
},
response: {
body: {}
},
errors: []
};

// Execute the willSendResponse handler - this should not throw even with telemetry disabled
await requestHandler.willSendResponse(mockRequestContext);

// Verify telemetry was not captured since it's disabled
expect(PostHog).not.toHaveBeenCalled();
} finally {
// Restore console.error
consoleErrorSpy.mockRestore();

// Restore original env
vi.stubEnv('DISABLE_TELEMETRY', originalEnv || '');
}
});
});

describe('extractOperationName', () => {
it('extracts operation name from call mutation with variables', () => {
const query = `
mutation CallApi($input: ApiInputRequest!, $payload: JSON, $credentials: JSON, $options: RequestOptions) {
call(input: $input, payload: $payload, credentials: $credentials, options: $options) {
id
success
}
}
`;
expect(telemetryModule.extractOperationName(query)).toBe('call');
});
});

describe('telemetry middleware', () => {
it('tracks call operation properly', () => {
const mockReq = {
body: {
query: `
mutation {
call(input: { id: "123" }, payload: {}) {
id
success
}
}
`
},
orgId: 'test-org'
};
const mockRes = {};
const mockNext = vi.fn();

telemetryModule.telemetryMiddleware(mockReq, mockRes, mockNext);

expect(mockNext).toHaveBeenCalled();
});
});

describe('telemetry plugin', () => {
it('creates plugin with handler', () => {
const plugin = telemetryModule.createTelemetryPlugin();
expect(plugin).toHaveProperty('requestDidStart');
});
});

it('verifies call operation exists in schema TRACKING BROKEN IF FAILS', () => {
// Get mutation operations defined in resolvers
const mutationOperations = Object.keys(resolvers.Mutation);

// Verify the call operation exists
expect(mutationOperations).toContain('call');
});
});
82 changes: 79 additions & 3 deletions packages/core/utils/telemetry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { config } from '../default.js';
export const sessionId = crypto.randomUUID();

export const isDebug = process.env.DEBUG === "true";
export const isSelfHosted = process.env.RUNS_ON_SUPERGLUE_CLOUD !== "true";
export const isTelemetryDisabled = process.env.DISABLE_TELEMETRY === "true";

export const telemetryClient = !isTelemetryDisabled && !isDebug ?
Expand Down Expand Up @@ -55,22 +56,97 @@ export const telemetryMiddleware = (req, res, next) => {
next();
};

export const handleQueryError = (errors: any[], query: string, orgId: string) => {

const createCallProperties = (query: string, responseBody: any, isSelfHosted: boolean, operation: string) => {
const properties: Record<string, any> = {};
properties.isSelfHosted = isSelfHosted;
properties.operation = operation;
properties.query = query;

switch(operation) {
case 'call':
const call = responseBody?.singleResult?.data?.call;
if(!call) break;
properties.endpointHost = call?.config?.urlHost;
properties.endpointPath = call?.config?.urlPath;
properties.apiConfigId = call?.config?.id;
properties.callMethod = call?.config?.method;
properties.documentationUrl = call?.config?.documentationUrl;
properties.authType = call?.config?.authentication;
properties.responseTimeMs = call?.completedAt?.getTime() - call?.startedAt?.getTime()
break;
default:
break;
}

return properties;
}

export const handleQueryError = (errors: any[], query: string, orgId: string, requestContext: any) => {
// in case of an error, we track the query and the error
// we do not track the variables or the response
// all errors are masked
const operation = extractOperationName(query);
const properties = createCallProperties(query, requestContext.response?.body, isSelfHosted, operation);
properties.success = false;
telemetryClient?.capture({
distinctId: orgId || sessionId,
event: operation + '_error',
properties: {
query,
...properties,
orgId: orgId,
errors: errors.map(e => ({
message: e.message,
path: e.path
}))
})),
success: false
},
groups: {
orgId: orgId
}
});
};

const handleQuerySuccess = (query: string, orgId: string, requestContext: any) => {
const distinctId = isSelfHosted ? `sh-inst-${requestContext.contextValue.datastore.storage?.tenant?.email}` : orgId;
const operation = extractOperationName(query);
const properties = createCallProperties(query, requestContext.response?.body, isSelfHosted, operation);
properties.success = true;

telemetryClient?.capture({
distinctId: distinctId,
event: operation,
properties: properties,
groups: {
orgId: orgId
}
});
};

export const createTelemetryPlugin = () => {
return {
requestDidStart: async () => ({
willSendResponse: async (requestContext: any) => {
const errors = requestContext.errors ||
requestContext?.response?.body?.singleResult?.errors ||
Object.values(requestContext?.response?.body?.singleResult?.data || {}).map((d: any) => d.error).filter(Boolean);

if (telemetryClient) {
if(errors && errors.length > 0) {
console.error(errors);
const orgId = requestContext.contextValue.orgId;
handleQueryError(errors, requestContext.request.query, orgId, requestContext);
} else {
const orgId = requestContext.contextValue.orgId;
handleQuerySuccess(requestContext.request.query, orgId, requestContext);
}
} else {
// disabled telemetry
if(errors && errors.length > 0) {
console.error(errors);
}
}
}
})
};
};

0 comments on commit 4774620

Please sign in to comment.