From 2334daa5a653ece84d32fbafee143dcec48a9f90 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 11 Nov 2025 10:01:46 +0100 Subject: [PATCH 1/4] Add failing truncation tests --- .../langchain/scenario-message-truncation.mjs | 72 +++++++++++++++++++ .../suites/tracing/langchain/test.ts | 42 +++++++++++ 2 files changed, 114 insertions(+) create mode 100644 dev-packages/node-integration-tests/suites/tracing/langchain/scenario-message-truncation.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-message-truncation.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-message-truncation.mjs new file mode 100644 index 000000000000..6dafe8572cec --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-message-truncation.mjs @@ -0,0 +1,72 @@ +import { ChatAnthropic } from '@langchain/anthropic'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +function startMockAnthropicServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/messages', (req, res) => { + const model = req.body.model; + + res.json({ + id: 'msg_truncation_test', + type: 'message', + role: 'assistant', + content: [ + { + type: 'text', + text: 'Response to truncated messages', + }, + ], + model: model, + stop_reason: 'end_turn', + stop_sequence: null, + usage: { + input_tokens: 10, + output_tokens: 15, + }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockAnthropicServer(); + const baseUrl = `http://localhost:${server.address().port}`; + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const model = new ChatAnthropic({ + model: 'claude-3-5-sonnet-20241022', + apiKey: 'mock-api-key', + clientOptions: { + baseURL: baseUrl, + }, + }); + + const largeContent1 = 'A'.repeat(15000); // ~15KB + const largeContent2 = 'B'.repeat(15000); // ~15KB + const largeContent3 = 'C'.repeat(25000); // ~25KB (will be truncated) + + // Create one very large string that gets truncated to only include Cs + await model.invoke(largeContent3 + largeContent2); + + // Create an array of messages that gets truncated to only include the last message (result should again contain only Cs) + await model.invoke([ + { role: 'system', content: largeContent1 }, + { role: 'user', content: largeContent2 }, + { role: 'user', content: largeContent3 }, + ]); + }); + + await Sentry.flush(2000); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts index 4de2f96b5dc5..dff5253ea423 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts @@ -194,4 +194,46 @@ describe('LangChain integration', () => { await createRunner().ignore('event').expect({ transaction: EXPECTED_TRANSACTION_TOOL_CALLS }).start().completed(); }); }); + + const EXPECTED_TRANSACTION_MESSAGE_TRUNCATION = { + transaction: 'main', + spans: expect.arrayContaining([ + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-5-sonnet-20241022', + // Messages should be present and should include truncated string input (contains only Cs) + 'gen_ai.request.messages': expect.stringMatching(/^\[{"role":"user","content":"C+"}\]$/), + }), + description: 'chat claude-3-5-sonnet-20241022', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'ok', + }), + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-5-sonnet-20241022', + // Messages should be present (truncation happened) and should be a JSON array of a single index (contains only Cs) + 'gen_ai.request.messages': expect.stringMatching(/^\[{"role":"user","content":"C+"}\]$/), + }), + description: 'chat claude-3-5-sonnet-20241022', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'ok', + }), + ]), + }; + + createEsmAndCjsTests(__dirname, 'scenario-message-truncation.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { + test('truncates messages when they exceed byte limit', async () => { + await createRunner().ignore('event').expect({ transaction: EXPECTED_TRANSACTION_MESSAGE_TRUNCATION }).start().completed(); + }); + }); }); From 7d92384ecbfd7ccbd386f33a1d659d73a5302bad Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 11 Nov 2025 10:15:51 +0100 Subject: [PATCH 2/4] Code formatting --- .../suites/tracing/langchain/test.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts index dff5253ea423..48743c6cc29e 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts @@ -231,9 +231,18 @@ describe('LangChain integration', () => { ]), }; - createEsmAndCjsTests(__dirname, 'scenario-message-truncation.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { - test('truncates messages when they exceed byte limit', async () => { - await createRunner().ignore('event').expect({ transaction: EXPECTED_TRANSACTION_MESSAGE_TRUNCATION }).start().completed(); - }); - }); + createEsmAndCjsTests( + __dirname, + 'scenario-message-truncation.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('truncates messages when they exceed byte limit', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: EXPECTED_TRANSACTION_MESSAGE_TRUNCATION }) + .start() + .completed(); + }); + }, + ); }); From 02d2b96f856b278f1f9514c92c3b070181513d04 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 11 Nov 2025 11:01:18 +0100 Subject: [PATCH 3/4] Add truncation --- packages/core/src/utils/langchain/utils.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/core/src/utils/langchain/utils.ts b/packages/core/src/utils/langchain/utils.ts index 8464e71aecb0..caacf5059bdc 100644 --- a/packages/core/src/utils/langchain/utils.ts +++ b/packages/core/src/utils/langchain/utils.ts @@ -23,6 +23,7 @@ import { GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, } from '../ai/gen-ai-attributes'; +import { truncateGenAiMessages } from '../ai/messageTruncation'; import { LANGCHAIN_ORIGIN, ROLE_MAP } from './constants'; import type { LangChainLLMResult, LangChainMessage, LangChainSerialized } from './types'; @@ -281,7 +282,8 @@ export function extractChatModelRequestAttributes( if (recordInputs && Array.isArray(langChainMessages) && langChainMessages.length > 0) { const normalized = normalizeLangChainMessages(langChainMessages.flat()); - setIfDefined(attrs, GEN_AI_REQUEST_MESSAGES_ATTRIBUTE, asString(normalized)); + const truncated = truncateGenAiMessages(normalized); + setIfDefined(attrs, GEN_AI_REQUEST_MESSAGES_ATTRIBUTE, asString(truncated)); } return attrs; From a42c5482065a84ead5e6ef532695b8d79f82b114 Mon Sep 17 00:00:00 2001 From: Nicolas Hrubec Date: Tue, 11 Nov 2025 11:28:38 +0100 Subject: [PATCH 4/4] Fix linting errors --- .../node-integration-tests/suites/tracing/langchain/test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts index 48743c6cc29e..ff0e95b8f8ad 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts @@ -206,7 +206,7 @@ describe('LangChain integration', () => { 'gen_ai.system': 'anthropic', 'gen_ai.request.model': 'claude-3-5-sonnet-20241022', // Messages should be present and should include truncated string input (contains only Cs) - 'gen_ai.request.messages': expect.stringMatching(/^\[{"role":"user","content":"C+"}\]$/), + 'gen_ai.request.messages': expect.stringMatching(/^\[\{"role":"user","content":"C+"\}\]$/), }), description: 'chat claude-3-5-sonnet-20241022', op: 'gen_ai.chat', @@ -221,7 +221,7 @@ describe('LangChain integration', () => { 'gen_ai.system': 'anthropic', 'gen_ai.request.model': 'claude-3-5-sonnet-20241022', // Messages should be present (truncation happened) and should be a JSON array of a single index (contains only Cs) - 'gen_ai.request.messages': expect.stringMatching(/^\[{"role":"user","content":"C+"}\]$/), + 'gen_ai.request.messages': expect.stringMatching(/^\[\{"role":"user","content":"C+"\}\]$/), }), description: 'chat claude-3-5-sonnet-20241022', op: 'gen_ai.chat',