Skip to content

Commit d455a20

Browse files
TaoChenOSUTaoChenOSU
authored
.Net & Python: Add tool definitions to agent invoke span (#13153)
### Motivation and Context <!-- Thank you for your contribution to the semantic-kernel repo! Please help reviewers and future users, providing the following information: 1. Why is this change required? 2. What problem does it solve? 3. What scenario does it contribute to? 4. If it fixes an open issue, please link to the issue here. --> The current GenAI Semantic Conventions have changed since the last time we modified the `agent_invoke` span. Previous PRs: #12940, #12928, #12834 ### Description The following changes were made: 1. `gen_ai.agent.invocation_input` -> `gen_ai.input.messages` 2. `gen_ai.agent.invocation_output` -> `gen_ai.output.messages` 3. Add `gen_ai.tool.definitions` .Net: <img width="2056" height="459" alt="image" src="https://github.com/user-attachments/assets/533b8e08-7674-4c70-bdbe-ba3600855b20" /> Python: <img width="1323" height="659" alt="image" src="https://github.com/user-attachments/assets/702171ac-dc0b-4187-933d-dad77f2719df" /> <!-- Describe your changes, the overall approach, the underlying design. These notes will help understanding how your code works. Thanks! --> ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [ ] I didn't break anyone 😄
1 parent 28ea2f4 commit d455a20

File tree

9 files changed

+83
-18
lines changed

9 files changed

+83
-18
lines changed

‎dotnet/src/Agents/AzureAI/AzureAIAgent.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ public async IAsyncEnumerable<AgentResponseItem<ChatMessageContent>> InvokeAsync
161161
new AzureAIAgentInvokeOptions() { AdditionalInstructions = mergedAdditionalInstructions } :
162162
new AzureAIAgentInvokeOptions(options) { AdditionalInstructions = mergedAdditionalInstructions };
163163

164-
using var activity = ModelDiagnostics.StartAgentInvocationActivity(this.Id, this.GetDisplayName(), this.Description, messages);
164+
using var activity = ModelDiagnostics.StartAgentInvocationActivity(this.Id, this.GetDisplayName(), this.Description, kernel, messages);
165165
List<ChatMessageContent>? chatMessageContents = activity is not null ? [] : null;
166166

167167
await foreach (var result in InternalInvokeAsync().ConfigureAwait(false))
@@ -264,7 +264,7 @@ public async IAsyncEnumerable<AgentResponseItem<StreamingChatMessageContent>> In
264264
new AzureAIAgentInvokeOptions() { AdditionalInstructions = mergedAdditionalInstructions } :
265265
new AzureAIAgentInvokeOptions(options) { AdditionalInstructions = mergedAdditionalInstructions };
266266

267-
using var activity = ModelDiagnostics.StartAgentInvocationActivity(this.Id, this.GetDisplayName(), this.Description, messages);
267+
using var activity = ModelDiagnostics.StartAgentInvocationActivity(this.Id, this.GetDisplayName(), this.Description, kernel, messages);
268268
List<StreamingChatMessageContent>? streamedContents = activity is not null ? [] : null;
269269

270270
// Invoke the Agent with the thread that we already added our message to, and with

‎dotnet/src/Agents/AzureAI/AzureAIChannel.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ protected override async Task ReceiveAsync(IEnumerable<ChatMessageContent> histo
4545
CancellationToken cancellationToken)
4646
{
4747
return ActivityExtensions.RunWithActivityAsync(
48-
() => ModelDiagnostics.StartAgentInvocationActivity(agent.Id, agent.GetDisplayName(), agent.Description, []),
48+
() => ModelDiagnostics.StartAgentInvocationActivity(agent.Id, agent.GetDisplayName(), agent.Description, agent.Kernel, []),
4949
() => AgentThreadActions.InvokeAsync(agent, client, threadId, invocationOptions: null, this.Logger, agent.Kernel, agent.Arguments, cancellationToken),
5050
cancellationToken);
5151
}
@@ -54,7 +54,7 @@ protected override async Task ReceiveAsync(IEnumerable<ChatMessageContent> histo
5454
protected override IAsyncEnumerable<StreamingChatMessageContent> InvokeStreamingAsync(AzureAIAgent agent, IList<ChatMessageContent> messages, CancellationToken cancellationToken = default)
5555
{
5656
return ActivityExtensions.RunWithActivityAsync(
57-
() => ModelDiagnostics.StartAgentInvocationActivity(agent.Id, agent.GetDisplayName(), agent.Description, messages),
57+
() => ModelDiagnostics.StartAgentInvocationActivity(agent.Id, agent.GetDisplayName(), agent.Description, agent.Kernel, messages),
5858
() => AgentThreadActions.InvokeStreamingAsync(agent, client, threadId, messages, invocationOptions: null, this.Logger, agent.Kernel, agent.Arguments, cancellationToken),
5959
cancellationToken);
6060
}

‎dotnet/src/Agents/Bedrock/BedrockAgent.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ public override async IAsyncEnumerable<AgentResponseItem<ChatMessageContent>> In
144144
};
145145
});
146146

147-
using var activity = ModelDiagnostics.StartAgentInvocationActivity(this.Id, this.GetDisplayName(), this.Description, messages);
147+
using var activity = ModelDiagnostics.StartAgentInvocationActivity(this.Id, this.GetDisplayName(), this.Description, this.Kernel, messages);
148148

149149
// Invoke the agent
150150
var invokeResults = this.InvokeInternalAsync(invokeAgentRequest, options?.KernelArguments, cancellationToken);
@@ -211,7 +211,7 @@ public async IAsyncEnumerable<AgentResponseItem<ChatMessageContent>> InvokeAsync
211211
invokeAgentRequest.SessionId = bedrockThread.Id;
212212
invokeAgentRequest = this.ConfigureAgentRequest(options, () => invokeAgentRequest);
213213

214-
using var activity = ModelDiagnostics.StartAgentInvocationActivity(this.Id, this.GetDisplayName(), this.Description, []);
214+
using var activity = ModelDiagnostics.StartAgentInvocationActivity(this.Id, this.GetDisplayName(), this.Description, this.Kernel, []);
215215
List<ChatMessageContent>? chatMessageContents = activity is not null ? [] : null;
216216

217217
// Invoke the agent
@@ -297,7 +297,7 @@ public override async IAsyncEnumerable<AgentResponseItem<StreamingChatMessageCon
297297
};
298298
});
299299

300-
using var activity = ModelDiagnostics.StartAgentInvocationActivity(this.Id, this.GetDisplayName(), this.Description, []);
300+
using var activity = ModelDiagnostics.StartAgentInvocationActivity(this.Id, this.GetDisplayName(), this.Description, this.Kernel, []);
301301
List<StreamingChatMessageContent>? streamedContents = activity is not null ? [] : null;
302302

303303
// Invoke the agent
@@ -365,7 +365,7 @@ public async IAsyncEnumerable<AgentResponseItem<StreamingChatMessageContent>> In
365365
invokeAgentRequest.SessionId = bedrockThread.Id;
366366
invokeAgentRequest = this.ConfigureAgentRequest(options, () => invokeAgentRequest);
367367

368-
using var activity = ModelDiagnostics.StartAgentInvocationActivity(this.Id, this.GetDisplayName(), this.Description, []);
368+
using var activity = ModelDiagnostics.StartAgentInvocationActivity(this.Id, this.GetDisplayName(), this.Description, this.Kernel, []);
369369
List<StreamingChatMessageContent>? streamedContents = activity is not null ? [] : null;
370370

371371
var invokeResults = this.InvokeStreamingInternalAsync(invokeAgentRequest, bedrockThread, options?.KernelArguments, cancellationToken);

‎dotnet/src/Agents/Core/ChatCompletionAgent.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -349,7 +349,7 @@ private async IAsyncEnumerable<ChatMessageContent> InternalInvokeAsync(
349349

350350
this.Logger.LogAgentChatServiceInvokingAgent(nameof(InvokeAsync), this.Id, agentName, serviceType);
351351

352-
using var activity = ModelDiagnostics.StartAgentInvocationActivity(this.Id, agentName, this.Description, chat);
352+
using var activity = ModelDiagnostics.StartAgentInvocationActivity(this.Id, agentName, this.Description, kernel, chat);
353353

354354
IReadOnlyList<ChatMessageContent> messages =
355355
await chatCompletionService.GetChatMessageContentsAsync(
@@ -402,7 +402,7 @@ private async IAsyncEnumerable<StreamingChatMessageContent> InternalInvokeStream
402402

403403
this.Logger.LogAgentChatServiceInvokingAgent(nameof(InvokeAsync), this.Id, agentName, serviceType);
404404

405-
using var activity = ModelDiagnostics.StartAgentInvocationActivity(this.Id, agentName, this.Description, chat);
405+
using var activity = ModelDiagnostics.StartAgentInvocationActivity(this.Id, agentName, this.Description, kernel, chat);
406406

407407
IAsyncEnumerable<StreamingChatMessageContent> messages =
408408
chatCompletionService.GetStreamingChatMessageContentsAsync(

‎dotnet/src/Agents/OpenAI/OpenAIAssistantAgent.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ public async IAsyncEnumerable<AgentResponseItem<ChatMessageContent>> InvokeAsync
158158
kernel.Plugins.AddFromAIContext(providersContext, "Tools");
159159
#pragma warning restore SKEXP0110, SKEXP0130 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
160160

161-
using var activity = ModelDiagnostics.StartAgentInvocationActivity(this.Id, this.GetDisplayName(), this.Description, messages);
161+
using var activity = ModelDiagnostics.StartAgentInvocationActivity(this.Id, this.GetDisplayName(), this.Description, kernel, messages);
162162
List<ChatMessageContent>? chatMessageContents = activity is not null ? [] : null;
163163

164164
// Notify the thread of new messages and return them to the caller.
@@ -267,7 +267,7 @@ public async IAsyncEnumerable<AgentResponseItem<StreamingChatMessageContent>> In
267267
});
268268

269269
#pragma warning disable SKEXP0001 // ModelDiagnostics is marked experimental.
270-
using var activity = ModelDiagnostics.StartAgentInvocationActivity(this.Id, this.GetDisplayName(), this.Description, messages);
270+
using var activity = ModelDiagnostics.StartAgentInvocationActivity(this.Id, this.GetDisplayName(), this.Description, kernel, messages);
271271
List<StreamingChatMessageContent>? streamedContents = activity is not null ? [] : null;
272272

273273
ChatHistory newMessagesReceiver = [];

‎dotnet/src/Agents/OpenAI/OpenAIAssistantChannel.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ protected override async Task ReceiveAsync(IEnumerable<ChatMessageContent> histo
5050
CancellationToken cancellationToken)
5151
{
5252
return ActivityExtensions.RunWithActivityAsync(
53-
() => ModelDiagnostics.StartAgentInvocationActivity(agent.Id, agent.GetDisplayName(), agent.Description, []),
53+
() => ModelDiagnostics.StartAgentInvocationActivity(agent.Id, agent.GetDisplayName(), agent.Description, agent.Kernel, []),
5454
() => AssistantThreadActions.InvokeAsync(agent, this._client, this._threadId, invocationOptions: null, providersAdditionalInstructions: null, this.Logger, agent.Kernel, agent.Arguments, cancellationToken),
5555
cancellationToken);
5656
}
@@ -59,7 +59,7 @@ protected override async Task ReceiveAsync(IEnumerable<ChatMessageContent> histo
5959
protected override IAsyncEnumerable<StreamingChatMessageContent> InvokeStreamingAsync(OpenAIAssistantAgent agent, IList<ChatMessageContent> messages, CancellationToken cancellationToken = default)
6060
{
6161
return ActivityExtensions.RunWithActivityAsync(
62-
() => ModelDiagnostics.StartAgentInvocationActivity(agent.Id, agent.GetDisplayName(), agent.Description, messages),
62+
() => ModelDiagnostics.StartAgentInvocationActivity(agent.Id, agent.GetDisplayName(), agent.Description, agent.Kernel, messages),
6363
() => AssistantThreadActions.InvokeStreamingAsync(agent, this._client, this._threadId, messages, invocationOptions: null, providersAdditionalInstructions: null, this.Logger, agent.Kernel, agent.Arguments, cancellationToken),
6464
cancellationToken);
6565
}

‎dotnet/src/InternalUtilities/src/Diagnostics/ModelDiagnostics.cs‎

Lines changed: 53 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ internal static class ModelDiagnostics
146146
string agentId,
147147
string agentName,
148148
string? agentDescription,
149+
Kernel? kernel,
149150
ICollection<ChatMessageContent> messages)
150151
{
151152
if (!IsModelDiagnosticsEnabled())
@@ -169,6 +170,12 @@ internal static class ModelDiagnostics
169170
activity?.SetTag(ModelDiagnosticsTags.AgentDescription, agentDescription);
170171
}
171172

173+
if (kernel is not null && kernel.Plugins.Count > 0)
174+
{
175+
var toolDefinitions = kernel.Plugins.GetFunctionsMetadata().Select(m => ToGenAIconventionsFormat(m));
176+
activity?.SetTag(ModelDiagnosticsTags.AgentToolDefinitions, JsonSerializer.Serialize(toolDefinitions));
177+
}
178+
172179
if (IsSensitiveEventsEnabled())
173180
{
174181
activity?.SetTag(
@@ -391,6 +398,49 @@ private static void ToGenAIConventionsFormat(ChatMessageContentItemCollection ch
391398
sb.Append(']');
392399
}
393400

401+
private static string ToGenAIconventionsFormat(KernelFunctionMetadata metadata)
402+
{
403+
var sb = new StringBuilder();
404+
405+
sb.Append("{\"type\": \"function\", \"name\": \"");
406+
sb.Append(metadata.Name);
407+
sb.Append("\", \"description\": \"");
408+
sb.Append(metadata.Description);
409+
sb.Append("\", \"parameters\": ");
410+
ToGenAIconventionsFormat(metadata.Parameters, sb);
411+
sb.Append('}');
412+
413+
return sb.ToString();
414+
}
415+
416+
private static void ToGenAIconventionsFormat(IEnumerable<KernelParameterMetadata> parameters, StringBuilder? sb = null)
417+
{
418+
var properties = new Dictionary<string, KernelJsonSchema>();
419+
var required = new List<string>();
420+
421+
foreach (var param in parameters)
422+
{
423+
if (param.Schema is not null)
424+
{
425+
properties[param.Name] = param.Schema;
426+
}
427+
if (param.IsRequired)
428+
{
429+
required.Add(param.Name);
430+
}
431+
}
432+
433+
var parametersJson = JsonSerializer.Serialize(new
434+
{
435+
type = "object",
436+
properties,
437+
required,
438+
});
439+
440+
sb ??= new StringBuilder();
441+
sb.Append(parametersJson);
442+
}
443+
394444
/// <summary>
395445
/// Convert a chat model response to a string aligned with the OTel GenAI Semantic Conventions format
396446
/// </summary>
@@ -614,8 +664,9 @@ private static class ModelDiagnosticsTags
614664
public const string AgentId = "gen_ai.agent.id";
615665
public const string AgentName = "gen_ai.agent.name";
616666
public const string AgentDescription = "gen_ai.agent.description";
617-
public const string AgentInvocationInput = "gen_ai.agent.invocation_input";
618-
public const string AgentInvocationOutput = "gen_ai.agent.invocation_output";
667+
public const string AgentInvocationInput = "gen_ai.input.messages";
668+
public const string AgentInvocationOutput = "gen_ai.output.messages";
669+
public const string AgentToolDefinitions = "gen_ai.tool.definitions";
619670

620671
// Activity events
621672
public const string EventName = "gen_ai.event.content";

‎python/semantic_kernel/utils/telemetry/agent_diagnostics/decorators.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,19 @@ def _start_as_current_span(agent: Agent):
189189
if agent.description:
190190
attributes[gen_ai_attributes.AGENT_DESCRIPTION] = agent.description
191191

192+
if agent.kernel.plugins:
193+
# This will only capture the tools that are available in the kernel at the time of agent creation.
194+
# If the agent is invoked with another kernel instance, the tools in that kernel will not be captured.
195+
from semantic_kernel.connectors.ai.function_calling_utils import (
196+
kernel_function_metadata_to_function_call_format,
197+
)
198+
199+
tool_definitions = [
200+
kernel_function_metadata_to_function_call_format(metadata)
201+
for metadata in agent.kernel.get_full_list_of_function_metadata()
202+
]
203+
attributes[gen_ai_attributes.AGENT_TOOL_DEFINITIONS] = json.dumps(tool_definitions)
204+
192205
return tracer.start_as_current_span(f"{OPERATION_NAME} {agent.name}", attributes=attributes)
193206

194207

‎python/semantic_kernel/utils/telemetry/agent_diagnostics/gen_ai_attributes.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
AGENT_ID = "gen_ai.agent.id"
1111
AGENT_NAME = "gen_ai.agent.name"
1212
AGENT_DESCRIPTION = "gen_ai.agent.description"
13-
AGENT_INVOCATION_INPUT = "gen_ai.agent.invocation_input"
14-
AGENT_INVOCATION_OUTPUT = "gen_ai.agent.invocation_output"
13+
AGENT_INVOCATION_INPUT = "gen_ai.input.messages"
14+
AGENT_INVOCATION_OUTPUT = "gen_ai.output.messages"
15+
AGENT_TOOL_DEFINITIONS = "gen_ai.tool.definitions"
1516
ERROR_TYPE = "error.type"

0 commit comments

Comments
 (0)