From 24ec28c7023bc1b0577c1cb107fe8e21debe86d7 Mon Sep 17 00:00:00 2001 From: Murphy Chen Date: Thu, 9 Oct 2025 10:43:08 +0800 Subject: [PATCH 01/11] typo --- cmd/epp/runner/runner.go | 8 ++ config/charts/inferencepool/README.md | 12 +++ .../templates/epp-deployment.yaml | 9 +++ config/charts/inferencepool/values.yaml | 31 ++++++++ go.mod | 10 +-- go.sum | 12 +-- pkg/common/traces.go | 75 +++++++++++++++++++ 7 files changed, 146 insertions(+), 11 deletions(-) create mode 100644 pkg/common/traces.go diff --git a/cmd/epp/runner/runner.go b/cmd/epp/runner/runner.go index 2842704bb..a7625ac3b 100644 --- a/cmd/epp/runner/runner.go +++ b/cmd/epp/runner/runner.go @@ -106,6 +106,7 @@ var ( modelServerMetricsScheme = flag.String("model-server-metrics-scheme", "http", "Scheme to scrape metrics from pods") modelServerMetricsHttpsInsecureSkipVerify = flag.Bool("model-server-metrics-https-insecure-skip-verify", true, "When using 'https' scheme for 'model-server-metrics-scheme', configure 'InsecureSkipVerify' (default to true)") haEnableLeaderElection = flag.Bool("ha-enable-leader-election", false, "Enables leader election for high availability. When enabled, readiness probes will only pass on the leader.") + tracing = flag.Bool("tracing", true, "Enables emitting traces") setupLog = ctrl.Log.WithName("setup") ) @@ -141,6 +142,13 @@ func (r *Runner) Run(ctx context.Context) error { flag.Parse() initLogging(&opts) + if *tracing { + err := common.InitTracing(ctx, setupLog) + if err != nil { + return err + } + } + setupLog.Info("GIE build", "commit-sha", version.CommitSHA, "build-ref", version.BuildRef) // Validate flags diff --git a/config/charts/inferencepool/README.md b/config/charts/inferencepool/README.md index 41fee834d..cd9b18876 100644 --- a/config/charts/inferencepool/README.md +++ b/config/charts/inferencepool/README.md @@ -214,6 +214,18 @@ These are the options available to you with `provider.name` set to `istio`: | `istio.destinationRule.host` | Custom host value for the destination rule. If not set this will use the default value which is derrived from the epp service name and release namespace to gerenate a valid service address. | | `istio.destinationRule.trafficPolicy.connectionPool` | Configure the connectionPool level settings of the traffic policy | +### Opentelemetry + +he following table list the configurable parameters of opentelemetry trace. + + +| **Parameter Name** | **Description** | +|--------------------------------|------------------------------------------------------------------------------| +| `opentelemetry.enabled` | Enables or disables OpenTelemetry tracing globally for the EndpointPicker. | +| `opentelemetry.autoENVInject.CRInstanceName` | Controls the behavior of opentelemetry-operator auto-instrument. | +| `opentelemetry.env` | A list of environment variables to manually configure the OpenTelemetry SDK. | + + ## Notes This chart will only deploy an InferencePool and its corresponding EndpointPicker extension. Before install the chart, please make sure that the inference extension CRDs are installed in the cluster. For more details, please refer to the [getting started guide](https://gateway-api-inference-extension.sigs.k8s.io/guides/). diff --git a/config/charts/inferencepool/templates/epp-deployment.yaml b/config/charts/inferencepool/templates/epp-deployment.yaml index f012c2e47..f440c5465 100644 --- a/config/charts/inferencepool/templates/epp-deployment.yaml +++ b/config/charts/inferencepool/templates/epp-deployment.yaml @@ -22,6 +22,10 @@ spec: metadata: labels: {{- include "gateway-api-inference-extension.selectorLabels" . | nindent 8 }} + annotations: + {{- if and .Values.opentelemetry.enabled }} + instrumentation.opentelemetry.io/inject-sdk: {{ .Values.opentelemetry.autoENVInject.CRInstanceName | quote }} + {{- end }} spec: serviceAccountName: {{ include "gateway-api-inference-extension.name" . }} # Conservatively, this timeout should mirror the longest grace period of the pods within the pool @@ -62,6 +66,8 @@ spec: - "--{{ .name }}" - "{{ .value }}" {{- end }} + - "--tracing" + - "false" ports: - name: grpc containerPort: 9002 @@ -104,6 +110,9 @@ spec: {{- if .Values.inferenceExtension.env }} {{- toYaml .Values.inferenceExtension.env | nindent 8 }} {{- end }} + {{- if and .Values.opentelemetry.enabled .Values.opentelemetry.env }} + {{- toYaml .Values.opentelemetry.env | nindent 8 }} + {{- end }} volumeMounts: - name: plugins-config-volume mountPath: "/config" diff --git a/config/charts/inferencepool/values.yaml b/config/charts/inferencepool/values.yaml index 91d6a48e6..a58a8483d 100644 --- a/config/charts/inferencepool/values.yaml +++ b/config/charts/inferencepool/values.yaml @@ -86,3 +86,34 @@ istio: # connectionPool: # http: # maxRequestsPerConnection: 256000 + +opentelemetry: + enabled: true + # With this setting you can send trace to the exist opentelemetry collector based on opentelemetry-operator + # See https://github.com/open-telemetry/opentelemetry-operator?tab=readme-ov-file#opentelemetry-auto-instrumentation-injection + autoENVInject: + # The possible values for the annotation can be + # "true" - inject and Instrumentation resource from the namespace. + # "my-instrumentation" - name of Instrumentation CR instance in the current namespace. + # "my-other-namespace/my-instrumentation" - name and namespace of Instrumentation CR instance in + # "false" - do not inject + CRInstanceName: "false" + # Add the required OTel environment manually + # If you also enabled autoENVInject setting, the auto env inject will be skipped by opentelemetry-operator, + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://localhost:4317" + - name: OTEL_SERVICE_NAME + value: "gateway-api-inference-extension" + - name: OTEL_RESOURCE_ATTRIBUTES_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: OTEL_RESOURCE_ATTRIBUTES_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: OTEL_RESOURCE_ATTRIBUTES + value: 'k8s.namespace.name=$(NAMESPACE),k8s.node.name=$(OTEL_RESOURCE_ATTRIBUTES_NODE_NAME),k8s.pod.name=$(OTEL_RESOURCE_ATTRIBUTES_POD_NAME)' \ No newline at end of file diff --git a/go.mod b/go.mod index 83a0a9fe9..4ea5d1c52 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,9 @@ require ( github.com/prometheus/common v0.66.1 github.com/prometheus/prometheus v0.306.0 github.com/stretchr/testify v1.11.1 + go.opentelemetry.io/otel v1.38.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 + go.opentelemetry.io/otel/sdk v1.37.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 golang.org/x/sync v0.17.0 @@ -95,12 +98,9 @@ require ( github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect - go.opentelemetry.io/otel v1.37.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 // indirect - go.opentelemetry.io/otel/metric v1.37.0 // indirect - go.opentelemetry.io/otel/sdk v1.37.0 // indirect - go.opentelemetry.io/otel/trace v1.37.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect go.opentelemetry.io/proto/otlp v1.7.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/automaxprocs v1.6.0 // indirect diff --git a/go.sum b/go.sum index d22adbb02..759bc3a8b 100644 --- a/go.sum +++ b/go.sum @@ -252,20 +252,20 @@ go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJyS go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9HsPl42NIXFIFSUSSs0fiqra0= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0/go.mod h1:90PoxvaEB5n6AOdZvi+yWJQoE95U8Dhhw2bSyRqnTD0= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 h1:JgtbA0xkWHnTmYk7YusopJFX6uleBmAuZ8n05NEh8nQ= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0/go.mod h1:179AK5aar5R3eS9FucPy6rggvU0g52cvKId8pv4+v0c= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= diff --git a/pkg/common/traces.go b/pkg/common/traces.go new file mode 100644 index 000000000..f6381f522 --- /dev/null +++ b/pkg/common/traces.go @@ -0,0 +1,75 @@ +package common + +import ( + "context" + "fmt" + "os" + + "github.com/go-logr/logr" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.37.0" + + "sigs.k8s.io/gateway-api-inference-extension/version" +) + +type errorHandler struct { + logger logr.Logger +} + +func (h *errorHandler) Handle(err error) { + h.logger.Error(err, "trace error occurred") +} + +func InitTracing(ctx context.Context, logger logr.Logger) error { + logger = logger.WithName("trace") + loggerWrap := &errorHandler{logger: logger} + + serviceName, ok := os.LookupEnv("OTEL_SERVICE_NAME") + if !ok { + serviceName = "gateway-api-inference-extension" + os.Setenv("OTEL_SERVICE_NAME", serviceName) + } + + collectorAddr, ok := os.LookupEnv("OTEL_EXPORTER_OTLP_ENDPOINT") + if !ok { + collectorAddr = "http://localhost:4317" + os.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", collectorAddr) + } + + traceExporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithInsecure()) + if err != nil { + loggerWrap.Handle(fmt.Errorf("%s: %v", "new OTel trace gRPC exporter fail", err)) + return nil + } + + logger.Info(fmt.Sprintf("OTel trace exporter connect to: %s with service name: %s", collectorAddr, serviceName)) + opt := []sdktrace.TracerProviderOption{ + sdktrace.WithBatcher(traceExporter), + sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.AlwaysSample())), + sdktrace.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceVersionKey.String(version.BuildRef), + )), + } + + tracerProvider := sdktrace.NewTracerProvider(opt...) + otel.SetTracerProvider(tracerProvider) + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) + otel.SetErrorHandler(loggerWrap) + + go func() { + <-ctx.Done() + err := tracerProvider.Shutdown(context.Background()) + if err != nil { + loggerWrap.Handle(fmt.Errorf("%s: %v", "failed to shutdown MeterProvider", err)) + } + + logger.Info("trace provider shutting down") + }() + + return nil +} From 2a3e397efbb70c8b24da9ce07698025f8068637e Mon Sep 17 00:00:00 2001 From: Murphy Chen Date: Wed, 24 Sep 2025 10:03:49 +0800 Subject: [PATCH 02/11] apply review's suggestion --- config/charts/inferencepool/README.md | 62 +++++++++---------- .../inferencepool/templates/_helpers.tpl | 19 ++++++ .../templates/epp-deployment.yaml | 15 ++--- config/charts/inferencepool/values.yaml | 59 ++++++++---------- go.mod | 3 +- go.sum | 10 +-- pkg/common/traces.go | 60 ++++++++++++++++-- 7 files changed, 141 insertions(+), 87 deletions(-) diff --git a/config/charts/inferencepool/README.md b/config/charts/inferencepool/README.md index cd9b18876..fad4f980d 100644 --- a/config/charts/inferencepool/README.md +++ b/config/charts/inferencepool/README.md @@ -166,30 +166,31 @@ $ helm uninstall pool-1 The following table list the configurable parameters of the chart. -| **Parameter Name** | **Description** | -|---------------------------------------------|------------------------------------------------------------------------------------------------------------------------| -| `inferencePool.apiVersion` | The API version of the InferencePool resource. Defaults to `inference.networking.k8s.io/v1`. This can be changed to `inference.networking.x-k8s.io/v1alpha2` to support older API versions. | -| `inferencePool.targetPortNumber` | Target port number for the vllm backends, will be used to scrape metrics by the inference extension. Defaults to 8000. | -| `inferencePool.modelServerType` | Type of the model servers in the pool, valid options are [vllm, triton-tensorrt-llm], default is vllm. | -| `inferencePool.modelServers.matchLabels` | Label selector to match vllm backends managed by the inference pool. | -| `inferenceExtension.replicas` | Number of replicas for the endpoint picker extension service. If More than one replica is used, EPP will run in HA active-passive mode. Defaults to `1`. | -| `inferenceExtension.image.name` | Name of the container image used for the endpoint picker. | -| `inferenceExtension.image.hub` | Registry URL where the endpoint picker image is hosted. | -| `inferenceExtension.image.tag` | Image tag of the endpoint picker. | -| `inferenceExtension.image.pullPolicy` | Image pull policy for the container. Possible values: `Always`, `IfNotPresent`, or `Never`. Defaults to `Always`. | -| `inferenceExtension.env` | List of environment variables to set in the endpoint picker container as free-form YAML. Defaults to `[]`. | -| `inferenceExtension.extraContainerPorts` | List of additional container ports to expose. Defaults to `[]`. | -| `inferenceExtension.extraServicePorts` | List of additional service ports to expose. Defaults to `[]`. | -| `inferenceExtension.flags` | List of flags which are passed through to endpoint picker. Example flags, enable-pprof, grpc-port etc. Refer [runner.go](https://github.com/kubernetes-sigs/gateway-api-inference-extension/blob/main/cmd/epp/runner/runner.go) for complete list. | -| `inferenceExtension.affinity` | Affinity for the endpoint picker. Defaults to `{}`. | -| `inferenceExtension.tolerations` | Tolerations for the endpoint picker. Defaults to `[]`. | | -| `inferenceExtension.monitoring.interval` | Metrics scraping interval for monitoring. Defaults to `10s`. | -| `inferenceExtension.monitoring.secret.name` | Name of the service account token secret for metrics authentication. Defaults to `inference-gateway-sa-metrics-reader-secret`. | -| `inferenceExtension.monitoring.prometheus.enabled` | Enable Prometheus ServiceMonitor creation for EPP metrics collection. Defaults to `false`. | -| `inferenceExtension.monitoring.gke.enabled` | Enable GKE monitoring resources (`PodMonitoring` and RBAC). Defaults to `false`. | -| `inferenceExtension.pluginsCustomConfig` | Custom config that is passed to EPP as inline yaml. | -| `provider.name` | Name of the Inference Gateway implementation being used. Possible values: [`none`, `gke`, or `istio`]. Defaults to `none`. | -| `provider.gke.autopilot` | Set to `true` if the cluster is a GKE Autopilot cluster. This is only used if `provider.name` is `gke`. Defaults to `false`. | +| **Parameter Name** | **Description** | +|---------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------| +| `inferencePool.apiVersion` | The API version of the InferencePool resource. Defaults to `inference.networking.k8s.io/v1`. This can be changed to `inference.networking.x-k8s.io/v1alpha2` to support older API versions. | +| `inferencePool.targetPortNumber` | Target port number for the vllm backends, will be used to scrape metrics by the inference extension. Defaults to 8000. | +| `inferencePool.modelServerType` | Type of the model servers in the pool, valid options are [vllm, triton-tensorrt-llm], default is vllm. | +| `inferencePool.modelServers.matchLabels` | Label selector to match vllm backends managed by the inference pool. | +| `inferenceExtension.replicas` | Number of replicas for the endpoint picker extension service. If More than one replica is used, EPP will run in HA active-passive mode. Defaults to `1`. | +| `inferenceExtension.image.name` | Name of the container image used for the endpoint picker. | +| `inferenceExtension.image.hub` | Registry URL where the endpoint picker image is hosted. | +| `inferenceExtension.image.tag` | Image tag of the endpoint picker. | +| `inferenceExtension.image.pullPolicy` | Image pull policy for the container. Possible values: `Always`, `IfNotPresent`, or `Never`. Defaults to `Always`. | +| `inferenceExtension.env` | List of environment variables to set in the endpoint picker container as free-form YAML. Defaults to `[]`. | +| `inferenceExtension.extraContainerPorts` | List of additional container ports to expose. Defaults to `[]`. | +| `inferenceExtension.extraServicePorts` | List of additional service ports to expose. Defaults to `[]`. | +| `inferenceExtension.flags` | List of flags which are passed through to endpoint picker. Example flags, enable-pprof, grpc-port etc. Refer [runner.go](https://github.com/kubernetes-sigs/gateway-api-inference-extension/blob/main/cmd/epp/runner/runner.go) for complete list. | +| `inferenceExtension.affinity` | Affinity for the endpoint picker. Defaults to `{}`. | +| `inferenceExtension.tolerations` | Tolerations for the endpoint picker. Defaults to `[]`. | | +| `inferenceExtension.monitoring.interval` | Metrics scraping interval for monitoring. Defaults to `10s`. | +| `inferenceExtension.monitoring.secret.name` | Name of the service account token secret for metrics authentication. Defaults to `inference-gateway-sa-metrics-reader-secret`. | +| `inferenceExtension.monitoring.prometheus.enabled` | Enable Prometheus ServiceMonitor creation for EPP metrics collection. Defaults to `false`. | +| `inferenceExtension.monitoring.gke.enabled` | Enable GKE monitoring resources (`PodMonitoring` and RBAC). Defaults to `false`. | +| `inferenceExtension.pluginsCustomConfig` | Custom config that is passed to EPP as inline yaml. | +| `inferenceExtension.trace.enabled` | Enables or disables OpenTelemetry tracing globally for the EndpointPicker. | +| `provider.name` | Name of the Inference Gateway implementation being used. Possible values: [`none`, `gke`, or `istio`]. Defaults to `none`. | +| `provider.gke.autopilot` | Set to `true` if the cluster is a GKE Autopilot cluster. This is only used if `provider.name` is `gke`. Defaults to `false`. | ### Provider Specific Configuration @@ -214,17 +215,10 @@ These are the options available to you with `provider.name` set to `istio`: | `istio.destinationRule.host` | Custom host value for the destination rule. If not set this will use the default value which is derrived from the epp service name and release namespace to gerenate a valid service address. | | `istio.destinationRule.trafficPolicy.connectionPool` | Configure the connectionPool level settings of the traffic policy | -### Opentelemetry - -he following table list the configurable parameters of opentelemetry trace. - - -| **Parameter Name** | **Description** | -|--------------------------------|------------------------------------------------------------------------------| -| `opentelemetry.enabled` | Enables or disables OpenTelemetry tracing globally for the EndpointPicker. | -| `opentelemetry.autoENVInject.CRInstanceName` | Controls the behavior of opentelemetry-operator auto-instrument. | -| `opentelemetry.env` | A list of environment variables to manually configure the OpenTelemetry SDK. | +## OpenTelemetry +The EndpointPicker supports OpenTelemetry-based tracing. To enable it, use `--set inferenceExtension.trace.enabled=true` +and configure the correct OpenTelemetry collector endpoint via the environment variable `OTEL_EXPORTER_OTLP_ENDPOINT` in `inferenceExtension.env`. ## Notes diff --git a/config/charts/inferencepool/templates/_helpers.tpl b/config/charts/inferencepool/templates/_helpers.tpl index fdc9b1a2b..1cbbcec25 100644 --- a/config/charts/inferencepool/templates/_helpers.tpl +++ b/config/charts/inferencepool/templates/_helpers.tpl @@ -31,3 +31,22 @@ Selector labels {{- define "gateway-api-inference-extension.selectorLabels" -}} inferencepool: {{ include "gateway-api-inference-extension.name" . }} {{- end -}} + + +{{/* +Generate environment variable list for inference extension +Exclude OTEL_ prefixed environment variables when tracing is not enabled +*/}} +{{- define "inferenceExtension.envs" -}} +{{- range .Values.inferenceExtension.env }} +{{- if and (not $.Values.inferenceExtension.trace.enabled) (hasPrefix "OTEL_" .name) }} +{{- else }} +- name: {{ .name }} + {{- if .value }} + value: "{{ .value }}" + {{- else if .valueFrom }} + valueFrom: {{ .valueFrom | toYaml | nindent 4 }} + {{- end }} +{{- end }} +{{- end }} +{{- end -}} \ No newline at end of file diff --git a/config/charts/inferencepool/templates/epp-deployment.yaml b/config/charts/inferencepool/templates/epp-deployment.yaml index f440c5465..1d7694849 100644 --- a/config/charts/inferencepool/templates/epp-deployment.yaml +++ b/config/charts/inferencepool/templates/epp-deployment.yaml @@ -22,10 +22,6 @@ spec: metadata: labels: {{- include "gateway-api-inference-extension.selectorLabels" . | nindent 8 }} - annotations: - {{- if and .Values.opentelemetry.enabled }} - instrumentation.opentelemetry.io/inject-sdk: {{ .Values.opentelemetry.autoENVInject.CRInstanceName | quote }} - {{- end }} spec: serviceAccountName: {{ include "gateway-api-inference-extension.name" . }} # Conservatively, this timeout should mirror the longest grace period of the pods within the pool @@ -67,7 +63,11 @@ spec: - "{{ .value }}" {{- end }} - "--tracing" + {{- if .Values.inferenceExtension.trace.enabled }} + - "true" + {{- else }} - "false" + {{- end }} ports: - name: grpc containerPort: 9002 @@ -107,12 +107,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - {{- if .Values.inferenceExtension.env }} - {{- toYaml .Values.inferenceExtension.env | nindent 8 }} - {{- end }} - {{- if and .Values.opentelemetry.enabled .Values.opentelemetry.env }} - {{- toYaml .Values.opentelemetry.env | nindent 8 }} - {{- end }} + {{- include "inferenceExtension.envs" . | nindent 8 }} volumeMounts: - name: plugins-config-volume mountPath: "/config" diff --git a/config/charts/inferencepool/values.yaml b/config/charts/inferencepool/values.yaml index a58a8483d..c508dfa3f 100644 --- a/config/charts/inferencepool/values.yaml +++ b/config/charts/inferencepool/values.yaml @@ -6,7 +6,29 @@ inferenceExtension: tag: main pullPolicy: Always extProcPort: 9002 - env: [] + env: + # The default OTEL_* environments is used to config the behaviour of OTel SDK + # If you also enabled trace.autoENVInject setting, the auto env inject will be skipped by opentelemetry-operator, + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://localhost:4317" + - name: OTEL_SERVICE_NAME + value: "gateway-api-inference-extension" + - name: OTEL_RESOURCE_ATTRIBUTES_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: OTEL_RESOURCE_ATTRIBUTES_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: OTEL_RESOURCE_ATTRIBUTES + value: 'k8s.namespace.name=$(NAMESPACE),k8s.node.name=$(OTEL_RESOURCE_ATTRIBUTES_NODE_NAME),k8s.pod.name=$(OTEL_RESOURCE_ATTRIBUTES_POD_NAME)' + - name: OTEL_TRACES_SAMPLER + value: "parentbased_traceidratio" + - name: OTEL_TRACES_SAMPLER_ARG + value: "0.1" pluginsConfigFile: "default-plugins.yaml" # Define additional container ports extraContainerPorts: [] @@ -53,6 +75,8 @@ inferenceExtension: gke: enabled: false + trace: + enabled: false inferencePool: targetPorts: @@ -85,35 +109,4 @@ istio: trafficPolicy: {} # connectionPool: # http: - # maxRequestsPerConnection: 256000 - -opentelemetry: - enabled: true - # With this setting you can send trace to the exist opentelemetry collector based on opentelemetry-operator - # See https://github.com/open-telemetry/opentelemetry-operator?tab=readme-ov-file#opentelemetry-auto-instrumentation-injection - autoENVInject: - # The possible values for the annotation can be - # "true" - inject and Instrumentation resource from the namespace. - # "my-instrumentation" - name of Instrumentation CR instance in the current namespace. - # "my-other-namespace/my-instrumentation" - name and namespace of Instrumentation CR instance in - # "false" - do not inject - CRInstanceName: "false" - # Add the required OTel environment manually - # If you also enabled autoENVInject setting, the auto env inject will be skipped by opentelemetry-operator, - env: - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: "http://localhost:4317" - - name: OTEL_SERVICE_NAME - value: "gateway-api-inference-extension" - - name: OTEL_RESOURCE_ATTRIBUTES_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: OTEL_RESOURCE_ATTRIBUTES_POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: OTEL_RESOURCE_ATTRIBUTES - value: 'k8s.namespace.name=$(NAMESPACE),k8s.node.name=$(OTEL_RESOURCE_ATTRIBUTES_NODE_NAME),k8s.pod.name=$(OTEL_RESOURCE_ATTRIBUTES_POD_NAME)' \ No newline at end of file + # maxRequestsPerConnection: 256000 \ No newline at end of file diff --git a/go.mod b/go.mod index 4ea5d1c52..323c240d5 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,8 @@ require ( github.com/stretchr/testify v1.11.1 go.opentelemetry.io/otel v1.38.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 - go.opentelemetry.io/otel/sdk v1.37.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0 + go.opentelemetry.io/otel/sdk v1.38.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 golang.org/x/sync v0.17.0 diff --git a/go.sum b/go.sum index 759bc3a8b..63c80992a 100644 --- a/go.sum +++ b/go.sum @@ -258,12 +258,14 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0/go.mod h1:90PoxvaEB5n6AOdZvi+yWJQoE95U8Dhhw2bSyRqnTD0= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 h1:JgtbA0xkWHnTmYk7YusopJFX6uleBmAuZ8n05NEh8nQ= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0/go.mod h1:179AK5aar5R3eS9FucPy6rggvU0g52cvKId8pv4+v0c= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0 h1:kJxSDN4SgWWTjG/hPp3O7LCGLcHXFlvS2/FFOrwL+SE= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0/go.mod h1:mgIOzS7iZeKJdeB8/NYHrJ48fdGc71Llo5bJ1J4DWUE= go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= -go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= -go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= -go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= -go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= diff --git a/pkg/common/traces.go b/pkg/common/traces.go index f6381f522..56d3ff03a 100644 --- a/pkg/common/traces.go +++ b/pkg/common/traces.go @@ -4,15 +4,18 @@ import ( "context" "fmt" "os" + "strconv" "github.com/go-logr/logr" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.37.0" + "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging" "sigs.k8s.io/gateway-api-inference-extension/version" ) @@ -21,7 +24,7 @@ type errorHandler struct { } func (h *errorHandler) Handle(err error) { - h.logger.Error(err, "trace error occurred") + h.logger.V(logging.DEFAULT).Error(err, "trace error occurred") } func InitTracing(ctx context.Context, logger logr.Logger) error { @@ -40,16 +43,36 @@ func InitTracing(ctx context.Context, logger logr.Logger) error { os.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", collectorAddr) } - traceExporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithInsecure()) + traceExporter, err := initTraceExporter(ctx, logger) if err != nil { - loggerWrap.Handle(fmt.Errorf("%s: %v", "new OTel trace gRPC exporter fail", err)) + loggerWrap.Handle(fmt.Errorf("%s: %v", "init trace exporter fail", err)) return nil } - logger.Info(fmt.Sprintf("OTel trace exporter connect to: %s with service name: %s", collectorAddr, serviceName)) + // Go SDK doesn't have an automatic sampler, handle manually + samplerType, ok := os.LookupEnv("OTEL_TRACES_SAMPLER") + if !ok { + samplerType = "parentbased_traceidratio" + } + samplerARG, ok := os.LookupEnv("OTEL_TRACES_SAMPLER_ARG") + if !ok { + samplerARG = "0.1" + } + + sampler := sdktrace.ParentBased(sdktrace.TraceIDRatioBased(0.1)) + if samplerType == "parentbased_traceidratio" { + fraction, err := strconv.ParseFloat(samplerARG, 64) + if err != nil { + fraction = 0.1 + } + sampler = sdktrace.ParentBased(sdktrace.TraceIDRatioBased(fraction)) + } else { + loggerWrap.Handle(fmt.Errorf("un supported sampler type: %s, fallback to parentbased_traceidratio with 0.1 Ratio", samplerType)) + } + opt := []sdktrace.TracerProviderOption{ sdktrace.WithBatcher(traceExporter), - sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.AlwaysSample())), + sdktrace.WithSampler(sampler), sdktrace.WithResource(resource.NewWithAttributes( semconv.SchemaURL, semconv.ServiceVersionKey.String(version.BuildRef), @@ -73,3 +96,30 @@ func InitTracing(ctx context.Context, logger logr.Logger) error { return nil } + +// initTraceExporter create a SpanExporter +// support exporter type +// - console: export spans in console for development use case +// - otlp: export spans through gRPC to an opentelemetry collector +func initTraceExporter(ctx context.Context, logger logr.Logger) (sdktrace.SpanExporter, error) { + var traceExporter sdktrace.SpanExporter + traceExporter, err := stdouttrace.New(stdouttrace.WithPrettyPrint()) + if err != nil { + return nil, fmt.Errorf("fail to create stdouttrace exporter: %w", err) + } + + exporterType, ok := os.LookupEnv("OTEL_TRACES_EXPORTER") + if !ok { + exporterType = "console" + } + + logger.Info("init OTel trace exporter", "type", exporterType) + if exporterType == "otlp" { + traceExporter, err = otlptracegrpc.New(ctx, otlptracegrpc.WithInsecure()) + if err != nil { + return nil, fmt.Errorf("fail to create otlp-grcp exporter: %w", err) + } + } + + return traceExporter, nil +} From 8cc4e8ce46f81f181908c60933fb472e142b8973 Mon Sep 17 00:00:00 2001 From: Murphy Chen Date: Wed, 24 Sep 2025 23:48:01 +0800 Subject: [PATCH 03/11] update --- pkg/common/{traces.go => telemetry.go} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename pkg/common/{traces.go => telemetry.go} (97%) diff --git a/pkg/common/traces.go b/pkg/common/telemetry.go similarity index 97% rename from pkg/common/traces.go rename to pkg/common/telemetry.go index 56d3ff03a..a0ccc10fb 100644 --- a/pkg/common/traces.go +++ b/pkg/common/telemetry.go @@ -88,7 +88,7 @@ func InitTracing(ctx context.Context, logger logr.Logger) error { <-ctx.Done() err := tracerProvider.Shutdown(context.Background()) if err != nil { - loggerWrap.Handle(fmt.Errorf("%s: %v", "failed to shutdown MeterProvider", err)) + loggerWrap.Handle(fmt.Errorf("%s: %v", "failed to shutdown TraceProvider", err)) } logger.Info("trace provider shutting down") From 831ba43d1ab9c90e63e8c30d7c4e9c879ccd05a5 Mon Sep 17 00:00:00 2001 From: Murphy Chen Date: Sat, 27 Sep 2025 19:28:36 +0800 Subject: [PATCH 04/11] move otel env to helm template --- .../inferencepool/templates/_helpers.tpl | 19 ------------- .../templates/epp-deployment.yaml | 26 ++++++++++++++++- config/charts/inferencepool/values.yaml | 28 ++++--------------- pkg/common/telemetry.go | 5 ++-- 4 files changed, 33 insertions(+), 45 deletions(-) diff --git a/config/charts/inferencepool/templates/_helpers.tpl b/config/charts/inferencepool/templates/_helpers.tpl index 1cbbcec25..fdc9b1a2b 100644 --- a/config/charts/inferencepool/templates/_helpers.tpl +++ b/config/charts/inferencepool/templates/_helpers.tpl @@ -31,22 +31,3 @@ Selector labels {{- define "gateway-api-inference-extension.selectorLabels" -}} inferencepool: {{ include "gateway-api-inference-extension.name" . }} {{- end -}} - - -{{/* -Generate environment variable list for inference extension -Exclude OTEL_ prefixed environment variables when tracing is not enabled -*/}} -{{- define "inferenceExtension.envs" -}} -{{- range .Values.inferenceExtension.env }} -{{- if and (not $.Values.inferenceExtension.trace.enabled) (hasPrefix "OTEL_" .name) }} -{{- else }} -- name: {{ .name }} - {{- if .value }} - value: "{{ .value }}" - {{- else if .valueFrom }} - valueFrom: {{ .valueFrom | toYaml | nindent 4 }} - {{- end }} -{{- end }} -{{- end }} -{{- end -}} \ No newline at end of file diff --git a/config/charts/inferencepool/templates/epp-deployment.yaml b/config/charts/inferencepool/templates/epp-deployment.yaml index 1d7694849..699b3cfc9 100644 --- a/config/charts/inferencepool/templates/epp-deployment.yaml +++ b/config/charts/inferencepool/templates/epp-deployment.yaml @@ -107,7 +107,31 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - {{- include "inferenceExtension.envs" . | nindent 8 }} + {{- if .Values.inferenceExtension.trace.enabled }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.inferenceExtension.trace.otelExporterEndpoint | default "http://localhost:4317" | quote }} + - name: OTEL_SERVICE_NAME + value: "gateway-api-inference-extension" + - name: OTEL_RESOURCE_ATTRIBUTES_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: OTEL_RESOURCE_ATTRIBUTES_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: OTEL_RESOURCE_ATTRIBUTES + value: 'k8s.namespace.name=$(NAMESPACE),k8s.node.name=$(OTEL_RESOURCE_ATTRIBUTES_NODE_NAME),k8s.pod.name=$(OTEL_RESOURCE_ATTRIBUTES_POD_NAME)' + - name: OTEL_TRACES_SAMPLER + value: {{ .Values.inferenceExtension.trace.sampling.sampler | default "parentbased_traceidratio" | quote }} + - name: OTEL_TRACES_SAMPLER_ARG + value: {{ .Values.inferenceExtension.trace.sampling.samplerArg | default "0.1" | quote }} + {{- end }} + {{- if .Values.inferenceExtension.env }} + {{- toYaml .Values.inferenceExtension.env | nindent 8 }} + {{- end }} volumeMounts: - name: plugins-config-volume mountPath: "/config" diff --git a/config/charts/inferencepool/values.yaml b/config/charts/inferencepool/values.yaml index c508dfa3f..b62c1deeb 100644 --- a/config/charts/inferencepool/values.yaml +++ b/config/charts/inferencepool/values.yaml @@ -6,29 +6,7 @@ inferenceExtension: tag: main pullPolicy: Always extProcPort: 9002 - env: - # The default OTEL_* environments is used to config the behaviour of OTel SDK - # If you also enabled trace.autoENVInject setting, the auto env inject will be skipped by opentelemetry-operator, - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: "http://localhost:4317" - - name: OTEL_SERVICE_NAME - value: "gateway-api-inference-extension" - - name: OTEL_RESOURCE_ATTRIBUTES_NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: OTEL_RESOURCE_ATTRIBUTES_POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: OTEL_RESOURCE_ATTRIBUTES - value: 'k8s.namespace.name=$(NAMESPACE),k8s.node.name=$(OTEL_RESOURCE_ATTRIBUTES_NODE_NAME),k8s.pod.name=$(OTEL_RESOURCE_ATTRIBUTES_POD_NAME)' - - name: OTEL_TRACES_SAMPLER - value: "parentbased_traceidratio" - - name: OTEL_TRACES_SAMPLER_ARG - value: "0.1" + env: [] pluginsConfigFile: "default-plugins.yaml" # Define additional container ports extraContainerPorts: [] @@ -77,6 +55,10 @@ inferenceExtension: enabled: false trace: enabled: false + otelExporterEndpoint: "http://localhost:4317" + sampling: + sampler: "parentbased_traceidratio" + samplerArg: "0.1" inferencePool: targetPorts: diff --git a/pkg/common/telemetry.go b/pkg/common/telemetry.go index a0ccc10fb..47747ee6b 100644 --- a/pkg/common/telemetry.go +++ b/pkg/common/telemetry.go @@ -31,13 +31,13 @@ func InitTracing(ctx context.Context, logger logr.Logger) error { logger = logger.WithName("trace") loggerWrap := &errorHandler{logger: logger} - serviceName, ok := os.LookupEnv("OTEL_SERVICE_NAME") + serviceName, ok := os.LookupEnv("OTEL_SERVICE_NAME") //nolint:ineffassign if !ok { serviceName = "gateway-api-inference-extension" os.Setenv("OTEL_SERVICE_NAME", serviceName) } - collectorAddr, ok := os.LookupEnv("OTEL_EXPORTER_OTLP_ENDPOINT") + collectorAddr, ok := os.LookupEnv("OTEL_EXPORTER_OTLP_ENDPOINT") //nolint:ineffassign if !ok { collectorAddr = "http://localhost:4317" os.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", collectorAddr) @@ -65,6 +65,7 @@ func InitTracing(ctx context.Context, logger logr.Logger) error { if err != nil { fraction = 0.1 } + sampler = sdktrace.ParentBased(sdktrace.TraceIDRatioBased(fraction)) } else { loggerWrap.Handle(fmt.Errorf("un supported sampler type: %s, fallback to parentbased_traceidratio with 0.1 Ratio", samplerType)) From 154f6fcbfc4ee4d5de5453f4f3836738d186dc04 Mon Sep 17 00:00:00 2001 From: Murphy Chen Date: Sat, 27 Sep 2025 20:04:22 +0800 Subject: [PATCH 05/11] update --- config/charts/inferencepool/templates/epp-deployment.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/config/charts/inferencepool/templates/epp-deployment.yaml b/config/charts/inferencepool/templates/epp-deployment.yaml index 699b3cfc9..b0fd3fe66 100644 --- a/config/charts/inferencepool/templates/epp-deployment.yaml +++ b/config/charts/inferencepool/templates/epp-deployment.yaml @@ -108,10 +108,12 @@ spec: fieldRef: fieldPath: metadata.namespace {{- if .Values.inferenceExtension.trace.enabled }} - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: {{ .Values.inferenceExtension.trace.otelExporterEndpoint | default "http://localhost:4317" | quote }} - name: OTEL_SERVICE_NAME value: "gateway-api-inference-extension" + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.inferenceExtension.trace.otelExporterEndpoint | default "http://localhost:4317" | quote }} + - name: OTEL_TRACES_EXPORTER + value: "otlp" - name: OTEL_RESOURCE_ATTRIBUTES_NODE_NAME valueFrom: fieldRef: From 124104480fc65a6b8975118594a2a8dd90fb1ad3 Mon Sep 17 00:00:00 2001 From: Murphy Chen Date: Sat, 27 Sep 2025 20:17:10 +0800 Subject: [PATCH 06/11] skip lint --- pkg/common/telemetry.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/common/telemetry.go b/pkg/common/telemetry.go index 47747ee6b..e4994943f 100644 --- a/pkg/common/telemetry.go +++ b/pkg/common/telemetry.go @@ -31,13 +31,13 @@ func InitTracing(ctx context.Context, logger logr.Logger) error { logger = logger.WithName("trace") loggerWrap := &errorHandler{logger: logger} - serviceName, ok := os.LookupEnv("OTEL_SERVICE_NAME") //nolint:ineffassign + serviceName, ok := os.LookupEnv("OTEL_SERVICE_NAME") //nolint:staticcheck if !ok { serviceName = "gateway-api-inference-extension" os.Setenv("OTEL_SERVICE_NAME", serviceName) } - collectorAddr, ok := os.LookupEnv("OTEL_EXPORTER_OTLP_ENDPOINT") //nolint:ineffassign + collectorAddr, ok := os.LookupEnv("OTEL_EXPORTER_OTLP_ENDPOINT") //nolint:staticcheck if !ok { collectorAddr = "http://localhost:4317" os.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", collectorAddr) From 0ec310dd129d2c58d59cc493c9d86d166c2780de Mon Sep 17 00:00:00 2001 From: Murphy Chen Date: Sat, 27 Sep 2025 20:33:36 +0800 Subject: [PATCH 07/11] fix lint --- pkg/common/telemetry.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/common/telemetry.go b/pkg/common/telemetry.go index e4994943f..3900d0441 100644 --- a/pkg/common/telemetry.go +++ b/pkg/common/telemetry.go @@ -31,16 +31,14 @@ func InitTracing(ctx context.Context, logger logr.Logger) error { logger = logger.WithName("trace") loggerWrap := &errorHandler{logger: logger} - serviceName, ok := os.LookupEnv("OTEL_SERVICE_NAME") //nolint:staticcheck + _, ok := os.LookupEnv("OTEL_SERVICE_NAME") if !ok { - serviceName = "gateway-api-inference-extension" - os.Setenv("OTEL_SERVICE_NAME", serviceName) + os.Setenv("OTEL_SERVICE_NAME", "gateway-api-inference-extension") } - collectorAddr, ok := os.LookupEnv("OTEL_EXPORTER_OTLP_ENDPOINT") //nolint:staticcheck + _, ok = os.LookupEnv("OTEL_EXPORTER_OTLP_ENDPOINT") if !ok { - collectorAddr = "http://localhost:4317" - os.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", collectorAddr) + os.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317") } traceExporter, err := initTraceExporter(ctx, logger) From 14fce188355c29ce98eceb6556da7a4ac9cb4500 Mon Sep 17 00:00:00 2001 From: Murphy Chen Date: Sat, 27 Sep 2025 20:51:38 +0800 Subject: [PATCH 08/11] add file header --- pkg/common/telemetry.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/common/telemetry.go b/pkg/common/telemetry.go index 3900d0441..9907e3974 100644 --- a/pkg/common/telemetry.go +++ b/pkg/common/telemetry.go @@ -1,3 +1,19 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package common import ( From 931dfa1e41207088a450ac25745a0a7dd9b7da15 Mon Sep 17 00:00:00 2001 From: Murphy Chen Date: Tue, 30 Sep 2025 10:25:09 +0800 Subject: [PATCH 09/11] update README.md --- config/charts/inferencepool/README.md | 69 +++++++++++-------- .../templates/epp-deployment.yaml | 10 +-- config/charts/inferencepool/values.yaml | 2 +- pkg/common/telemetry.go | 2 +- 4 files changed, 48 insertions(+), 35 deletions(-) diff --git a/config/charts/inferencepool/README.md b/config/charts/inferencepool/README.md index fad4f980d..f6354bfee 100644 --- a/config/charts/inferencepool/README.md +++ b/config/charts/inferencepool/README.md @@ -166,31 +166,34 @@ $ helm uninstall pool-1 The following table list the configurable parameters of the chart. -| **Parameter Name** | **Description** | -|---------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------| -| `inferencePool.apiVersion` | The API version of the InferencePool resource. Defaults to `inference.networking.k8s.io/v1`. This can be changed to `inference.networking.x-k8s.io/v1alpha2` to support older API versions. | -| `inferencePool.targetPortNumber` | Target port number for the vllm backends, will be used to scrape metrics by the inference extension. Defaults to 8000. | -| `inferencePool.modelServerType` | Type of the model servers in the pool, valid options are [vllm, triton-tensorrt-llm], default is vllm. | -| `inferencePool.modelServers.matchLabels` | Label selector to match vllm backends managed by the inference pool. | -| `inferenceExtension.replicas` | Number of replicas for the endpoint picker extension service. If More than one replica is used, EPP will run in HA active-passive mode. Defaults to `1`. | -| `inferenceExtension.image.name` | Name of the container image used for the endpoint picker. | -| `inferenceExtension.image.hub` | Registry URL where the endpoint picker image is hosted. | -| `inferenceExtension.image.tag` | Image tag of the endpoint picker. | -| `inferenceExtension.image.pullPolicy` | Image pull policy for the container. Possible values: `Always`, `IfNotPresent`, or `Never`. Defaults to `Always`. | -| `inferenceExtension.env` | List of environment variables to set in the endpoint picker container as free-form YAML. Defaults to `[]`. | -| `inferenceExtension.extraContainerPorts` | List of additional container ports to expose. Defaults to `[]`. | -| `inferenceExtension.extraServicePorts` | List of additional service ports to expose. Defaults to `[]`. | -| `inferenceExtension.flags` | List of flags which are passed through to endpoint picker. Example flags, enable-pprof, grpc-port etc. Refer [runner.go](https://github.com/kubernetes-sigs/gateway-api-inference-extension/blob/main/cmd/epp/runner/runner.go) for complete list. | -| `inferenceExtension.affinity` | Affinity for the endpoint picker. Defaults to `{}`. | -| `inferenceExtension.tolerations` | Tolerations for the endpoint picker. Defaults to `[]`. | | -| `inferenceExtension.monitoring.interval` | Metrics scraping interval for monitoring. Defaults to `10s`. | -| `inferenceExtension.monitoring.secret.name` | Name of the service account token secret for metrics authentication. Defaults to `inference-gateway-sa-metrics-reader-secret`. | -| `inferenceExtension.monitoring.prometheus.enabled` | Enable Prometheus ServiceMonitor creation for EPP metrics collection. Defaults to `false`. | -| `inferenceExtension.monitoring.gke.enabled` | Enable GKE monitoring resources (`PodMonitoring` and RBAC). Defaults to `false`. | -| `inferenceExtension.pluginsCustomConfig` | Custom config that is passed to EPP as inline yaml. | -| `inferenceExtension.trace.enabled` | Enables or disables OpenTelemetry tracing globally for the EndpointPicker. | -| `provider.name` | Name of the Inference Gateway implementation being used. Possible values: [`none`, `gke`, or `istio`]. Defaults to `none`. | -| `provider.gke.autopilot` | Set to `true` if the cluster is a GKE Autopilot cluster. This is only used if `provider.name` is `gke`. Defaults to `false`. | +| **Parameter Name** | **Description** | +|----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `inferencePool.apiVersion` | The API version of the InferencePool resource. Defaults to `inference.networking.k8s.io/v1`. This can be changed to `inference.networking.x-k8s.io/v1alpha2` to support older API versions. | +| `inferencePool.targetPortNumber` | Target port number for the vllm backends, will be used to scrape metrics by the inference extension. Defaults to 8000. | +| `inferencePool.modelServerType` | Type of the model servers in the pool, valid options are [vllm, triton-tensorrt-llm], default is vllm. | +| `inferencePool.modelServers.matchLabels` | Label selector to match vllm backends managed by the inference pool. | +| `inferenceExtension.replicas` | Number of replicas for the endpoint picker extension service. If More than one replica is used, EPP will run in HA active-passive mode. Defaults to `1`. | +| `inferenceExtension.image.name` | Name of the container image used for the endpoint picker. | +| `inferenceExtension.image.hub` | Registry URL where the endpoint picker image is hosted. | +| `inferenceExtension.image.tag` | Image tag of the endpoint picker. | +| `inferenceExtension.image.pullPolicy` | Image pull policy for the container. Possible values: `Always`, `IfNotPresent`, or `Never`. Defaults to `Always`. | +| `inferenceExtension.env` | List of environment variables to set in the endpoint picker container as free-form YAML. Defaults to `[]`. | +| `inferenceExtension.extraContainerPorts` | List of additional container ports to expose. Defaults to `[]`. | +| `inferenceExtension.extraServicePorts` | List of additional service ports to expose. Defaults to `[]`. | +| `inferenceExtension.flags` | List of flags which are passed through to endpoint picker. Example flags, enable-pprof, grpc-port etc. Refer [runner.go](https://github.com/kubernetes-sigs/gateway-api-inference-extension/blob/main/cmd/epp/runner/runner.go) for complete list. | +| `inferenceExtension.affinity` | Affinity for the endpoint picker. Defaults to `{}`. | +| `inferenceExtension.tolerations` | Tolerations for the endpoint picker. Defaults to `[]`. | +| `inferenceExtension.monitoring.interval` | Metrics scraping interval for monitoring. Defaults to `10s`. | +| `inferenceExtension.monitoring.secret.name` | Name of the service account token secret for metrics authentication. Defaults to `inference-gateway-sa-metrics-reader-secret`. | +| `inferenceExtension.monitoring.prometheus.enabled` | Enable Prometheus ServiceMonitor creation for EPP metrics collection. Defaults to `false`. | +| `inferenceExtension.monitoring.gke.enabled` | Enable GKE monitoring resources (`PodMonitoring` and RBAC). Defaults to `false`. | +| `inferenceExtension.pluginsCustomConfig` | Custom config that is passed to EPP as inline yaml. | +| `inferenceExtension.tracing.enabled` | Enables or disables OpenTelemetry tracing globally for the EndpointPicker. | +| `inferenceExtension.tracing.otelExporterEndpoint` | OpenTelemetry collector endpoint. | +| `inferenceExtension.tracing.sampling.sampler` | The trace sampler to use. Currently, only `parentbased_traceidratio` is supported. This sampler respects the parent span’s sampling decision when present, and applies the configured ratio for root spans. | +| `inferenceExtension.tracing.sampling.samplerArg` | Sampler-specific argument. For `parentbased_traceidratio`, this defines the base sampling rate for new traces (root spans), as a float string in the range [0.0, 1.0]. For example, "0.1" enables 10% sampling. | +| `provider.name` | Name of the Inference Gateway implementation being used. Possible values: [`none`, `gke`, or `istio`]. Defaults to `none`. | +| `provider.gke.autopilot` | Set to `true` if the cluster is a GKE Autopilot cluster. This is only used if `provider.name` is `gke`. Defaults to `false`. | ### Provider Specific Configuration @@ -215,10 +218,20 @@ These are the options available to you with `provider.name` set to `istio`: | `istio.destinationRule.host` | Custom host value for the destination rule. If not set this will use the default value which is derrived from the epp service name and release namespace to gerenate a valid service address. | | `istio.destinationRule.trafficPolicy.connectionPool` | Configure the connectionPool level settings of the traffic policy | -## OpenTelemetry +#### OpenTelemetry -The EndpointPicker supports OpenTelemetry-based tracing. To enable it, use `--set inferenceExtension.trace.enabled=true` -and configure the correct OpenTelemetry collector endpoint via the environment variable `OTEL_EXPORTER_OTLP_ENDPOINT` in `inferenceExtension.env`. +The EndpointPicker supports OpenTelemetry-based tracing. To enable trace collection, use the following configuration: +```yaml +inferenceExtension: + tracing: + enabled: true + otelExporterEndpoint: "http://localhost:4317" + sampling: + sampler: "parentbased_traceidratio" + samplerArg: "0.1" +``` +Make sure that the `otelExporterEndpoint` points to your OpenTelemetry collector endpoint. +Current only the `parentbased_traceidratio` sampler is supported. You can adjust the base sampling ratio using the `samplerArg` (e.g., 0.1 means 10% of traces will be sampled). ## Notes diff --git a/config/charts/inferencepool/templates/epp-deployment.yaml b/config/charts/inferencepool/templates/epp-deployment.yaml index b0fd3fe66..55eb16776 100644 --- a/config/charts/inferencepool/templates/epp-deployment.yaml +++ b/config/charts/inferencepool/templates/epp-deployment.yaml @@ -63,7 +63,7 @@ spec: - "{{ .value }}" {{- end }} - "--tracing" - {{- if .Values.inferenceExtension.trace.enabled }} + {{- if .Values.inferenceExtension.tracing.enabled }} - "true" {{- else }} - "false" @@ -107,11 +107,11 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - {{- if .Values.inferenceExtension.trace.enabled }} + {{- if .Values.inferenceExtension.tracing.enabled }} - name: OTEL_SERVICE_NAME value: "gateway-api-inference-extension" - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: {{ .Values.inferenceExtension.trace.otelExporterEndpoint | default "http://localhost:4317" | quote }} + value: {{ .Values.inferenceExtension.tracing.otelExporterEndpoint | default "http://localhost:4317" | quote }} - name: OTEL_TRACES_EXPORTER value: "otlp" - name: OTEL_RESOURCE_ATTRIBUTES_NODE_NAME @@ -127,9 +127,9 @@ spec: - name: OTEL_RESOURCE_ATTRIBUTES value: 'k8s.namespace.name=$(NAMESPACE),k8s.node.name=$(OTEL_RESOURCE_ATTRIBUTES_NODE_NAME),k8s.pod.name=$(OTEL_RESOURCE_ATTRIBUTES_POD_NAME)' - name: OTEL_TRACES_SAMPLER - value: {{ .Values.inferenceExtension.trace.sampling.sampler | default "parentbased_traceidratio" | quote }} + value: {{ .Values.inferenceExtension.tracing.sampling.sampler | default "parentbased_traceidratio" | quote }} - name: OTEL_TRACES_SAMPLER_ARG - value: {{ .Values.inferenceExtension.trace.sampling.samplerArg | default "0.1" | quote }} + value: {{ .Values.inferenceExtension.tracing.sampling.samplerArg | default "0.1" | quote }} {{- end }} {{- if .Values.inferenceExtension.env }} {{- toYaml .Values.inferenceExtension.env | nindent 8 }} diff --git a/config/charts/inferencepool/values.yaml b/config/charts/inferencepool/values.yaml index b62c1deeb..f901f7f0f 100644 --- a/config/charts/inferencepool/values.yaml +++ b/config/charts/inferencepool/values.yaml @@ -53,7 +53,7 @@ inferenceExtension: gke: enabled: false - trace: + tracing: enabled: false otelExporterEndpoint: "http://localhost:4317" sampling: diff --git a/pkg/common/telemetry.go b/pkg/common/telemetry.go index 9907e3974..33a16ba6b 100644 --- a/pkg/common/telemetry.go +++ b/pkg/common/telemetry.go @@ -60,7 +60,7 @@ func InitTracing(ctx context.Context, logger logr.Logger) error { traceExporter, err := initTraceExporter(ctx, logger) if err != nil { loggerWrap.Handle(fmt.Errorf("%s: %v", "init trace exporter fail", err)) - return nil + return err } // Go SDK doesn't have an automatic sampler, handle manually From bbb49c64128fd01ebb800cc7385a54ab7f73d8ea Mon Sep 17 00:00:00 2001 From: Murphy Chen Date: Wed, 1 Oct 2025 07:11:44 +0800 Subject: [PATCH 10/11] typo --- pkg/common/telemetry.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/common/telemetry.go b/pkg/common/telemetry.go index 33a16ba6b..3723e0f7d 100644 --- a/pkg/common/telemetry.go +++ b/pkg/common/telemetry.go @@ -59,7 +59,7 @@ func InitTracing(ctx context.Context, logger logr.Logger) error { traceExporter, err := initTraceExporter(ctx, logger) if err != nil { - loggerWrap.Handle(fmt.Errorf("%s: %v", "init trace exporter fail", err)) + loggerWrap.Handle(fmt.Errorf("%s: %v", "init trace exporter failed", err)) return err } @@ -82,7 +82,7 @@ func InitTracing(ctx context.Context, logger logr.Logger) error { sampler = sdktrace.ParentBased(sdktrace.TraceIDRatioBased(fraction)) } else { - loggerWrap.Handle(fmt.Errorf("un supported sampler type: %s, fallback to parentbased_traceidratio with 0.1 Ratio", samplerType)) + loggerWrap.Handle(fmt.Errorf("unsupported sampler type: %s, fallback to parentbased_traceidratio with 0.1 Ratio", samplerType)) } opt := []sdktrace.TracerProviderOption{ @@ -106,7 +106,7 @@ func InitTracing(ctx context.Context, logger logr.Logger) error { loggerWrap.Handle(fmt.Errorf("%s: %v", "failed to shutdown TraceProvider", err)) } - logger.Info("trace provider shutting down") + logger.V(logging.DEFAULT).Info("trace provider shutting down") }() return nil @@ -120,7 +120,7 @@ func initTraceExporter(ctx context.Context, logger logr.Logger) (sdktrace.SpanEx var traceExporter sdktrace.SpanExporter traceExporter, err := stdouttrace.New(stdouttrace.WithPrettyPrint()) if err != nil { - return nil, fmt.Errorf("fail to create stdouttrace exporter: %w", err) + return nil, fmt.Errorf("failed to create stdouttrace exporter: %w", err) } exporterType, ok := os.LookupEnv("OTEL_TRACES_EXPORTER") @@ -132,7 +132,7 @@ func initTraceExporter(ctx context.Context, logger logr.Logger) (sdktrace.SpanEx if exporterType == "otlp" { traceExporter, err = otlptracegrpc.New(ctx, otlptracegrpc.WithInsecure()) if err != nil { - return nil, fmt.Errorf("fail to create otlp-grcp exporter: %w", err) + return nil, fmt.Errorf("failed to create otlp-grcp exporter: %w", err) } } From c6a6e42383c465f76a1634c8c1dc32adde490ef5 Mon Sep 17 00:00:00 2001 From: Murphy Chen Date: Fri, 10 Oct 2025 09:57:07 +0800 Subject: [PATCH 11/11] apply review's suggestion --- config/charts/inferencepool/templates/epp-deployment.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/charts/inferencepool/templates/epp-deployment.yaml b/config/charts/inferencepool/templates/epp-deployment.yaml index 55eb16776..120240217 100644 --- a/config/charts/inferencepool/templates/epp-deployment.yaml +++ b/config/charts/inferencepool/templates/epp-deployment.yaml @@ -111,7 +111,7 @@ spec: - name: OTEL_SERVICE_NAME value: "gateway-api-inference-extension" - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: {{ .Values.inferenceExtension.tracing.otelExporterEndpoint | default "http://localhost:4317" | quote }} + value: {{ .Values.inferenceExtension.tracing.otelExporterEndpoint | quote }} - name: OTEL_TRACES_EXPORTER value: "otlp" - name: OTEL_RESOURCE_ATTRIBUTES_NODE_NAME @@ -127,9 +127,9 @@ spec: - name: OTEL_RESOURCE_ATTRIBUTES value: 'k8s.namespace.name=$(NAMESPACE),k8s.node.name=$(OTEL_RESOURCE_ATTRIBUTES_NODE_NAME),k8s.pod.name=$(OTEL_RESOURCE_ATTRIBUTES_POD_NAME)' - name: OTEL_TRACES_SAMPLER - value: {{ .Values.inferenceExtension.tracing.sampling.sampler | default "parentbased_traceidratio" | quote }} + value: {{ .Values.inferenceExtension.tracing.sampling.sampler | quote }} - name: OTEL_TRACES_SAMPLER_ARG - value: {{ .Values.inferenceExtension.tracing.sampling.samplerArg | default "0.1" | quote }} + value: {{ .Values.inferenceExtension.tracing.sampling.samplerArg | quote }} {{- end }} {{- if .Values.inferenceExtension.env }} {{- toYaml .Values.inferenceExtension.env | nindent 8 }}