Include pod UID and container name as optional trace labels

It's useful to include these fields in the trace span attributes to help correlate traffic within a cluster.

This propagates these labels through the trace initialization, similar to how we handle the service name (and other labels that last for the entire pod lifetime).

See linkerd/linkerd2#13501 for the corresponding control plane change.

Signed-off-by: Scott Fleener <[email protected]>

linkerd/app/src/env.rs

@@ -266,6 +266,9 @@ const ENV_OUTBOUND_HTTP1_CONNECTION_POOL_IDLE_TIMEOUT: &str =
 
 const ENV_SHUTDOWN_GRACE_PERIOD: &str = "LINKERD2_PROXY_SHUTDOWN_GRACE_PERIOD";
 
+const ENV_POD_UID: &str = "_pod_uid";
+const ENV_POD_CONTAINER_NAME: &str = "_pod_containerName";
+
 // Default values for various configuration fields
 const DEFAULT_OUTBOUND_LISTEN_ADDR: &str = "127.0.0.1:4140";
 pub const DEFAULT_INBOUND_LISTEN_ADDR: &str = "0.0.0.0:4143";
@@ -395,6 +398,9 @@ pub fn parse_config<S: Strings>(strings: &S) -> Result<super::Config, EnvError>
 
     let shutdown_grace_period = parse(strings, ENV_SHUTDOWN_GRACE_PERIOD, parse_duration);
 
+    let pod_uid = strings.get(ENV_POD_UID);
+    let container_name = strings.get(ENV_POD_CONTAINER_NAME);
+
     let inbound_discovery_idle_timeout =
         parse(strings, ENV_INBOUND_DISCOVERY_IDLE_TIMEOUT, parse_duration);
     let outbound_discovery_idle_timeout =
@@ -845,10 +851,14 @@ pub fn parse_config<S: Strings>(strings: &S) -> Result<super::Config, EnvError>
                 .unwrap_or_default();
 
             let trace_service_name = trace_service_name.ok().flatten();
+            let pod_uid = pod_uid.ok().flatten();
+            let container_name = container_name.ok().flatten();
 
             trace_collector::Config::Enabled(Box::new(trace_collector::EnabledConfig {
                 attributes,
                 hostname: hostname?,
+                pod_uid,
+                container_name,
                 service_name: trace_service_name,
                 control: ControlConfig {
                     addr,

linkerd/app/src/trace_collector.rs

@@ -1,11 +1,14 @@
-use linkerd_app_core::http_tracing::{CollectorProtocol, SpanSink};
-use linkerd_app_core::metrics::ControlHttp as HttpMetrics;
-use linkerd_app_core::svc::NewService;
-use linkerd_app_core::{control, dns, identity, opencensus, opentelemetry};
+use linkerd_app_core::{
+    control, dns,
+    http_tracing::{CollectorProtocol, SpanSink},
+    identity,
+    metrics::ControlHttp as HttpMetrics,
+    opencensus, opentelemetry,
+    svc::NewService,
+};
 use linkerd_error::Error;
-use std::collections::HashMap;
-use std::future::Future;
-use std::pin::Pin;
+use otel_collector::OtelCollectorAttributes;
+use std::{collections::HashMap, future::Future, pin::Pin};
 
 pub mod oc_collector;
 pub mod otel_collector;
@@ -24,6 +27,8 @@ pub struct EnabledConfig {
     pub control: control::Config,
     pub attributes: HashMap<String, String>,
     pub hostname: Option<String>,
+    pub pod_uid: Option<String>,
+    pub container_name: Option<String>,
     pub service_name: Option<String>,
     pub kind: CollectorProtocol,
 }
@@ -92,14 +97,21 @@ impl Config {
                         svc,
                         legacy_oc_metrics,
                     ),
-                    CollectorProtocol::OpenTelemetry => otel_collector::create_collector(
-                        addr.clone(),
-                        inner.hostname,
-                        svc_name,
-                        inner.attributes,
-                        svc,
-                        legacy_otel_metrics,
-                    ),
+                    CollectorProtocol::OpenTelemetry => {
+                        let attributes = OtelCollectorAttributes {
+                            hostname: inner.hostname,
+                            pod_uid: inner.pod_uid,
+                            container_name: inner.container_name,
+                            service_name: svc_name,
+                            extra: inner.attributes,
+                        };
+                        otel_collector::create_collector(
+                            addr.clone(),
+                            attributes,
+                            svc,
+                            legacy_otel_metrics,
+                        )
+                    }
                 };
 
                 Ok(TraceCollector::Enabled(Box::new(collector)))

linkerd/app/src/trace_collector/otel_collector.rs

@@ -4,20 +4,31 @@ use linkerd_app_core::{
 };
 use linkerd_opentelemetry::{
     self as opentelemetry, metrics,
-    proto::proto::common::v1::{any_value, AnyValue, KeyValue},
-    proto::transform::common::ResourceAttributesWithSchema,
+    proto::{
+        proto::common::v1::{any_value, AnyValue, KeyValue},
+        transform::common::ResourceAttributesWithSchema,
+    },
+};
+use std::{
+    collections::HashMap,
+    time::{SystemTime, UNIX_EPOCH},
 };
-use std::{collections::HashMap, time::SystemTime, time::UNIX_EPOCH};
 use tokio::sync::mpsc;
 use tokio_stream::wrappers::ReceiverStream;
 use tonic::{body::BoxBody, client::GrpcService};
 use tracing::Instrument;
 
+pub(super) struct OtelCollectorAttributes {
+    pub hostname: Option<String>,
+    pub pod_uid: Option<String>,
+    pub container_name: Option<String>,
+    pub service_name: String,
+    pub extra: HashMap<String, String>,
+}
+
 pub(super) fn create_collector<S>(
     addr: ControlAddr,
-    hostname: Option<String>,
-    service_name: String,
-    attributes: HashMap<String, String>,
+    attributes: OtelCollectorAttributes,
     svc: S,
     legacy_metrics: metrics::Registry,
 ) -> EnabledCollector
@@ -36,7 +47,7 @@ where
     resources
         .attributes
         .0
-        .push(service_name.with_key("service.name"));
+        .push(attributes.service_name.with_key("service.name"));
     resources
         .attributes
         .0
@@ -49,13 +60,25 @@ where
             .unwrap_or_else(|e| -(e.duration().as_secs() as i64))
             .with_key("process.start_timestamp"),
     );
-    resources
-        .attributes
-        .0
-        .push(hostname.unwrap_or_default().with_key("host.name"));
+    resources.attributes.0.push(
+        attributes
+            .hostname
+            .unwrap_or_default()
+            .with_key("host.name"),
+    );
+    if let Some(pod_uid) = attributes.pod_uid {
+        resources.attributes.0.push(pod_uid.with_key("k8s.pod.uid"));
+    }
+    if let Some(container_name) = attributes.container_name {
+        resources
+            .attributes
+            .0
+            .push(container_name.with_key("k8s.container.name"));
+    }
 
     resources.attributes.0.extend(
         attributes
+            .extra
             .into_iter()
             .map(|(key, value)| value.with_key(&key)),
     );