第18章 eBPF 与可观测性¶

# 安装 Cilium（Helm）
# helm repo add cilium https://helm.cilium.io/
# helm install cilium cilium/cilium --namespace kube-system \
#   --set hubble.relay.enabled=true \
#   --set hubble.ui.enabled=true

# Hubble 网络可观测性 — 查看实时流量
# hubble observe --namespace default
# hubble observe --protocol http --verdict DROPPED

# Cilium NetworkPolicy 示例
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
  name: allow-frontend-to-api
spec:
  endpointSelector:
    matchLabels:
      app: api-server
  ingress:
    - fromEndpoints:
        - matchLabels:
            app: frontend
      toPorts:
        - ports:
            - port: "8080"
              protocol: TCP
          rules:
            http:
              - method: "GET"
                path: "/api/v1/.*"

# === Falco：运行时安全检测 ===
# Falco 使用 eBPF 探针监控系统调用，匹配安全规则

# 自定义告警规则
- rule: Unexpected Outbound Connection
  desc: Detect unexpected outbound network connections from containers
  condition: >
    evt.type=connect and
    evt.dir=< and
    container and
    not (fd.sport in (80, 443, 53)) and
    not k8s.ns.name in (kube-system, monitoring)
  output: >
    Unexpected outbound connection
    (command=%proc.cmdline connection=%fd.name
     container=%container.name namespace=%k8s.ns.name
     pod=%k8s.pod.name image=%container.image.repository)
  priority: WARNING
  tags: [network, mitre_command_and_control]

# === Tetragon：eBPF 原生安全可观测 ===
# Tetragon 是 Cilium 旗下的安全可观测性工具
apiVersion: cilium.io/v1alpha1
kind: TracingPolicy
metadata:
  name: monitor-sensitive-files
spec:
  kprobes:
    - call: fd_install
      syscall: false
      args:
        - index: 0
          type: int
        - index: 1
          type: "file"
      selectors:
        - matchArgs:
            - index: 1
              operator: Prefix
              values:
                - /etc/shadow
                - /etc/passwd
                - /etc/kubernetes/
          matchActions:
            - action: Sigkill  # 直接终止进程

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

  # 从 Prometheus 端点抓取指标
  prometheus:
    config:
      scrape_configs:
        - job_name: "kubernetes-pods"
          kubernetes_sd_configs:
            - role: pod
          relabel_configs:
            - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
              action: keep
              regex: true

  # 收集 Kubernetes 事件
  k8s_events:
    namespaces: [default, production]

processors:
  # 批处理：减少网络调用
  batch:
    send_batch_size: 8192
    timeout: 200ms

  # 内存限制：防止 OOM
  memory_limiter:
    check_interval: 1s
    limit_mib: 1024
    spike_limit_mib: 256

  # 属性处理
  attributes:
    actions:
      - key: environment
        value: production
        action: upsert
      - key: db.password
        action: delete  # 移除敏感信息

  # 尾部采样（基于完整 trace 决策）
  tail_sampling:
    decision_wait: 10s
    policies:
      - name: error-traces
        type: status_code
        status_code: { status_codes: [ERROR] }
      - name: slow-traces
        type: latency
        latency: { threshold_ms: 1000 }
      - name: sample-rest
        type: probabilistic
        probabilistic: { sampling_percentage: 10 }

exporters:
  # 导出到 Prometheus
  prometheusremotewrite:
    endpoint: "http://prometheus:9090/api/v1/write"

  # 导出到 Jaeger
  otlp/jaeger:
    endpoint: "jaeger-collector:4317"
    tls:
      insecure: true

  # 导出到 Loki（日志）
  loki:
    endpoint: "http://loki:3100/loki/api/v1/push"

  # 导出到 Grafana Tempo（Traces）
  otlp/tempo:
    endpoint: "tempo:4317"
    tls:
      insecure: true

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, tail_sampling, batch]
      exporters: [otlp/tempo, otlp/jaeger]
    metrics:
      receivers: [otlp, prometheus]
      processors: [memory_limiter, attributes, batch]
      exporters: [prometheusremotewrite]
    logs:
      receivers: [otlp, k8s_events]
      processors: [memory_limiter, attributes, batch]
      exporters: [loki]

# === Kubernetes Operator 方式（推荐） ===
# 安装 OpenTelemetry Operator
# kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/latest/download/opentelemetry-operator.yaml

# 创建 Instrumentation 资源
apiVersion: opentelemetry.io/v1alpha1
kind: Instrumentation
metadata:
  name: auto-instrumentation
  namespace: default
spec:
  exporter:
    endpoint: http://otel-collector:4317
  propagators:
    - tracecontext
    - baggage
    - b3
  sampler:
    type: parentbased_traceidratio
    argument: "0.25"  # 25% 采样率

  python:
    image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-python:latest
    env:
      - name: OTEL_PYTHON_LOG_CORRELATION
        value: "true"

  java:
    image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-java:latest
    env:
      - name: OTEL_INSTRUMENTATION_JDBC_ENABLED
        value: "true"

  nodejs:
    image: ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-nodejs:latest
---  # YAML文档分隔符
# 在 Deployment 上添加注解即可自动注入
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-python-app
spec:
  template:
    metadata:
      annotations:
        instrumentation.opentelemetry.io/inject-python: "true"
    spec:
      containers:
        - name: app
          image: my-python-app:latest

# === 完整的 Kubernetes 可观测性栈部署（Helm） ===

# 1. 部署 Prometheus + Grafana（kube-prometheus-stack）
# helm install prometheus prometheus-community/kube-prometheus-stack \
#   --namespace monitoring --create-namespace \
#   --set grafana.enabled=true

# 2. 部署 Grafana Tempo（分布式追踪）
# helm install tempo grafana/tempo-distributed \
#   --namespace monitoring

# 3. 部署 Grafana Loki（日志聚合）
# helm install loki grafana/loki-stack \
#   --namespace monitoring \
#   --set promtail.enabled=true

# 4. 部署 OpenTelemetry Collector（DaemonSet 模式）
apiVersion: opentelemetry.io/v1beta1  # apiVersion指定K8s API版本
kind: OpenTelemetryCollector  # kind指定资源类型
metadata:
  name: otel-collector
  namespace: monitoring
spec:  # spec定义资源的期望状态
  mode: daemonset
  config:
    receivers:
      otlp:
        protocols:
          grpc:
            endpoint: 0.0.0.0:4317
      kubeletstats:
        collection_interval: 30s
        auth_type: serviceAccount
        endpoint: "https://${env:K8S_NODE_NAME}:10250"
        insecure_skip_verify: true

    processors:
      k8sattributes:
        extract:
          metadata:
            - k8s.pod.name
            - k8s.namespace.name
            - k8s.deployment.name
            - k8s.node.name
        pod_association:
          - sources:
              - from: resource_attribute
                name: k8s.pod.ip
      batch:
        send_batch_size: 4096
        timeout: 5s

    exporters:
      prometheusremotewrite:
        endpoint: "http://prometheus-kube-prometheus-prometheus:9090/api/v1/write"
      otlp/tempo:
        endpoint: "tempo-distributor:4317"
        tls:
          insecure: true
      loki:
        endpoint: "http://loki:3100/loki/api/v1/push"

    service:
      pipelines:
        traces:
          receivers: [otlp]
          processors: [k8sattributes, batch]
          exporters: [otlp/tempo]
        metrics:
          receivers: [otlp, kubeletstats]
          processors: [k8sattributes, batch]
          exporters: [prometheusremotewrite]
        logs:
          receivers: [otlp]
          processors: [k8sattributes, batch]
          exporters: [loki]