gRPC Monitoring Guide: Metrics, Tracing & Observability (2026)

// go get github.com/grpc-ecosystem/go-grpc-prometheus

package main

import (
    grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
    "google.golang.org/grpc"
    "github.com/prometheus/client_golang/prometheus/promhttp"
    "net/http"
)

func main() {
    // Server setup with Prometheus interceptors
    srv := grpc.NewServer(
        grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor),
        grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor),
    )

    // Register your gRPC services
    pb.RegisterMyServiceServer(srv, &MyService{})

    // Enable default metrics with histograms for latency
    grpc_prometheus.EnableHandlingTimeHistogram()

    // Initialize server metrics after registering services
    grpc_prometheus.Register(srv)

    // Expose Prometheus metrics on /metrics
    go func() {
        http.Handle("/metrics", promhttp.Handler())
        http.ListenAndServe(":9090", nil)
    }()

    // Serve gRPC on :50051
    lis, _ := net.Listen("tcp", ":50051")
    srv.Serve(lis)
}

// Client setup
conn, _ := grpc.Dial(
    "server:50051",
    grpc.WithUnaryInterceptor(grpc_prometheus.UnaryClientInterceptor),
    grpc.WithStreamInterceptor(grpc_prometheus.StreamClientInterceptor),
)

// This emits:
// grpc_server_handled_total{grpc_code, grpc_method, grpc_service, grpc_type}
// grpc_server_handling_seconds{grpc_method, grpc_service, grpc_type} histogram
// grpc_server_msg_received_total  — for streaming
// grpc_server_msg_sent_total

// Key alert rules:
// High error rate by status code:
// rate(grpc_server_handled_total{grpc_code!="OK"}[5m])
//   / rate(grpc_server_handled_total[5m]) > 0.01

// UNAVAILABLE spike (server down):
// increase(grpc_server_handled_total{grpc_code="UNAVAILABLE"}[1m]) > 10

// Latency regression:
// histogram_quantile(0.99, rate(grpc_server_handling_seconds_bucket[5m])) > 1.0

// Go: otelgrpc interceptors
// go get go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc

import "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"

// Server
srv := grpc.NewServer(
    grpc.StatsHandler(otelgrpc.NewServerHandler()),
)

// Client
conn, _ := grpc.Dial(
    "downstream:50051",
    grpc.WithStatsHandler(otelgrpc.NewClientHandler()),
)

// The interceptors automatically:
// 1. Extract W3C Trace Context from incoming metadata (traceparent header)
// 2. Create a span for each RPC with rpc.method, rpc.service, rpc.grpc.status_code
// 3. Inject trace context into outgoing client calls
// 4. Record errors and status codes on spans

// Python: auto-instrumentation
# pip install opentelemetry-instrumentation-grpc
from opentelemetry.instrumentation.grpc import GrpcInstrumentorServer, GrpcInstrumentorClient

GrpcInstrumentorServer().instrument()   # instrument all grpc.server instances
GrpcInstrumentorClient().instrument()   # instrument all grpc.channel instances

# Java: OpenTelemetry agent (zero code change)
# java -javaagent:opentelemetry-javaagent.jar \
#   -Dotel.service.name=my-grpc-service \
#   -Dotel.exporter.otlp.endpoint=http://collector:4317 \
#   -jar my-service.jar

# What you see in traces:
# - Full RPC call tree across microservices
# - Exact method that failed (rpc.method="OrderService/PlaceOrder")
# - gRPC status code as span attribute
# - Time breakdown: client send, server process, server send, client receive

# Test health check manually
grpc-health-probe -addr=:50051

# Kubernetes deployment with gRPC health probes
spec:
  containers:
  - name: grpc-service
    livenessProbe:
      exec:
        command: ["/bin/grpc_health_probe", "-addr=:50051"]
      initialDelaySeconds: 10
      periodSeconds: 30
    readinessProbe:
      exec:
        command: ["/bin/grpc_health_probe", "-addr=:50051", "-service=MyService"]
      initialDelaySeconds: 5
      periodSeconds: 10

# Kubernetes 1.24+ supports native gRPC probes:
livenessProbe:
  grpc:
    port: 50051
    service: ""        # empty = overall server health

# Go: implement the health server
import "google.golang.org/grpc/health"
import healthpb "google.golang.org/grpc/health/grpc_health_v1"

healthServer := health.NewServer()
healthpb.RegisterHealthServer(srv, healthServer)

// Mark service healthy/unhealthy based on dependency checks
healthServer.SetServingStatus("MyService", healthpb.HealthCheckResponse_SERVING)
healthServer.SetServingStatus("MyService", healthpb.HealthCheckResponse_NOT_SERVING)