Merge pull request containerd#9845 from dcantah/prometheus-middleware-deprecated

Replace go-grpc-prometheus with go-grpc-middleware/providers/prometheus

Author: estesp

cmd/containerd/server/server.go

@@ -37,9 +37,9 @@ import (
 	"github.com/containerd/log"
 	"github.com/containerd/ttrpc"
 	"github.com/docker/go-metrics"
-	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
-	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
+	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/prometheus/client_golang/prometheus"
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/backoff"
@@ -144,16 +144,25 @@ func New(ctx context.Context, config *srvconfig.Config) (*Server, error) {
 		diff.RegisterProcessor(diff.BinaryHandler(id, p.Returns, p.Accepts, p.Path, p.Args, p.Env))
 	}
 
+	var prometheusServerMetricsOpts []grpc_prometheus.ServerMetricsOption
+	if config.Metrics.GRPCHistogram {
+		// Enable grpc time histograms to measure rpc latencies
+		prometheusServerMetricsOpts = append(prometheusServerMetricsOpts, grpc_prometheus.WithServerHandlingTimeHistogram())
+	}
+
+	prometheusServerMetrics := grpc_prometheus.NewServerMetrics(prometheusServerMetricsOpts...)
+	prometheus.MustRegister(prometheusServerMetrics)
+
 	serverOpts := []grpc.ServerOption{
 		grpc.StatsHandler(otelgrpc.NewServerHandler()),
-		grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(
-			grpc_prometheus.StreamServerInterceptor,
+		grpc.ChainStreamInterceptor(
 			streamNamespaceInterceptor,
-		)),
-		grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
-			grpc_prometheus.UnaryServerInterceptor,
+			prometheusServerMetrics.StreamServerInterceptor(),
+		),
+		grpc.ChainUnaryInterceptor(
 			unaryNamespaceInterceptor,
-		)),
+			prometheusServerMetrics.UnaryServerInterceptor(),
+		),
 	}
 	if config.GRPC.MaxRecvMsgSize > 0 {
 		serverOpts = append(serverOpts, grpc.MaxRecvMsgSize(config.GRPC.MaxRecvMsgSize))
@@ -213,10 +222,11 @@ func New(ctx context.Context, config *srvconfig.Config) (*Server, error) {
 		ttrpcServices []ttrpcService
 
 		s = &Server{
-			grpcServer:  grpcServer,
-			tcpServer:   tcpServer,
-			ttrpcServer: ttrpcServer,
-			config:      config,
+			prometheusServerMetrics: prometheusServerMetrics,
+			grpcServer:              grpcServer,
+			tcpServer:               tcpServer,
+			ttrpcServer:             ttrpcServer,
+			config:                  config,
 		}
 		initialized = plugin.NewPluginSet()
 		required    = make(map[string]struct{})
@@ -364,24 +374,18 @@ func recordConfigDeprecations(ctx context.Context, config *srvconfig.Config, set
 
 // Server is the containerd main daemon
 type Server struct {
-	grpcServer  *grpc.Server
-	ttrpcServer *ttrpc.Server
-	tcpServer   *grpc.Server
-	config      *srvconfig.Config
-	plugins     []*plugin.Plugin
-	ready       sync.WaitGroup
+	prometheusServerMetrics *grpc_prometheus.ServerMetrics
+	grpcServer              *grpc.Server
+	ttrpcServer             *ttrpc.Server
+	tcpServer               *grpc.Server
+	config                  *srvconfig.Config
+	plugins                 []*plugin.Plugin
+	ready                   sync.WaitGroup
 }
 
 // ServeGRPC provides the containerd grpc APIs on the provided listener
 func (s *Server) ServeGRPC(l net.Listener) error {
-	if s.config.Metrics.GRPCHistogram {
-		// enable grpc time histograms to measure rpc latencies
-		grpc_prometheus.EnableHandlingTimeHistogram()
-	}
-	// before we start serving the grpc API register the grpc_prometheus metrics
-	// handler.  This needs to be the last service registered so that it can collect
-	// metrics for every other service
-	grpc_prometheus.Register(s.grpcServer)
+	s.prometheusServerMetrics.InitializeMetrics(s.grpcServer)
 	return trapClosedConnErr(s.grpcServer.Serve(l))
 }
 
@@ -403,7 +407,7 @@ func (s *Server) ServeMetrics(l net.Listener) error {
 
 // ServeTCP allows services to serve over tcp
 func (s *Server) ServeTCP(l net.Listener) error {
-	grpc_prometheus.Register(s.tcpServer)
+	s.prometheusServerMetrics.InitializeMetrics(s.tcpServer)
 	return trapClosedConnErr(s.tcpServer.Serve(l))
 }
 

go.mod

@@ -33,8 +33,7 @@ require (
 	github.com/fsnotify/fsnotify v1.7.0
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
-	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
-	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
+	github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.0
 	github.com/intel/goresctrl v0.6.0
 	github.com/klauspost/compress v1.17.6
 	github.com/minio/sha256-simd v1.0.1
@@ -98,6 +97,7 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.3 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect