Expose Prometheus metrics on a separate port

Metrics are served on METRICS_PORT (default 2112, "off" to disable), never on
the public port. Besides the Go runtime and process metrics this adds:

- http_requests_total and http_request_duration_seconds per route pattern,
  from an Echo middleware.
- musicserver_songs_requested_total and musicserver_songs_played_total by
  pick mode (random, low, classic). The mode is now stored with each queued
  song (migration 000006, not released yet).
- Library size, sync duration, last successful sync, broken songs from the
  last full sync, and the database connection pool.

Refs #35, #36, #37

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/server/routes.go

@@ -6,6 +6,7 @@ import (
 
 	"music-server/cmd/web"
 	"music-server/internal/logging"
+	"music-server/internal/metrics"
 	"music-server/internal/server/middleware"
 
 	"github.com/a-h/templ"
@@ -39,6 +40,8 @@ func (s *Server) routes() (*echo.Echo, huma.API) {
 	}
 
 	e.Use(logging.RequestLogger())
+	// Outside Recover, so requests that panic are counted with status 500.
+	e.Use(metrics.Middleware())
 	e.Use(echoMiddleware.Recover())
 	// Without allowed origins no CORS headers are sent, so only same-origin browser requests work.
 	// The API uses Bearer tokens, not cookies, so credentials are not allowed.

internal/server/routes_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"net"
 	"net/http"
+	"net/http/httptest"
 	"path/filepath"
 	"sync/atomic"
 	"testing"
@@ -223,3 +224,32 @@ func TestCORSWithoutAllowedOrigins(t *testing.T) {
 	assert.Equal(t, http.StatusOK, rec.Code, "the request itself works; the browser blocks reading it")
 	assert.Empty(t, rec.Header().Get("Access-Control-Allow-Origin"))
 }
+
+func TestMetricsAreNotServedOnTheMainPort(t *testing.T) {
+	e := newTestEnv(t, func(c *config.Config) { c.MetricsPort = 2112 })
+	require.NotNil(t, e.server.metrics)
+	for _, path := range []string{"/metrics", "/api/v1/metrics"} {
+		rec := e.do(http.MethodGet, path)
+		assert.Equal(t, http.StatusNotFound, rec.Code, path)
+		assert.NotContains(t, rec.Body.String(), "go_goroutines", path)
+	}
+}
+
+func TestMetricsOff(t *testing.T) {
+	assert.Nil(t, newTestEnv(t).server.metrics, "MetricsPort 0 means no metrics server")
+}
+
+func TestRequestMetricsUseRoutePatterns(t *testing.T) {
+	e := newTestEnv(t, func(c *config.Config) { c.MetricsPort = 2112 })
+	song := e.addSong("Game A", "Song", "x")
+	e.do(http.MethodGet, "/api/v1/songs/"+song.Uuid.String())
+	e.do(http.MethodGet, "/no/such/page?secret=1")
+
+	rec := httptest.NewRecorder()
+	e.server.metrics.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil))
+	body := rec.Body.String()
+	assert.Contains(t, body, `http_requests_total{code="200",method="GET",route="/api/v1/songs/:uuid"}`)
+	assert.NotContains(t, body, song.Uuid.String(), "IDs never become labels")
+	assert.NotContains(t, body, "secret", "query strings never become labels")
+	assert.NotContains(t, body, "/no/such/page", "unknown URLs count under the frontend's /* route")
+}

internal/server/server.go

@@ -12,7 +12,9 @@ import (
 	"music-server/internal/db"
 	"music-server/internal/db/repository"
 	"music-server/internal/logging"
+	"music-server/internal/metrics"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/zap"
 )
 
@@ -34,6 +36,8 @@ type Server struct {
 	stats      *backend.StatisticsService
 	search     *backend.SearchCache
 	httpServer *http.Server
+	// metrics serves /metrics on its own port; nil when metrics are off.
+	metrics *metrics.Server
 }
 
 // New creates a Server with all services wired up.
@@ -56,6 +60,9 @@ func New(cfg config.Config, q repository.Querier, database Database) (*Server, e
 	s.search = backend.NewSearchCache(s.music.SoundtrackNames)
 	s.sync = backend.NewSyncService(q, cfg.MusicPath, s.afterSync)
 
+	if cfg.MetricsPort > 0 {
+		s.metrics = metrics.NewServer(cfg.MetricsPort)
+	}
 	s.httpServer = &http.Server{
 		Addr:         fmt.Sprintf(":%d", cfg.Port),
 		Handler:      s.RegisterRoutes(),
@@ -111,11 +118,21 @@ func NewServerInstance() *Server {
 	if err != nil {
 		logger.Fatal("Failed to create server", zap.Error(err))
 	}
+	prometheus.MustRegister(metrics.NewPoolCollector(database.Pool))
+	if err := s.sync.UpdateLibraryMetrics(ctx); err != nil {
+		logger.Error("Failed to count the library", zap.Error(err))
+	}
 	return s
 }
 
-// ListenAndServe starts the HTTP server. It returns nil after a graceful shutdown.
+// ListenAndServe starts the HTTP server and the metrics server. It returns nil after a
+// graceful shutdown. The server keeps running without metrics if their port cannot be opened.
 func (s *Server) ListenAndServe() error {
+	if s.metrics != nil {
+		if err := s.metrics.Start(); err != nil {
+			logging.GetLogger().Error("Failed to start the metrics server", zap.Error(err))
+		}
+	}
 	logging.GetLogger().Info("Server starting", zap.String("address", s.httpServer.Addr))
 	if err := s.httpServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
 		return err
@@ -126,6 +143,9 @@ func (s *Server) ListenAndServe() error {
 // Shutdown stops accepting requests, waits for running requests and then closes the database.
 func (s *Server) Shutdown(ctx context.Context) error {
 	err := s.httpServer.Shutdown(ctx)
+	if s.metrics != nil {
+		err = errors.Join(err, s.metrics.Shutdown(ctx))
+	}
 	s.sync.Wait()
 	s.characters.Close()
 	s.db.Close()