Expose Prometheus metrics on a separate port

Metrics are served on METRICS_PORT (default 2112, "off" to disable), never on
the public port. Besides the Go runtime and process metrics this adds:

- http_requests_total and http_request_duration_seconds per route pattern,
  from an Echo middleware.
- musicserver_songs_requested_total and musicserver_songs_played_total by
  pick mode (random, low, classic). The mode is now stored with each queued
  song (migration 000006, not released yet).
- Library size, sync duration, last successful sync, broken songs from the
  last full sync, and the database connection pool.

Refs #35, #36, #37

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/backend/library_integration_test.go

@@ -29,6 +29,11 @@ func TestLibraryAndStatisticsWithPostgres(t *testing.T) {
 	s.Wait()
 	require.Empty(t, s.Result().CatchedErrors)
 
+	count, err := q.CountLibrary(ctx)
+	require.NoError(t, err)
+	assert.Equal(t, int64(4), count.Soundtracks)
+	require.NoError(t, s.UpdateLibraryMetrics(ctx))
+
 	library := NewLibraryService(q)
 	soundtracks, err := library.Soundtracks(ctx)
 	require.NoError(t, err)

internal/backend/metrics_test.go

@@ -0,0 +1,108 @@
+package backend
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"music-server/internal/db/repository"
+	"music-server/internal/metrics"
+	"music-server/internal/testutil/fakedb"
+
+	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func requested(mode PickMode) float64 {
+	return testutil.ToFloat64(metrics.SongsRequested.WithLabelValues(string(mode)))
+}
+
+func played(mode PickMode) float64 {
+	return testutil.ToFloat64(metrics.SongsPlayed.WithLabelValues(string(mode)))
+}
+
+func syncCount(syncType string) uint64 {
+	m := &dto.Metric{}
+	_ = metrics.SyncDuration.WithLabelValues(syncType).(prometheus.Metric).Write(m)
+	return m.GetHistogram().GetSampleCount()
+}
+
+func TestSongMetricsByPickMode(t *testing.T) {
+	store := fakedb.New()
+	addSoundtrackWithSongs(t, store, t.TempDir(), "Game A", 0, "Song 1")
+	m := NewMusicService(store, t.TempDir())
+	ctx := context.Background()
+	q := newQueue(t, m)
+	before := map[PickMode][2]float64{}
+	for _, mode := range []PickMode{PickRandom, PickLow, PickClassic} {
+		before[mode] = [2]float64{requested(mode), played(mode)}
+	}
+
+	pick := map[PickMode]func(context.Context, uuid.UUID) (string, error){
+		PickRandom:  func(ctx context.Context, id uuid.UUID) (string, error) { return m.RandomSong(ctx, id) },
+		PickLow:     func(ctx context.Context, id uuid.UUID) (string, error) { return m.RandomSongLowChance(ctx, id) },
+		PickClassic: func(ctx context.Context, id uuid.UUID) (string, error) { return m.RandomSongClassic(ctx, id) },
+	}
+	for songNo, mode := range []PickMode{PickRandom, PickLow, PickClassic} {
+		_, err := pick[mode](ctx, q)
+		require.NoError(t, err)
+		require.NoError(t, m.AddLatestToQueue(ctx, q))
+		require.NoError(t, m.SetPlayed(ctx, q, songNo))
+	}
+	// Fetched but never added: requested, not played.
+	_, err := m.RandomSong(ctx, q)
+	require.NoError(t, err)
+	require.NoError(t, m.AddLatestPlayed(ctx, q), "the current song is the classic one")
+
+	assert.Equal(t, before[PickRandom][0]+2, requested(PickRandom))
+	assert.Equal(t, before[PickLow][0]+1, requested(PickLow))
+	assert.Equal(t, before[PickClassic][0]+1, requested(PickClassic))
+	assert.Equal(t, before[PickRandom][1]+1, played(PickRandom))
+	assert.Equal(t, before[PickLow][1]+1, played(PickLow))
+	assert.Equal(t, before[PickClassic][1]+2, played(PickClassic))
+}
+
+func TestSongsPlayedIgnoresSongsOutsideTheQueue(t *testing.T) {
+	m := NewMusicService(fakedb.New(), t.TempDir())
+	before := played(PickRandom)
+	require.NoError(t, m.SetPlayed(context.Background(), newQueue(t, m), 3))
+	assert.Equal(t, before, played(PickRandom))
+}
+
+func TestSyncMetrics(t *testing.T) {
+	f := newSyncFixture(t)
+	f.write("Game A/01.mp3", "a1")
+	f.write("Game A/02.mp3", "a2")
+	f.write("Game B/01.mp3", "b1")
+	changesBefore, fullBefore := syncCount("changes"), syncCount("full")
+	metrics.SyncLastSuccess.Set(0)
+
+	f.run(false)
+
+	assert.Equal(t, changesBefore+1, syncCount("changes"))
+	assert.Equal(t, 2.0, testutil.ToFloat64(metrics.LibrarySoundtracks))
+	assert.Equal(t, 3.0, testutil.ToFloat64(metrics.LibrarySongs))
+	assert.InDelta(t, float64(time.Now().Unix()), testutil.ToFloat64(metrics.SyncLastSuccess), 5)
+
+	f.store.AddTestSong(repository.Song{SoundtrackID: 999, SongName: "gone", Path: "/does/not/exist.mp3"})
+	f.run(true)
+
+	assert.Equal(t, fullBefore+1, syncCount("full"))
+	assert.Equal(t, 1.0, testutil.ToFloat64(metrics.BrokenSongs))
+	assert.Equal(t, 3.0, testutil.ToFloat64(metrics.LibrarySongs), "the broken song is removed")
+}
+
+func TestSyncWithErrorsIsNotALastSuccess(t *testing.T) {
+	metrics.SyncLastSuccess.Set(0)
+	s := NewSyncService(fakedb.New(), filepath.Join(t.TempDir(), "missing"), nil)
+	require.True(t, s.Start(false))
+	s.Wait()
+	require.NotEmpty(t, s.Result().CatchedErrors)
+
+	assert.Zero(t, testutil.ToFloat64(metrics.SyncLastSuccess))
+}

internal/backend/music.go

@@ -12,6 +12,7 @@ import (
 
 	"music-server/internal/db/repository"
 	"music-server/internal/logging"
+	"music-server/internal/metrics"
 
 	"github.com/google/uuid"
 	"github.com/jackc/pgx/v5"
@@ -21,6 +22,24 @@ import (
 // maxPickAttempts bounds how many broken songs are skipped before giving up.
 const maxPickAttempts = 100
 
+// PickMode is how a random song was picked. It is stored with queued songs and used as
+// the mode label of the song metrics.
+type PickMode string
+
+const (
+	PickRandom  PickMode = "random"
+	PickLow     PickMode = "low"
+	PickClassic PickMode = "classic"
+)
+
+func init() {
+	// Start every mode at 0, so the series exist before the first song.
+	for _, mode := range []PickMode{PickRandom, PickLow, PickClassic} {
+		metrics.SongsRequested.WithLabelValues(string(mode))
+		metrics.SongsPlayed.WithLabelValues(string(mode))
+	}
+}
+
 type SongInfo struct {
 	SoundtrackUuid   uuid.UUID `json:"SoundtrackUuid"`
 	Soundtrack       string    `json:"Soundtrack"`
@@ -145,7 +164,11 @@ func (m *MusicService) markPlayed(ctx context.Context, songs []repository.ListQu
 	if err := m.q.AddSoundtrackPlayedByUUID(ctx, song.SoundtrackUuid); err != nil {
 		return err
 	}
-	return m.q.AddSongPlayedByUUID(ctx, song.SongUuid)
+	if err := m.q.AddSongPlayedByUUID(ctx, song.SongUuid); err != nil {
+		return err
+	}
+	metrics.SongsPlayed.WithLabelValues(song.Mode).Inc()
+	return nil
 }
 
 func (m *MusicService) loadQueue(ctx context.Context, queueID uuid.UUID) (repository.Queue, []repository.ListQueueSongsRow, error) {
@@ -167,7 +190,7 @@ func (m *MusicService) RandomSong(ctx context.Context, queueID uuid.UUID) (strin
 	for i, s := range soundtracks {
 		candidates[i] = weightedSoundtrack{soundtrack: s, weight: 1}
 	}
-	return m.pickAndRemember(ctx, queueID, candidates)
+	return m.pickAndRemember(ctx, queueID, candidates, PickRandom)
 }
 
 // RandomSongLowChance picks a random song, favouring soundtracks played less than average.
@@ -176,23 +199,30 @@ func (m *MusicService) RandomSongLowChance(ctx context.Context, queueID uuid.UUI
 	if err != nil {
 		return "", err
 	}
-	return m.pickAndRemember(ctx, queueID, lowChanceWeights(soundtracks))
+	return m.pickAndRemember(ctx, queueID, lowChanceWeights(soundtracks), PickLow)
 }
 
-func (m *MusicService) pickAndRemember(ctx context.Context, queueID uuid.UUID, candidates []weightedSoundtrack) (string, error) {
+func (m *MusicService) pickAndRemember(
+	ctx context.Context, queueID uuid.UUID, candidates []weightedSoundtrack, mode PickMode,
+) (string, error) {
 	song, err := m.pickSong(ctx, candidates)
 	if err != nil {
 		return "", err
 	}
-	return m.remember(ctx, queueID, song)
+	return m.remember(ctx, queueID, song, mode)
 }
 
-// remember stores the song as the queue's last fetched song, for AddLatestToQueue.
-func (m *MusicService) remember(ctx context.Context, queueID uuid.UUID, song repository.Song) (string, error) {
-	err := m.q.SetQueueLastFetched(ctx, repository.SetQueueLastFetchedParams{Uuid: queueID, LastFetchedSongUuid: song.Uuid})
+// remember stores the song and how it was picked as the queue's last fetched song,
+// for AddLatestToQueue.
+func (m *MusicService) remember(ctx context.Context, queueID uuid.UUID, song repository.Song, mode PickMode) (string, error) {
+	modeName := string(mode)
+	err := m.q.SetQueueLastFetched(ctx, repository.SetQueueLastFetchedParams{
+		Uuid: queueID, LastFetchedSongUuid: song.Uuid, LastFetchedMode: &modeName,
+	})
 	if err != nil {
 		return "", err
 	}
+	metrics.SongsRequested.WithLabelValues(modeName).Inc()
 	return song.Path, nil
 }
 
@@ -212,7 +242,7 @@ func (m *MusicService) RandomSongClassic(ctx context.Context, queueID uuid.UUID)
 			}
 			continue
 		}
-		return m.remember(ctx, queueID, row.Song)
+		return m.remember(ctx, queueID, row.Song, PickClassic)
 	}
 	return "", ErrNoSongs
 }

internal/backend/queue_integration_test.go

@@ -47,7 +47,31 @@ func TestQueueWithPostgres(t *testing.T) {
 		assert.NotEqual(t, uuid.Nil, song.SoundtrackUuid)
 	}
 
-	path, err := m.PreviousSong(ctx, queueID)
+	var modes []string
+	rows, err := database.Pool.Query(ctx, "SELECT mode FROM queue_song WHERE queue_uuid = $1 ORDER BY song_no", queueID)
+	require.NoError(t, err)
+	for rows.Next() {
+		var mode string
+		require.NoError(t, rows.Scan(&mode))
+		modes = append(modes, mode)
+	}
+	require.NoError(t, rows.Err())
+	assert.Equal(t, []string{"random", "random", "random"}, modes)
+
+	_, err = m.RandomSongClassic(ctx, queueID)
+	require.NoError(t, err)
+	require.NoError(t, m.AddLatestToQueue(ctx, queueID))
+	var classic string
+	require.NoError(t, database.Pool.QueryRow(ctx, "SELECT mode FROM queue_song WHERE queue_uuid = $1 AND song_no = 3", queueID).Scan(&classic))
+	assert.Equal(t, "classic", classic)
+	played, err = m.PlayedSongs(ctx, queueID)
+	require.NoError(t, err)
+	require.Len(t, played, 4)
+
+	path, err := m.Song(ctx, queueID, 2)
+	require.NoError(t, err)
+	assert.Equal(t, paths[2], path)
+	path, err = m.PreviousSong(ctx, queueID)
 	require.NoError(t, err)
 	assert.Equal(t, paths[1], path)
 	require.NoError(t, m.AddLatestPlayed(ctx, queueID))
@@ -82,7 +106,7 @@ func TestQueueWithPostgres(t *testing.T) {
 	require.NoError(t, q.ClearSongs(ctx))
 	played, err = m.PlayedSongs(ctx, queueID)
 	require.NoError(t, err)
-	require.Len(t, played, 3)
+	require.Len(t, played, 4)
 	assert.NotEmpty(t, played[0].Song)
 	_, err = m.Song(ctx, queueID, 0)
 	assert.ErrorIs(t, err, ErrNotFound)

internal/backend/sync.go

@@ -19,6 +19,7 @@ import (
 
 	"music-server/internal/db/repository"
 	"music-server/internal/logging"
+	"music-server/internal/metrics"
 
 	"github.com/MShekow/directory-checksum/directory_checksum"
 	"github.com/google/uuid"
@@ -199,6 +200,7 @@ func (s *SyncService) run(ctx context.Context, full bool) {
 	run := &syncRun{q: s.q, musicPath: s.musicPath, full: full, progress: func() { s.foldersSynced.Add(1) }}
 	result := run.execute(ctx, func(total int) { s.foldersTotal.Store(int64(total)) })
 	result.TotalTime = formatDuration(time.Since(start))
+	s.recordMetrics(ctx, full, time.Since(start), len(result.CatchedErrors) == 0)
 
 	if s.onComplete != nil {
 		s.onComplete(ctx)
@@ -218,6 +220,42 @@ func (s *SyncService) run(ctx context.Context, full bool) {
 		zap.Int("errors", len(result.CatchedErrors)))
 }
 
+func init() {
+	// Start both sync types at 0, so the series exist before the first sync.
+	metrics.SyncDuration.WithLabelValues(syncType(false))
+	metrics.SyncDuration.WithLabelValues(syncType(true))
+}
+
+// syncType is the type label of the sync metrics.
+func syncType(full bool) string {
+	if full {
+		return "full"
+	}
+	return "changes"
+}
+
+// recordMetrics updates the sync and library metrics after a sync.
+func (s *SyncService) recordMetrics(ctx context.Context, full bool, took time.Duration, success bool) {
+	metrics.SyncDuration.WithLabelValues(syncType(full)).Observe(took.Seconds())
+	if success {
+		metrics.SyncLastSuccess.SetToCurrentTime()
+	}
+	if err := s.UpdateLibraryMetrics(ctx); err != nil {
+		logging.GetLogger().Error("Failed to count the library", zap.Error(err))
+	}
+}
+
+// UpdateLibraryMetrics sets the library size metrics from the database.
+func (s *SyncService) UpdateLibraryMetrics(ctx context.Context) error {
+	count, err := s.q.CountLibrary(ctx)
+	if err != nil {
+		return err
+	}
+	metrics.LibrarySoundtracks.Set(float64(count.Soundtracks))
+	metrics.LibrarySongs.Set(float64(count.Songs))
+	return nil
+}
+
 func (r *syncRun) execute(ctx context.Context, setTotal func(int)) SyncResponse {
 	var err error
 	r.existing, err = r.q.GetAllSoundtracksIncludingDeleted(ctx)
@@ -623,6 +661,7 @@ func (r *syncRun) removeBrokenSongs(ctx context.Context) {
 	}
 	_ = g.Wait()
 
+	metrics.BrokenSongs.Set(float64(len(broken)))
 	if len(broken) > 0 {
 		if err := r.q.RemoveBrokenSongs(ctx, broken); err != nil {
 			r.addError("RemoveBrokenSongs", err, "")