Expose Prometheus metrics on a separate port

Metrics are served on METRICS_PORT (default 2112, "off" to disable), never on
the public port. Besides the Go runtime and process metrics this adds:

- http_requests_total and http_request_duration_seconds per route pattern,
  from an Echo middleware.
- musicserver_songs_requested_total and musicserver_songs_played_total by
  pick mode (random, low, classic). The mode is now stored with each queued
  song (migration 000006, not released yet).
- Library size, sync duration, last successful sync, broken songs from the
  last full sync, and the database connection pool.

Refs #35, #36, #37

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Dockerfile

@@ -30,6 +30,8 @@ COPY ./songs/ ./songs/
 # The music volume must be readable by uid 1000, and writable for the .id file cleanup during sync.
 USER 1000:1000
 EXPOSE 8080
+# Prometheus metrics (METRICS_PORT). Keep it off the public proxy.
+EXPOSE 2112
 VOLUME /sorted
 VOLUME /characters
 

go.mod

@@ -1,6 +1,6 @@
 module music-server
 
-go 1.25.0
+go 1.26.0
 
 require (
 	github.com/MShekow/directory-checksum v1.4.18
@@ -11,7 +11,7 @@ require (
 	github.com/labstack/echo/v5 v5.3.0
 	github.com/lib/pq v1.12.3 // indirect
 	github.com/spf13/afero v1.15.0
-	github.com/stretchr/testify v1.11.1
+	github.com/stretchr/testify v1.12.1
 	github.com/testcontainers/testcontainers-go v0.42.0
 	go.uber.org/zap v1.28.0
 )
@@ -20,7 +20,9 @@ require (
 	github.com/danielgtaylor/huma/v2 v2.39.1
 	github.com/getkin/kin-openapi v0.149.0
 	github.com/google/uuid v1.6.0
-	golang.org/x/sync v0.22.0
+	github.com/prometheus/client_golang v1.25.0
+	github.com/prometheus/client_model v0.6.3
+	golang.org/x/sync v0.23.0
 )
 
 require (
@@ -29,6 +31,7 @@ require (
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/a-h/parse v0.0.0-20250122154542-74294addb73e // indirect
 	github.com/andybalholm/brotli v1.2.2 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cli/browser v1.3.0 // indirect
@@ -37,7 +40,6 @@ require (
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/containerd/platforms v0.2.1 // indirect
 	github.com/cpuguy83/dockercfg v0.3.2 // indirect
-	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/distribution/reference v0.6.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/ebitengine/purego v0.10.0 // indirect
@@ -55,7 +57,8 @@ require (
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
-	github.com/klauspost/compress v1.19.0 // indirect
+	github.com/klauspost/compress v1.20.1 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/labstack/echo/v4 v4.15.4 // indirect
 	github.com/labstack/gommon v0.5.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
@@ -71,13 +74,15 @@ require (
 	github.com/moby/sys/user v0.4.0 // indirect
 	github.com/moby/sys/userns v0.1.0 // indirect
 	github.com/moby/term v0.5.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/natefinch/atomic v1.0.1 // indirect
 	github.com/oasdiff/yaml v0.1.1 // indirect
 	github.com/oasdiff/yaml3 v0.0.14 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.1 // indirect
-	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
+	github.com/prometheus/common v0.72.0 // indirect
+	github.com/prometheus/procfs v0.22.0 // indirect
 	github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 // indirect
 	github.com/shirou/gopsutil/v4 v4.26.3 // indirect
 	github.com/sirupsen/logrus v1.9.4 // indirect
@@ -92,14 +97,15 @@ require (
 	go.opentelemetry.io/otel/metric v1.41.0 // indirect
 	go.opentelemetry.io/otel/trace v1.41.0 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
-	golang.org/x/crypto v0.54.0 // indirect
+	go.yaml.in/yaml/v3 v3.0.5 // indirect
-	golang.org/x/mod v0.37.0 // indirect
+	golang.org/x/crypto v0.57.0 // indirect
-	golang.org/x/net v0.57.0 // indirect
+	golang.org/x/mod v0.41.0 // indirect
-	golang.org/x/sys v0.47.0 // indirect
+	golang.org/x/net v0.59.0 // indirect
-	golang.org/x/text v0.40.0 // indirect
+	golang.org/x/sys v0.48.0 // indirect
+	golang.org/x/text v0.42.0 // indirect
 	golang.org/x/time v0.15.0 // indirect
-	golang.org/x/tools v0.47.0 // indirect
+	golang.org/x/tools v0.49.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
+	google.golang.org/protobuf v1.36.12 // indirect
 )
 
 tool github.com/a-h/templ/cmd/templ

go.sum

@@ -14,6 +14,8 @@ github.com/a-h/templ v0.3.1020 h1:ypAT/L5ySWEnZ6Zft/5yfoWXYYkhFNvEFOeeqecg4tw=
 github.com/a-h/templ v0.3.1020/go.mod h1:A2DlK61v+K+NRoGnhmYbNYVmtYHcFO5/AisMvBdDxTM=
 github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM=
 github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -35,8 +37,6 @@ github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfv
 github.com/danielgtaylor/huma/v2 v2.39.1 h1:0kwF4ltQoYZ+IU55VPy+BcGekzgF44R64daTGde1H+g=
 github.com/danielgtaylor/huma/v2 v2.39.1/go.mod h1:zcnQ38duIJ3VUHwFaBoZ6x8T+KN/mr33oyqxcj0HTug=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
-github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4=
 github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU=
 github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
@@ -99,12 +99,14 @@ github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
 github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
 github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
 github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
-github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ=
+github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
-github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
+github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/labstack/echo/v4 v4.15.4 h1:DL45vVYa+BWE+XuW+zZNd9H0YEdZ80UAWJGcTVW4EVs=
 github.com/labstack/echo/v4 v4.15.4/go.mod h1:CuMetKIRwsuO/qlAgMq+KTAalwGoB/h4tC+yPdrTj1g=
 github.com/labstack/echo/v5 v5.3.0 h1:KT74Mprk053PQEHwSZdeCDIz1BigTZOZhavMD0c9Fjs=
@@ -141,6 +143,8 @@ github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ=
 github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A=
 github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM=
 github.com/oasdiff/yaml v0.1.1 h1:6nHx+pn9gBRM6YpBlFZFQGCCd1nuvqOBtTD3KKTgGxY=
@@ -154,10 +158,16 @@ github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgr
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
-github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU=
 github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/prometheus/client_golang v1.25.0 h1:cDtW1TcFsMi1RJc8khwCKB4VhzwV04aKqbYtNOXuiK0=
+github.com/prometheus/client_golang v1.25.0/go.mod h1:j2X/4/kaQHLPnmyEU8zjmkndo1T317VpCIQEml8LTKk=
+github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo=
+github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM=
+github.com/prometheus/common v0.72.0 h1:tAYsE+sPJxIncDAobm4H5aQjmox9ZxEIIqPbiffa8G4=
+github.com/prometheus/common v0.72.0/go.mod h1:77NWqAQ2tXT7BIK40qjJdw5Acrsrg1TlHAnsQi3i6mk=
+github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics=
+github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
 github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28=
@@ -173,8 +183,8 @@ github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4=
 github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
-github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
 github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY=
 github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30=
 github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA=
@@ -211,36 +221,38 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
 go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo=
 go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q=
-go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
-go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
-golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
+go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
-golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
+go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
-golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
+golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
-golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
+golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
-golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
+golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
-golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
+golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
-golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
+golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
-golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
+golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
+golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
-golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
-golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
+golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
-golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
+golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
-golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
+golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
-golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
+golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
 golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
 golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
-golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
-golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
+golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
+google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
 gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
 pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk=

internal/backend/library_integration_test.go

@@ -29,6 +29,11 @@ func TestLibraryAndStatisticsWithPostgres(t *testing.T) {
 	s.Wait()
 	require.Empty(t, s.Result().CatchedErrors)
 
+	count, err := q.CountLibrary(ctx)
+	require.NoError(t, err)
+	assert.Equal(t, int64(4), count.Soundtracks)
+	require.NoError(t, s.UpdateLibraryMetrics(ctx))
+
 	library := NewLibraryService(q)
 	soundtracks, err := library.Soundtracks(ctx)
 	require.NoError(t, err)

internal/backend/metrics_test.go

@@ -0,0 +1,108 @@
+package backend
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"music-server/internal/db/repository"
+	"music-server/internal/metrics"
+	"music-server/internal/testutil/fakedb"
+
+	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func requested(mode PickMode) float64 {
+	return testutil.ToFloat64(metrics.SongsRequested.WithLabelValues(string(mode)))
+}
+
+func played(mode PickMode) float64 {
+	return testutil.ToFloat64(metrics.SongsPlayed.WithLabelValues(string(mode)))
+}
+
+func syncCount(syncType string) uint64 {
+	m := &dto.Metric{}
+	_ = metrics.SyncDuration.WithLabelValues(syncType).(prometheus.Metric).Write(m)
+	return m.GetHistogram().GetSampleCount()
+}
+
+func TestSongMetricsByPickMode(t *testing.T) {
+	store := fakedb.New()
+	addSoundtrackWithSongs(t, store, t.TempDir(), "Game A", 0, "Song 1")
+	m := NewMusicService(store, t.TempDir())
+	ctx := context.Background()
+	q := newQueue(t, m)
+	before := map[PickMode][2]float64{}
+	for _, mode := range []PickMode{PickRandom, PickLow, PickClassic} {
+		before[mode] = [2]float64{requested(mode), played(mode)}
+	}
+
+	pick := map[PickMode]func(context.Context, uuid.UUID) (string, error){
+		PickRandom:  func(ctx context.Context, id uuid.UUID) (string, error) { return m.RandomSong(ctx, id) },
+		PickLow:     func(ctx context.Context, id uuid.UUID) (string, error) { return m.RandomSongLowChance(ctx, id) },
+		PickClassic: func(ctx context.Context, id uuid.UUID) (string, error) { return m.RandomSongClassic(ctx, id) },
+	}
+	for songNo, mode := range []PickMode{PickRandom, PickLow, PickClassic} {
+		_, err := pick[mode](ctx, q)
+		require.NoError(t, err)
+		require.NoError(t, m.AddLatestToQueue(ctx, q))
+		require.NoError(t, m.SetPlayed(ctx, q, songNo))
+	}
+	// Fetched but never added: requested, not played.
+	_, err := m.RandomSong(ctx, q)
+	require.NoError(t, err)
+	require.NoError(t, m.AddLatestPlayed(ctx, q), "the current song is the classic one")
+
+	assert.Equal(t, before[PickRandom][0]+2, requested(PickRandom))
+	assert.Equal(t, before[PickLow][0]+1, requested(PickLow))
+	assert.Equal(t, before[PickClassic][0]+1, requested(PickClassic))
+	assert.Equal(t, before[PickRandom][1]+1, played(PickRandom))
+	assert.Equal(t, before[PickLow][1]+1, played(PickLow))
+	assert.Equal(t, before[PickClassic][1]+2, played(PickClassic))
+}
+
+func TestSongsPlayedIgnoresSongsOutsideTheQueue(t *testing.T) {
+	m := NewMusicService(fakedb.New(), t.TempDir())
+	before := played(PickRandom)
+	require.NoError(t, m.SetPlayed(context.Background(), newQueue(t, m), 3))
+	assert.Equal(t, before, played(PickRandom))
+}
+
+func TestSyncMetrics(t *testing.T) {
+	f := newSyncFixture(t)
+	f.write("Game A/01.mp3", "a1")
+	f.write("Game A/02.mp3", "a2")
+	f.write("Game B/01.mp3", "b1")
+	changesBefore, fullBefore := syncCount("changes"), syncCount("full")
+	metrics.SyncLastSuccess.Set(0)
+
+	f.run(false)
+
+	assert.Equal(t, changesBefore+1, syncCount("changes"))
+	assert.Equal(t, 2.0, testutil.ToFloat64(metrics.LibrarySoundtracks))
+	assert.Equal(t, 3.0, testutil.ToFloat64(metrics.LibrarySongs))
+	assert.InDelta(t, float64(time.Now().Unix()), testutil.ToFloat64(metrics.SyncLastSuccess), 5)
+
+	f.store.AddTestSong(repository.Song{SoundtrackID: 999, SongName: "gone", Path: "/does/not/exist.mp3"})
+	f.run(true)
+
+	assert.Equal(t, fullBefore+1, syncCount("full"))
+	assert.Equal(t, 1.0, testutil.ToFloat64(metrics.BrokenSongs))
+	assert.Equal(t, 3.0, testutil.ToFloat64(metrics.LibrarySongs), "the broken song is removed")
+}
+
+func TestSyncWithErrorsIsNotALastSuccess(t *testing.T) {
+	metrics.SyncLastSuccess.Set(0)
+	s := NewSyncService(fakedb.New(), filepath.Join(t.TempDir(), "missing"), nil)
+	require.True(t, s.Start(false))
+	s.Wait()
+	require.NotEmpty(t, s.Result().CatchedErrors)
+
+	assert.Zero(t, testutil.ToFloat64(metrics.SyncLastSuccess))
+}

internal/backend/music.go

@@ -12,6 +12,7 @@ import (
 
 	"music-server/internal/db/repository"
 	"music-server/internal/logging"
+	"music-server/internal/metrics"
 
 	"github.com/google/uuid"
 	"github.com/jackc/pgx/v5"
@@ -21,6 +22,24 @@ import (
 // maxPickAttempts bounds how many broken songs are skipped before giving up.
 const maxPickAttempts = 100
 
+// PickMode is how a random song was picked. It is stored with queued songs and used as
+// the mode label of the song metrics.
+type PickMode string
+
+const (
+	PickRandom  PickMode = "random"
+	PickLow     PickMode = "low"
+	PickClassic PickMode = "classic"
+)
+
+func init() {
+	// Start every mode at 0, so the series exist before the first song.
+	for _, mode := range []PickMode{PickRandom, PickLow, PickClassic} {
+		metrics.SongsRequested.WithLabelValues(string(mode))
+		metrics.SongsPlayed.WithLabelValues(string(mode))
+	}
+}
+
 type SongInfo struct {
 	SoundtrackUuid   uuid.UUID `json:"SoundtrackUuid"`
 	Soundtrack       string    `json:"Soundtrack"`
@@ -145,7 +164,11 @@ func (m *MusicService) markPlayed(ctx context.Context, songs []repository.ListQu
 	if err := m.q.AddSoundtrackPlayedByUUID(ctx, song.SoundtrackUuid); err != nil {
 		return err
 	}
-	return m.q.AddSongPlayedByUUID(ctx, song.SongUuid)
+	if err := m.q.AddSongPlayedByUUID(ctx, song.SongUuid); err != nil {
+		return err
+	}
+	metrics.SongsPlayed.WithLabelValues(song.Mode).Inc()
+	return nil
 }
 
 func (m *MusicService) loadQueue(ctx context.Context, queueID uuid.UUID) (repository.Queue, []repository.ListQueueSongsRow, error) {
@@ -167,7 +190,7 @@ func (m *MusicService) RandomSong(ctx context.Context, queueID uuid.UUID) (strin
 	for i, s := range soundtracks {
 		candidates[i] = weightedSoundtrack{soundtrack: s, weight: 1}
 	}
-	return m.pickAndRemember(ctx, queueID, candidates)
+	return m.pickAndRemember(ctx, queueID, candidates, PickRandom)
 }
 
 // RandomSongLowChance picks a random song, favouring soundtracks played less than average.
@@ -176,23 +199,30 @@ func (m *MusicService) RandomSongLowChance(ctx context.Context, queueID uuid.UUI
 	if err != nil {
 		return "", err
 	}
-	return m.pickAndRemember(ctx, queueID, lowChanceWeights(soundtracks))
+	return m.pickAndRemember(ctx, queueID, lowChanceWeights(soundtracks), PickLow)
 }
 
-func (m *MusicService) pickAndRemember(ctx context.Context, queueID uuid.UUID, candidates []weightedSoundtrack) (string, error) {
+func (m *MusicService) pickAndRemember(
+	ctx context.Context, queueID uuid.UUID, candidates []weightedSoundtrack, mode PickMode,
+) (string, error) {
 	song, err := m.pickSong(ctx, candidates)
 	if err != nil {
 		return "", err
 	}
-	return m.remember(ctx, queueID, song)
+	return m.remember(ctx, queueID, song, mode)
 }
 
-// remember stores the song as the queue's last fetched song, for AddLatestToQueue.
+// remember stores the song and how it was picked as the queue's last fetched song,
-func (m *MusicService) remember(ctx context.Context, queueID uuid.UUID, song repository.Song) (string, error) {
+// for AddLatestToQueue.
-	err := m.q.SetQueueLastFetched(ctx, repository.SetQueueLastFetchedParams{Uuid: queueID, LastFetchedSongUuid: song.Uuid})
+func (m *MusicService) remember(ctx context.Context, queueID uuid.UUID, song repository.Song, mode PickMode) (string, error) {
+	modeName := string(mode)
+	err := m.q.SetQueueLastFetched(ctx, repository.SetQueueLastFetchedParams{
+		Uuid: queueID, LastFetchedSongUuid: song.Uuid, LastFetchedMode: &modeName,
+	})
 	if err != nil {
 		return "", err
 	}
+	metrics.SongsRequested.WithLabelValues(modeName).Inc()
 	return song.Path, nil
 }
 
@@ -212,7 +242,7 @@ func (m *MusicService) RandomSongClassic(ctx context.Context, queueID uuid.UUID)
 			}
 			continue
 		}
-		return m.remember(ctx, queueID, row.Song)
+		return m.remember(ctx, queueID, row.Song, PickClassic)
 	}
 	return "", ErrNoSongs
 }

internal/backend/queue_integration_test.go

@@ -47,7 +47,31 @@ func TestQueueWithPostgres(t *testing.T) {
 		assert.NotEqual(t, uuid.Nil, song.SoundtrackUuid)
 	}
 
-	path, err := m.PreviousSong(ctx, queueID)
+	var modes []string
+	rows, err := database.Pool.Query(ctx, "SELECT mode FROM queue_song WHERE queue_uuid = $1 ORDER BY song_no", queueID)
+	require.NoError(t, err)
+	for rows.Next() {
+		var mode string
+		require.NoError(t, rows.Scan(&mode))
+		modes = append(modes, mode)
+	}
+	require.NoError(t, rows.Err())
+	assert.Equal(t, []string{"random", "random", "random"}, modes)
+
+	_, err = m.RandomSongClassic(ctx, queueID)
+	require.NoError(t, err)
+	require.NoError(t, m.AddLatestToQueue(ctx, queueID))
+	var classic string
+	require.NoError(t, database.Pool.QueryRow(ctx, "SELECT mode FROM queue_song WHERE queue_uuid = $1 AND song_no = 3", queueID).Scan(&classic))
+	assert.Equal(t, "classic", classic)
+	played, err = m.PlayedSongs(ctx, queueID)
+	require.NoError(t, err)
+	require.Len(t, played, 4)
+
+	path, err := m.Song(ctx, queueID, 2)
+	require.NoError(t, err)
+	assert.Equal(t, paths[2], path)
+	path, err = m.PreviousSong(ctx, queueID)
 	require.NoError(t, err)
 	assert.Equal(t, paths[1], path)
 	require.NoError(t, m.AddLatestPlayed(ctx, queueID))
@@ -82,7 +106,7 @@ func TestQueueWithPostgres(t *testing.T) {
 	require.NoError(t, q.ClearSongs(ctx))
 	played, err = m.PlayedSongs(ctx, queueID)
 	require.NoError(t, err)
-	require.Len(t, played, 3)
+	require.Len(t, played, 4)
 	assert.NotEmpty(t, played[0].Song)
 	_, err = m.Song(ctx, queueID, 0)
 	assert.ErrorIs(t, err, ErrNotFound)

internal/backend/sync.go

@@ -19,6 +19,7 @@ import (
 
 	"music-server/internal/db/repository"
 	"music-server/internal/logging"
+	"music-server/internal/metrics"
 
 	"github.com/MShekow/directory-checksum/directory_checksum"
 	"github.com/google/uuid"
@@ -199,6 +200,7 @@ func (s *SyncService) run(ctx context.Context, full bool) {
 	run := &syncRun{q: s.q, musicPath: s.musicPath, full: full, progress: func() { s.foldersSynced.Add(1) }}
 	result := run.execute(ctx, func(total int) { s.foldersTotal.Store(int64(total)) })
 	result.TotalTime = formatDuration(time.Since(start))
+	s.recordMetrics(ctx, full, time.Since(start), len(result.CatchedErrors) == 0)
 
 	if s.onComplete != nil {
 		s.onComplete(ctx)
@@ -218,6 +220,42 @@ func (s *SyncService) run(ctx context.Context, full bool) {
 		zap.Int("errors", len(result.CatchedErrors)))
 }
 
+func init() {
+	// Start both sync types at 0, so the series exist before the first sync.
+	metrics.SyncDuration.WithLabelValues(syncType(false))
+	metrics.SyncDuration.WithLabelValues(syncType(true))
+}
+
+// syncType is the type label of the sync metrics.
+func syncType(full bool) string {
+	if full {
+		return "full"
+	}
+	return "changes"
+}
+
+// recordMetrics updates the sync and library metrics after a sync.
+func (s *SyncService) recordMetrics(ctx context.Context, full bool, took time.Duration, success bool) {
+	metrics.SyncDuration.WithLabelValues(syncType(full)).Observe(took.Seconds())
+	if success {
+		metrics.SyncLastSuccess.SetToCurrentTime()
+	}
+	if err := s.UpdateLibraryMetrics(ctx); err != nil {
+		logging.GetLogger().Error("Failed to count the library", zap.Error(err))
+	}
+}
+
+// UpdateLibraryMetrics sets the library size metrics from the database.
+func (s *SyncService) UpdateLibraryMetrics(ctx context.Context) error {
+	count, err := s.q.CountLibrary(ctx)
+	if err != nil {
+		return err
+	}
+	metrics.LibrarySoundtracks.Set(float64(count.Soundtracks))
+	metrics.LibrarySongs.Set(float64(count.Songs))
+	return nil
+}
+
 func (r *syncRun) execute(ctx context.Context, setTotal func(int)) SyncResponse {
 	var err error
 	r.existing, err = r.q.GetAllSoundtracksIncludingDeleted(ctx)
@@ -623,6 +661,7 @@ func (r *syncRun) removeBrokenSongs(ctx context.Context) {
 	}
 	_ = g.Wait()
 
+	metrics.BrokenSongs.Set(float64(len(broken)))
 	if len(broken) > 0 {
 		if err := r.q.RemoveBrokenSongs(ctx, broken); err != nil {
 			r.addError("RemoveBrokenSongs", err, "")

internal/config/config.go

@@ -17,6 +17,9 @@ const DefaultReleasesURL = "https://gitea.sanplex.xyz/api/v1/repos/sansan/MusicP
 // Config holds all settings read from the environment.
 type Config struct {
 	Port int
+	// MetricsPort serves Prometheus metrics on a port of its own, so /metrics is never
+	// reachable through the public port. 0 means metrics are not served.
+	MetricsPort int
 
 	DBHost     string
 	DBPort     string
@@ -77,6 +80,11 @@ func load(getenv func(string) string) (Config, error) {
 	}
 	cfg.Port = port
 
+	cfg.MetricsPort, err = parseMetricsPort(getenv("METRICS_PORT"))
+	if err != nil {
+		errs = append(errs, fmt.Errorf("METRICS_PORT: %w", err))
+	}
+
 	for name, value := range map[string]string{
 		"DB_HOST":         cfg.DBHost,
 		"DB_PORT":         cfg.DBPort,
@@ -120,6 +128,24 @@ func parseCIDRs(value string) ([]*net.IPNet, error) {
 	return nets, nil
 }
 
+// DefaultMetricsPort is used when METRICS_PORT is not set.
+const DefaultMetricsPort = 2112
+
+// parseMetricsPort reads METRICS_PORT: empty for the default, "off" for no metrics, or a port.
+func parseMetricsPort(value string) (int, error) {
+	switch value {
+	case "":
+		return DefaultMetricsPort, nil
+	case "off":
+		return 0, nil
+	}
+	port, err := strconv.Atoi(value)
+	if err != nil || port < 1 || port > 65535 {
+		return 0, fmt.Errorf("%q is not a port or \"off\"", value)
+	}
+	return port, nil
+}
+
 // parseOrigins parses a comma-separated list of origins (scheme://host[:port], nothing else).
 func parseOrigins(value string) ([]string, error) {
 	var origins []string

internal/config/config_test.go

@@ -32,6 +32,26 @@ func TestLoadDefaults(t *testing.T) {
 	assert.Equal(t, DefaultReleasesURL, cfg.ReleasesURL)
 	assert.Empty(t, cfg.TrustedProxies)
 	assert.Empty(t, cfg.CORSAllowedOrigins)
+	assert.Equal(t, DefaultMetricsPort, cfg.MetricsPort)
+}
+
+func TestLoadMetricsPort(t *testing.T) {
+	values := validEnv()
+	values["METRICS_PORT"] = "9100"
+	cfg, err := load(env(values))
+	require.NoError(t, err)
+	assert.Equal(t, 9100, cfg.MetricsPort)
+
+	values["METRICS_PORT"] = "off"
+	cfg, err = load(env(values))
+	require.NoError(t, err)
+	assert.Zero(t, cfg.MetricsPort)
+
+	for _, invalid := range []string{"abc", "0", "70000", "-1", "OFF"} {
+		values["METRICS_PORT"] = invalid
+		_, err = load(env(values))
+		assert.ErrorContains(t, err, "METRICS_PORT", invalid)
+	}
 }
 
 func TestLoadMissingRequired(t *testing.T) {

internal/db/migrations/000006_add_fingerprint_and_queue.up.sql

@@ -7,8 +7,9 @@ ALTER TABLE soundtrack ADD COLUMN fingerprint TEXT NOT NULL DEFAULT '';
 CREATE TABLE queue (
     uuid UUID PRIMARY KEY DEFAULT gen_random_uuid(),
     current_song INT4 NOT NULL DEFAULT -1,
-    -- The last random song, added to the queue by /music/addQue.
+    -- The last random song, added to the queue by /music/addQue, and how it was picked.
     last_fetched_song_uuid UUID NULL,
+    last_fetched_mode VARCHAR(10) NULL,
     started_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
     ended_at TIMESTAMPTZ NULL
 );
@@ -21,6 +22,8 @@ CREATE TABLE queue_song (
     song_uuid UUID NOT NULL,
     soundtrack_name VARCHAR NOT NULL,
     song_name VARCHAR NOT NULL,
+    -- How the song was picked: random, low or classic.
+    mode VARCHAR(10) NOT NULL,
     added_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
     PRIMARY KEY (queue_uuid, song_no)
 );

internal/db/queries/queue.sql

@@ -16,7 +16,7 @@ UPDATE queue SET ended_at = NOW()
 WHERE ended_at IS NULL;
 
 -- name: SetQueueLastFetched :exec
-UPDATE queue SET last_fetched_song_uuid = $2
+UPDATE queue SET last_fetched_song_uuid = $2, last_fetched_mode = $3
 WHERE uuid = $1;
 
 -- name: SetQueueCurrentSong :exec
@@ -27,23 +27,23 @@ WHERE uuid = $1;
 -- Appends the queue's last fetched song and makes it the current song.
 -- Does nothing when no song has been fetched since the last call.
 WITH added AS (
-    INSERT INTO queue_song (queue_uuid, song_no, song_uuid, soundtrack_name, song_name)
+    INSERT INTO queue_song (queue_uuid, song_no, song_uuid, soundtrack_name, song_name, mode)
     SELECT q.uuid,
            (SELECT COALESCE(MAX(qs.song_no), -1) + 1 FROM queue_song qs WHERE qs.queue_uuid = q.uuid),
-           s.uuid, st.soundtrack_name, s.song_name
+           s.uuid, st.soundtrack_name, s.song_name, COALESCE(q.last_fetched_mode, 'unknown')
     FROM queue q
     JOIN song s ON s.uuid = q.last_fetched_song_uuid
     JOIN soundtrack st ON st.id = s.soundtrack_id
     WHERE q.uuid = $1
     RETURNING queue_uuid, song_no
 )
-UPDATE queue SET current_song = added.song_no, last_fetched_song_uuid = NULL
+UPDATE queue SET current_song = added.song_no, last_fetched_song_uuid = NULL, last_fetched_mode = NULL
 FROM added
 WHERE queue.uuid = added.queue_uuid;
 
 -- name: ListQueueSongs :many
 -- The song columns are NULL when the song has been removed from the library since.
-SELECT qs.song_no, qs.song_uuid, qs.soundtrack_name, qs.song_name,
+SELECT qs.song_no, qs.song_uuid, qs.soundtrack_name, qs.song_name, qs.mode,
        s.path, s.times_played, s.soundtrack_uuid
 FROM queue_song qs
 LEFT JOIN song s ON s.uuid = qs.song_uuid

internal/db/queries/statistics.sql

@@ -207,3 +207,9 @@ LEFT JOIN song s ON g.id = s.soundtrack_id
 WHERE g.deleted IS NULL
 GROUP BY g.id, g.soundtrack_name, g.times_played, g.last_played
 ORDER BY g.soundtrack_name;
+
+-- name: CountLibrary :one
+-- Soundtracks and songs in the library, without deleted soundtracks.
+SELECT
+    (SELECT COUNT(*) FROM soundtrack WHERE deleted IS NULL) AS soundtracks,
+    (SELECT COUNT(*) FROM song s JOIN soundtrack st ON st.id = s.soundtrack_id WHERE st.deleted IS NULL) AS songs;

internal/db/repository/models.go

@@ -15,6 +15,7 @@ type Queue struct {
 	Uuid                uuid.UUID          `json:"uuid"`
 	CurrentSong         int32              `json:"current_song"`
 	LastFetchedSongUuid uuid.UUID          `json:"last_fetched_song_uuid"`
+	LastFetchedMode     *string            `json:"last_fetched_mode"`
 	StartedAt           pgtype.Timestamptz `json:"started_at"`
 	EndedAt             pgtype.Timestamptz `json:"ended_at"`
 }
@@ -25,6 +26,7 @@ type QueueSong struct {
 	SongUuid       uuid.UUID          `json:"song_uuid"`
 	SoundtrackName string             `json:"soundtrack_name"`
 	SongName       string             `json:"song_name"`
+	Mode           string             `json:"mode"`
 	AddedAt        pgtype.Timestamptz `json:"added_at"`
 }
 

internal/db/repository/querier.go

@@ -26,6 +26,8 @@ type Querier interface {
 	ClearSongs(ctx context.Context) error
 	ClearSongsBySoundtrackId(ctx context.Context, soundtrackID int32) error
 	ClearSoundtracks(ctx context.Context) error
+	// Soundtracks and songs in the library, without deleted soundtracks.
+	CountLibrary(ctx context.Context) (CountLibraryRow, error)
 	CreateQueue(ctx context.Context) (Queue, error)
 	CreateSession(ctx context.Context, arg CreateSessionParams) (Session, error)
 	DeleteExpiredSessions(ctx context.Context) error

internal/db/repository/queue.sql.go

@@ -13,17 +13,17 @@ import (
 
 const addLastFetchedToQueue = `-- name: AddLastFetchedToQueue :exec
 WITH added AS (
-    INSERT INTO queue_song (queue_uuid, song_no, song_uuid, soundtrack_name, song_name)
+    INSERT INTO queue_song (queue_uuid, song_no, song_uuid, soundtrack_name, song_name, mode)
     SELECT q.uuid,
            (SELECT COALESCE(MAX(qs.song_no), -1) + 1 FROM queue_song qs WHERE qs.queue_uuid = q.uuid),
-           s.uuid, st.soundtrack_name, s.song_name
+           s.uuid, st.soundtrack_name, s.song_name, COALESCE(q.last_fetched_mode, 'unknown')
     FROM queue q
     JOIN song s ON s.uuid = q.last_fetched_song_uuid
     JOIN soundtrack st ON st.id = s.soundtrack_id
     WHERE q.uuid = $1
     RETURNING queue_uuid, song_no
 )
-UPDATE queue SET current_song = added.song_no, last_fetched_song_uuid = NULL
+UPDATE queue SET current_song = added.song_no, last_fetched_song_uuid = NULL, last_fetched_mode = NULL
 FROM added
 WHERE queue.uuid = added.queue_uuid
 `
@@ -37,7 +37,7 @@ func (q *Queries) AddLastFetchedToQueue(ctx context.Context, argUuid uuid.UUID)
 
 const createQueue = `-- name: CreateQueue :one
 INSERT INTO queue DEFAULT VALUES
-RETURNING uuid, current_song, last_fetched_song_uuid, started_at, ended_at
+RETURNING uuid, current_song, last_fetched_song_uuid, last_fetched_mode, started_at, ended_at
 `
 
 func (q *Queries) CreateQueue(ctx context.Context) (Queue, error) {
@@ -47,6 +47,7 @@ func (q *Queries) CreateQueue(ctx context.Context) (Queue, error) {
 		&i.Uuid,
 		&i.CurrentSong,
 		&i.LastFetchedSongUuid,
+		&i.LastFetchedMode,
 		&i.StartedAt,
 		&i.EndedAt,
 	)
@@ -74,7 +75,7 @@ func (q *Queries) EndQueue(ctx context.Context, argUuid uuid.UUID) error {
 }
 
 const getQueue = `-- name: GetQueue :one
-SELECT uuid, current_song, last_fetched_song_uuid, started_at, ended_at
+SELECT uuid, current_song, last_fetched_song_uuid, last_fetched_mode, started_at, ended_at
 FROM queue
 WHERE uuid = $1
 `
@@ -86,6 +87,7 @@ func (q *Queries) GetQueue(ctx context.Context, argUuid uuid.UUID) (Queue, error
 		&i.Uuid,
 		&i.CurrentSong,
 		&i.LastFetchedSongUuid,
+		&i.LastFetchedMode,
 		&i.StartedAt,
 		&i.EndedAt,
 	)
@@ -93,7 +95,7 @@ func (q *Queries) GetQueue(ctx context.Context, argUuid uuid.UUID) (Queue, error
 }
 
 const listQueueSongs = `-- name: ListQueueSongs :many
-SELECT qs.song_no, qs.song_uuid, qs.soundtrack_name, qs.song_name,
+SELECT qs.song_no, qs.song_uuid, qs.soundtrack_name, qs.song_name, qs.mode,
        s.path, s.times_played, s.soundtrack_uuid
 FROM queue_song qs
 LEFT JOIN song s ON s.uuid = qs.song_uuid
@@ -106,6 +108,7 @@ type ListQueueSongsRow struct {
 	SongUuid       uuid.UUID `json:"song_uuid"`
 	SoundtrackName string    `json:"soundtrack_name"`
 	SongName       string    `json:"song_name"`
+	Mode           string    `json:"mode"`
 	Path           *string   `json:"path"`
 	TimesPlayed    *int32    `json:"times_played"`
 	SoundtrackUuid uuid.UUID `json:"soundtrack_uuid"`
@@ -126,6 +129,7 @@ func (q *Queries) ListQueueSongs(ctx context.Context, queueUuid uuid.UUID) ([]Li
 			&i.SongUuid,
 			&i.SoundtrackName,
 			&i.SongName,
+			&i.Mode,
 			&i.Path,
 			&i.TimesPlayed,
 			&i.SoundtrackUuid,
@@ -156,16 +160,17 @@ func (q *Queries) SetQueueCurrentSong(ctx context.Context, arg SetQueueCurrentSo
 }
 
 const setQueueLastFetched = `-- name: SetQueueLastFetched :exec
-UPDATE queue SET last_fetched_song_uuid = $2
+UPDATE queue SET last_fetched_song_uuid = $2, last_fetched_mode = $3
 WHERE uuid = $1
 `
 
 type SetQueueLastFetchedParams struct {
 	Uuid                uuid.UUID `json:"uuid"`
 	LastFetchedSongUuid uuid.UUID `json:"last_fetched_song_uuid"`
+	LastFetchedMode     *string   `json:"last_fetched_mode"`
 }
 
 func (q *Queries) SetQueueLastFetched(ctx context.Context, arg SetQueueLastFetchedParams) error {
-	_, err := q.db.Exec(ctx, setQueueLastFetched, arg.Uuid, arg.LastFetchedSongUuid)
+	_, err := q.db.Exec(ctx, setQueueLastFetched, arg.Uuid, arg.LastFetchedSongUuid, arg.LastFetchedMode)
 	return err
 }

internal/db/repository/statistics.sql.go

@@ -12,6 +12,25 @@ import (
 	"github.com/google/uuid"
 )
 
+const countLibrary = `-- name: CountLibrary :one
+SELECT
+    (SELECT COUNT(*) FROM soundtrack WHERE deleted IS NULL) AS soundtracks,
+    (SELECT COUNT(*) FROM song s JOIN soundtrack st ON st.id = s.soundtrack_id WHERE st.deleted IS NULL) AS songs
+`
+
+type CountLibraryRow struct {
+	Soundtracks int64 `json:"soundtracks"`
+	Songs       int64 `json:"songs"`
+}
+
+// Soundtracks and songs in the library, without deleted soundtracks.
+func (q *Queries) CountLibrary(ctx context.Context) (CountLibraryRow, error) {
+	row := q.db.QueryRow(ctx, countLibrary)
+	var i CountLibraryRow
+	err := row.Scan(&i.Soundtracks, &i.Songs)
+	return i, err
+}
+
 const getAllSoundtracksWithSongs = `-- name: GetAllSoundtracksWithSongs :many
 SELECT 
     g.id as soundtrack_id,

internal/metrics/app.go

@@ -0,0 +1,53 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+// MusicServer's own metrics. Labels only take values from small fixed sets: never song
+// names, IDs, tokens or IPs, since every value creates a new time series.
+var (
+	// SongsRequested counts random songs handed out, by how they were picked.
+	SongsRequested = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "musicserver_songs_requested_total",
+		Help: "Random songs handed out, by how they were picked (random, low, classic).",
+	}, []string{"mode"})
+
+	// SongsPlayed counts songs marked as played, by how they were picked.
+	SongsPlayed = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "musicserver_songs_played_total",
+		Help: "Songs marked as played, by how they were picked (random, low, classic).",
+	}, []string{"mode"})
+
+	// LibrarySoundtracks is the number of soundtracks in the library.
+	LibrarySoundtracks = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "musicserver_library_soundtracks",
+		Help: "Soundtracks in the library, counted at startup and after each sync.",
+	})
+
+	// LibrarySongs is the number of songs in the library.
+	LibrarySongs = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "musicserver_library_songs",
+		Help: "Songs in the library, counted at startup and after each sync.",
+	})
+
+	// SyncDuration is how long syncs take, by type.
+	SyncDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "musicserver_sync_duration_seconds",
+		Help:    "How long a sync took, by type (full, changes).",
+		Buckets: []float64{1, 5, 10, 30, 60, 120, 300, 600, 1200, 1800, 3600},
+	}, []string{"type"})
+
+	// SyncLastSuccess is when the last sync without errors finished.
+	SyncLastSuccess = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "musicserver_sync_last_success_timestamp_seconds",
+		Help: "Unix time when the last sync without errors finished.",
+	})
+
+	// BrokenSongs is the number of songs the last full sync found missing and removed.
+	BrokenSongs = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "musicserver_broken_songs",
+		Help: "Songs whose file was missing in the last full sync. They are removed from the library.",
+	})
+)

internal/metrics/http.go

@@ -0,0 +1,49 @@
+package metrics
+
+import (
+	"strconv"
+	"time"
+
+	"github.com/labstack/echo/v5"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	httpRequests = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "http_requests_total",
+		Help: "HTTP requests by method, route and status code.",
+	}, []string{"method", "route", "code"})
+
+	httpDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Name: "http_request_duration_seconds",
+		Help: "HTTP request latency by method and route. Song downloads include the transfer.",
+		// Up to 30 s, since songs can take long to send to slow clients.
+		Buckets: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30},
+	}, []string{"method", "route"})
+)
+
+// unmatchedRoute labels requests that matched no route, so raw URLs never become labels.
+const unmatchedRoute = "unmatched"
+
+// Middleware records the count and latency of requests per route. The route label is the
+// registered pattern, like /api/v1/songs/:uuid, never the URL with its IDs and query.
+// Register it outside Recover, so requests that panic are counted as 500.
+func Middleware() echo.MiddlewareFunc {
+	return func(next echo.HandlerFunc) echo.HandlerFunc {
+		return func(c *echo.Context) error {
+			start := time.Now()
+			err := next(c)
+
+			route := c.Path()
+			if route == "" {
+				route = unmatchedRoute
+			}
+			_, status := echo.ResolveResponseStatus(c.Response(), err)
+			method := c.Request().Method
+			httpRequests.WithLabelValues(method, route, strconv.Itoa(status)).Inc()
+			httpDuration.WithLabelValues(method, route).Observe(time.Since(start).Seconds())
+			return err
+		}
+	}
+}

internal/metrics/http_test.go

@@ -0,0 +1,75 @@
+package metrics
+
+import (
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/labstack/echo/v5"
+	echoMiddleware "github.com/labstack/echo/v5/middleware"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/assert"
+)
+
+func newEcho() *echo.Echo {
+	e := echo.New()
+	e.Use(Middleware())
+	e.Use(echoMiddleware.Recover())
+	e.GET("/items/:id", func(c *echo.Context) error { return c.String(http.StatusOK, "item") })
+	e.GET("/fail", func(c *echo.Context) error { return errors.New("broken") })
+	e.GET("/panic", func(c *echo.Context) error { panic("boom") })
+	return e
+}
+
+func serve(e *echo.Echo, method, target string) {
+	e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(method, target, nil))
+}
+
+func requests(method, route, code string) float64 {
+	return testutil.ToFloat64(httpRequests.WithLabelValues(method, route, code))
+}
+
+func TestMiddlewareCountsRequestsByRoutePattern(t *testing.T) {
+	e := newEcho()
+	before := requests("GET", "/items/:id", "200")
+
+	serve(e, http.MethodGet, "/items/1?x=secret")
+	serve(e, http.MethodGet, "/items/2")
+
+	assert.Equal(t, before+2, requests("GET", "/items/:id", "200"))
+	assert.Equal(t, 0.0, requests("GET", "/items/1?x=secret", "200"), "the URL is never a label")
+	assert.Equal(t, 0.0, requests("GET", "/items/1", "200"))
+}
+
+func TestMiddlewareRecordsErrorStatus(t *testing.T) {
+	e := newEcho()
+	failBefore := requests("GET", "/fail", "500")
+	panicBefore := requests("GET", "/panic", "500")
+	missingBefore := requests("GET", unmatchedRoute, "404")
+
+	serve(e, http.MethodGet, "/fail")
+	serve(e, http.MethodGet, "/panic")
+	serve(e, http.MethodGet, "/no/such/page?q=1")
+
+	assert.Equal(t, failBefore+1, requests("GET", "/fail", "500"))
+	assert.Equal(t, panicBefore+1, requests("GET", "/panic", "500"), "Recover turns the panic into a 500")
+	assert.Equal(t, missingBefore+1, requests("GET", unmatchedRoute, "404"))
+}
+
+func observations(method, route string) uint64 {
+	m := &dto.Metric{}
+	_ = httpDuration.WithLabelValues(method, route).(prometheus.Metric).Write(m)
+	return m.GetHistogram().GetSampleCount()
+}
+
+func TestMiddlewareRecordsLatency(t *testing.T) {
+	e := newEcho()
+	before := observations("GET", "/items/:id")
+
+	serve(e, http.MethodGet, "/items/1")
+
+	assert.Equal(t, before+1, observations("GET", "/items/:id"))
+}

internal/metrics/pool.go

@@ -0,0 +1,70 @@
+package metrics
+
+import (
+	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// poolCollector reports the database connection pool's statistics on every scrape.
+type poolCollector struct {
+	pool *pgxpool.Pool
+
+	acquiredConns        *prometheus.Desc
+	idleConns            *prometheus.Desc
+	totalConns           *prometheus.Desc
+	maxConns             *prometheus.Desc
+	acquires             *prometheus.Desc
+	acquireWait          *prometheus.Desc
+	emptyAcquires        *prometheus.Desc
+	canceledAcquires     *prometheus.Desc
+	newConns             *prometheus.Desc
+	maxLifetimeDestroyed *prometheus.Desc
+	maxIdleDestroyed     *prometheus.Desc
+}
+
+// NewPoolCollector returns a collector for the musicserver_db_pool_* metrics of pool.
+// Register it once, for example with prometheus.MustRegister.
+func NewPoolCollector(pool *pgxpool.Pool) prometheus.Collector {
+	desc := func(name, help string) *prometheus.Desc {
+		return prometheus.NewDesc("musicserver_db_pool_"+name, help, nil, nil)
+	}
+	return &poolCollector{
+		pool:                 pool,
+		acquiredConns:        desc("acquired_conns", "Connections in use."),
+		idleConns:            desc("idle_conns", "Idle connections."),
+		totalConns:           desc("total_conns", "Open connections, in use, idle and being opened."),
+		maxConns:             desc("max_conns", "Most connections the pool opens."),
+		acquires:             desc("acquires_total", "Connections handed out."),
+		acquireWait:          desc("acquire_wait_seconds_total", "Total time spent waiting for a connection."),
+		emptyAcquires:        desc("empty_acquires_total", "Acquires that had to wait because no connection was idle."),
+		canceledAcquires:     desc("canceled_acquires_total", "Acquires canceled while waiting, for example by a timeout."),
+		newConns:             desc("new_conns_total", "Connections opened."),
+		maxLifetimeDestroyed: desc("max_lifetime_destroyed_total", "Connections closed for reaching their maximum lifetime."),
+		maxIdleDestroyed:     desc("max_idle_destroyed_total", "Connections closed for being idle too long."),
+	}
+}
+
+func (c *poolCollector) Describe(ch chan<- *prometheus.Desc) {
+	prometheus.DescribeByCollect(c, ch)
+}
+
+func (c *poolCollector) Collect(ch chan<- prometheus.Metric) {
+	s := c.pool.Stat()
+	gauge := func(d *prometheus.Desc, v float64) {
+		ch <- prometheus.MustNewConstMetric(d, prometheus.GaugeValue, v)
+	}
+	counter := func(d *prometheus.Desc, v float64) {
+		ch <- prometheus.MustNewConstMetric(d, prometheus.CounterValue, v)
+	}
+	gauge(c.acquiredConns, float64(s.AcquiredConns()))
+	gauge(c.idleConns, float64(s.IdleConns()))
+	gauge(c.totalConns, float64(s.TotalConns()))
+	gauge(c.maxConns, float64(s.MaxConns()))
+	counter(c.acquires, float64(s.AcquireCount()))
+	counter(c.acquireWait, s.AcquireDuration().Seconds())
+	counter(c.emptyAcquires, float64(s.EmptyAcquireCount()))
+	counter(c.canceledAcquires, float64(s.CanceledAcquireCount()))
+	counter(c.newConns, float64(s.NewConnsCount()))
+	counter(c.maxLifetimeDestroyed, float64(s.MaxLifetimeDestroyCount()))
+	counter(c.maxIdleDestroyed, float64(s.MaxIdleDestroyCount()))
+}

internal/metrics/pool_test.go

@@ -0,0 +1,33 @@
+package metrics
+
+import (
+	"context"
+	"strings"
+	"testing"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPoolCollector(t *testing.T) {
+	// The pool connects lazily, so no database is needed to read its statistics.
+	pool, err := pgxpool.New(context.Background(), "postgres://user:pass@127.0.0.1:1/music?pool_max_conns=7")
+	require.NoError(t, err)
+	t.Cleanup(pool.Close)
+
+	registry := prometheus.NewRegistry()
+	registry.MustRegister(NewPoolCollector(pool))
+
+	assert.Equal(t, 11, testutil.CollectAndCount(registry))
+	require.NoError(t, testutil.GatherAndCompare(registry, strings.NewReader(`
+# HELP musicserver_db_pool_max_conns Most connections the pool opens.
+# TYPE musicserver_db_pool_max_conns gauge
+musicserver_db_pool_max_conns 7
+# HELP musicserver_db_pool_acquired_conns Connections in use.
+# TYPE musicserver_db_pool_acquired_conns gauge
+musicserver_db_pool_acquired_conns 0
+`), "musicserver_db_pool_max_conns", "musicserver_db_pool_acquired_conns"))
+}

internal/metrics/server.go

@@ -0,0 +1,60 @@
+// Package metrics exposes Prometheus metrics: the Go runtime and process metrics that the
+// default registry collects, HTTP request metrics and MusicServer's own metrics.
+package metrics
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"time"
+
+	"music-server/internal/logging"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"go.uber.org/zap"
+)
+
+// Server serves /metrics on a port of its own, so it is never reachable through the
+// public port.
+type Server struct {
+	httpServer *http.Server
+}
+
+// NewServer creates a metrics server for the given port.
+func NewServer(port int) *Server {
+	mux := http.NewServeMux()
+	mux.Handle("GET /metrics", promhttp.Handler())
+	return &Server{httpServer: &http.Server{
+		Addr:              fmt.Sprintf(":%d", port),
+		Handler:           mux,
+		ReadHeaderTimeout: 5 * time.Second,
+	}}
+}
+
+// Handler returns the handler that serves /metrics.
+func (s *Server) Handler() http.Handler {
+	return s.httpServer.Handler
+}
+
+// Start listens on the port and serves metrics in the background. It returns an error
+// when the port cannot be opened.
+func (s *Server) Start() error {
+	ln, err := net.Listen("tcp", s.httpServer.Addr)
+	if err != nil {
+		return err
+	}
+	logging.GetLogger().Info("Metrics server starting", zap.String("address", s.httpServer.Addr))
+	go func() {
+		if err := s.httpServer.Serve(ln); !errors.Is(err, http.ErrServerClosed) {
+			logging.GetLogger().Error("Metrics server error", zap.Error(err))
+		}
+	}()
+	return nil
+}
+
+// Shutdown stops the metrics server.
+func (s *Server) Shutdown(ctx context.Context) error {
+	return s.httpServer.Shutdown(ctx)
+}

internal/metrics/server_test.go

@@ -0,0 +1,63 @@
+package metrics
+
+import (
+	"context"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"music-server/internal/logging"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMain(m *testing.M) {
+	logging.Init("fatal", false)
+	os.Exit(m.Run())
+}
+
+func TestServerServesRuntimeMetrics(t *testing.T) {
+	rec := httptest.NewRecorder()
+	NewServer(0).Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil))
+
+	assert.Equal(t, http.StatusOK, rec.Code)
+	assert.Contains(t, rec.Body.String(), "go_goroutines")
+	assert.Contains(t, rec.Body.String(), "process_start_time_seconds")
+}
+
+func TestServerServesOnlyMetrics(t *testing.T) {
+	rec := httptest.NewRecorder()
+	NewServer(0).Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil))
+	assert.Equal(t, http.StatusNotFound, rec.Code)
+}
+
+func TestServerStartAndShutdown(t *testing.T) {
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	port := ln.Addr().(*net.TCPAddr).Port
+	require.NoError(t, ln.Close())
+
+	s := NewServer(port)
+	require.NoError(t, s.Start())
+	resp, err := http.Get("http://" + ln.Addr().String() + "/metrics")
+	require.NoError(t, err)
+	body, _ := io.ReadAll(resp.Body)
+	resp.Body.Close()
+	assert.Contains(t, string(body), "go_goroutines")
+
+	require.NoError(t, s.Shutdown(context.Background()))
+	_, err = http.Get("http://" + ln.Addr().String() + "/metrics")
+	assert.Error(t, err, "the port is closed after shutdown")
+}
+
+func TestServerStartFailsWhenPortIsTaken(t *testing.T) {
+	ln, err := net.Listen("tcp", ":0")
+	require.NoError(t, err)
+	defer ln.Close()
+
+	assert.Error(t, NewServer(ln.Addr().(*net.TCPAddr).Port).Start())
+}

internal/server/routes.go

@@ -6,6 +6,7 @@ import (
 
 	"music-server/cmd/web"
 	"music-server/internal/logging"
+	"music-server/internal/metrics"
 	"music-server/internal/server/middleware"
 
 	"github.com/a-h/templ"
@@ -39,6 +40,8 @@ func (s *Server) routes() (*echo.Echo, huma.API) {
 	}
 
 	e.Use(logging.RequestLogger())
+	// Outside Recover, so requests that panic are counted with status 500.
+	e.Use(metrics.Middleware())
 	e.Use(echoMiddleware.Recover())
 	// Without allowed origins no CORS headers are sent, so only same-origin browser requests work.
 	// The API uses Bearer tokens, not cookies, so credentials are not allowed.

internal/server/routes_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"net"
 	"net/http"
+	"net/http/httptest"
 	"path/filepath"
 	"sync/atomic"
 	"testing"
@@ -223,3 +224,32 @@ func TestCORSWithoutAllowedOrigins(t *testing.T) {
 	assert.Equal(t, http.StatusOK, rec.Code, "the request itself works; the browser blocks reading it")
 	assert.Empty(t, rec.Header().Get("Access-Control-Allow-Origin"))
 }
+
+func TestMetricsAreNotServedOnTheMainPort(t *testing.T) {
+	e := newTestEnv(t, func(c *config.Config) { c.MetricsPort = 2112 })
+	require.NotNil(t, e.server.metrics)
+	for _, path := range []string{"/metrics", "/api/v1/metrics"} {
+		rec := e.do(http.MethodGet, path)
+		assert.Equal(t, http.StatusNotFound, rec.Code, path)
+		assert.NotContains(t, rec.Body.String(), "go_goroutines", path)
+	}
+}
+
+func TestMetricsOff(t *testing.T) {
+	assert.Nil(t, newTestEnv(t).server.metrics, "MetricsPort 0 means no metrics server")
+}
+
+func TestRequestMetricsUseRoutePatterns(t *testing.T) {
+	e := newTestEnv(t, func(c *config.Config) { c.MetricsPort = 2112 })
+	song := e.addSong("Game A", "Song", "x")
+	e.do(http.MethodGet, "/api/v1/songs/"+song.Uuid.String())
+	e.do(http.MethodGet, "/no/such/page?secret=1")
+
+	rec := httptest.NewRecorder()
+	e.server.metrics.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil))
+	body := rec.Body.String()
+	assert.Contains(t, body, `http_requests_total{code="200",method="GET",route="/api/v1/songs/:uuid"}`)
+	assert.NotContains(t, body, song.Uuid.String(), "IDs never become labels")
+	assert.NotContains(t, body, "secret", "query strings never become labels")
+	assert.NotContains(t, body, "/no/such/page", "unknown URLs count under the frontend's /* route")
+}

internal/server/server.go

@@ -12,7 +12,9 @@ import (
 	"music-server/internal/db"
 	"music-server/internal/db/repository"
 	"music-server/internal/logging"
+	"music-server/internal/metrics"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/zap"
 )
 
@@ -34,6 +36,8 @@ type Server struct {
 	stats      *backend.StatisticsService
 	search     *backend.SearchCache
 	httpServer *http.Server
+	// metrics serves /metrics on its own port; nil when metrics are off.
+	metrics *metrics.Server
 }
 
 // New creates a Server with all services wired up.
@@ -56,6 +60,9 @@ func New(cfg config.Config, q repository.Querier, database Database) (*Server, e
 	s.search = backend.NewSearchCache(s.music.SoundtrackNames)
 	s.sync = backend.NewSyncService(q, cfg.MusicPath, s.afterSync)
 
+	if cfg.MetricsPort > 0 {
+		s.metrics = metrics.NewServer(cfg.MetricsPort)
+	}
 	s.httpServer = &http.Server{
 		Addr:         fmt.Sprintf(":%d", cfg.Port),
 		Handler:      s.RegisterRoutes(),
@@ -111,11 +118,21 @@ func NewServerInstance() *Server {
 	if err != nil {
 		logger.Fatal("Failed to create server", zap.Error(err))
 	}
+	prometheus.MustRegister(metrics.NewPoolCollector(database.Pool))
+	if err := s.sync.UpdateLibraryMetrics(ctx); err != nil {
+		logger.Error("Failed to count the library", zap.Error(err))
+	}
 	return s
 }
 
-// ListenAndServe starts the HTTP server. It returns nil after a graceful shutdown.
+// ListenAndServe starts the HTTP server and the metrics server. It returns nil after a
+// graceful shutdown. The server keeps running without metrics if their port cannot be opened.
 func (s *Server) ListenAndServe() error {
+	if s.metrics != nil {
+		if err := s.metrics.Start(); err != nil {
+			logging.GetLogger().Error("Failed to start the metrics server", zap.Error(err))
+		}
+	}
 	logging.GetLogger().Info("Server starting", zap.String("address", s.httpServer.Addr))
 	if err := s.httpServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
 		return err
@@ -126,6 +143,9 @@ func (s *Server) ListenAndServe() error {
 // Shutdown stops accepting requests, waits for running requests and then closes the database.
 func (s *Server) Shutdown(ctx context.Context) error {
 	err := s.httpServer.Shutdown(ctx)
+	if s.metrics != nil {
+		err = errors.Join(err, s.metrics.Shutdown(ctx))
+	}
 	s.sync.Wait()
 	s.characters.Close()
 	s.db.Close()

internal/testutil/fakedb/fakedb.go

@@ -431,7 +431,9 @@ func (s *Store) EndActiveQueues(context.Context) error {
 }
 
 func (s *Store) SetQueueLastFetched(_ context.Context, arg repository.SetQueueLastFetchedParams) error {
-	return s.updateQueue(arg.Uuid, func(q *repository.Queue) { q.LastFetchedSongUuid = arg.LastFetchedSongUuid })
+	return s.updateQueue(arg.Uuid, func(q *repository.Queue) {
+		q.LastFetchedSongUuid, q.LastFetchedMode = arg.LastFetchedSongUuid, arg.LastFetchedMode
+	})
 }
 
 func (s *Store) SetQueueCurrentSong(_ context.Context, arg repository.SetQueueCurrentSongParams) error {
@@ -457,6 +459,10 @@ func (s *Store) AddLastFetchedToQueue(_ context.Context, id uuid.UUID) error {
 	if st := s.soundtrackIndex(song.SoundtrackID); st >= 0 {
 		soundtrackName = s.Soundtracks[st].SoundtrackName
 	}
+	mode := "unknown"
+	if queue.LastFetchedMode != nil {
+		mode = *queue.LastFetchedMode
+	}
 	songNo := int32(0)
 	for _, qs := range s.QueueSongs {
 		if qs.QueueUuid == id {
@@ -465,9 +471,10 @@ func (s *Store) AddLastFetchedToQueue(_ context.Context, id uuid.UUID) error {
 	}
 	s.QueueSongs = append(s.QueueSongs, repository.QueueSong{
 		QueueUuid: id, SongNo: songNo, SongUuid: song.Uuid, SoundtrackName: soundtrackName, SongName: song.SongName,
+		Mode: mode,
 	})
 	queue.CurrentSong = songNo
-	queue.LastFetchedSongUuid = uuid.Nil
+	queue.LastFetchedSongUuid, queue.LastFetchedMode = uuid.Nil, nil
 	return nil
 }
 
@@ -481,7 +488,9 @@ func (s *Store) ListQueueSongs(_ context.Context, id uuid.UUID) ([]repository.Li
 		if qs.QueueUuid != id {
 			continue
 		}
-		row := repository.ListQueueSongsRow{SongNo: qs.SongNo, SongUuid: qs.SongUuid, SoundtrackName: qs.SoundtrackName, SongName: qs.SongName}
+		row := repository.ListQueueSongsRow{
+			SongNo: qs.SongNo, SongUuid: qs.SongUuid, SoundtrackName: qs.SoundtrackName, SongName: qs.SongName, Mode: qs.Mode,
+		}
 		if i := slices.IndexFunc(s.Songs, func(song repository.Song) bool { return song.Uuid == qs.SongUuid }); i >= 0 {
 			song := s.Songs[i]
 			row.Path, row.TimesPlayed, row.SoundtrackUuid = &song.Path, &song.TimesPlayed, song.SoundtrackUuid
@@ -492,6 +501,26 @@ func (s *Store) ListQueueSongs(_ context.Context, id uuid.UUID) ([]repository.Li
 	return rows, nil
 }
 
+func (s *Store) CountLibrary(context.Context) (repository.CountLibraryRow, error) {
+	defer s.lock()()
+	if s.Err != nil {
+		return repository.CountLibraryRow{}, s.Err
+	}
+	var count repository.CountLibraryRow
+	for _, st := range s.Soundtracks {
+		if st.Deleted != nil {
+			continue
+		}
+		count.Soundtracks++
+		for _, song := range s.Songs {
+			if song.SoundtrackID == st.ID {
+				count.Songs++
+			}
+		}
+	}
+	return count, nil
+}
+
 // Statistics queries return empty results; tests that need data use their own fakes.
 
 func (s *Store) GetMostPlayedSoundtracksWithSongs(context.Context, int32) ([]repository.GetMostPlayedSoundtracksWithSongsRow, error) {