package backend import ( "crypto/md5" "encoding/hex" "errors" "fmt" "io" "io/fs" "music-server/internal/db" "music-server/internal/db/repository" "music-server/internal/logging" "os" "sort" "strconv" "strings" "sync" "time" "github.com/panjf2000/ants/v2" "github.com/MShekow/directory-checksum/directory_checksum" "github.com/spf13/afero" "go.uber.org/zap" ) var Syncing = false var foldersSynced float32 var numberOfFoldersToSync float32 var start time.Time var totalTime time.Duration var timeSpent time.Duration var allGames []repository.Game var gamesBeforeSync []repository.Game var gamesAfterSync []repository.Game var gamesAdded []string var gamesReAdded []string var gamesChangedTitle map[string]string var gamesChangedContent []string var gamesRemoved []string var catchedErrors []string var brokenSongs []string var pool *ants.Pool var poolSong *ants.Pool type SyncResponse struct { GamesAdded []string `json:"games_added"` GamesReAdded []string `json:"games_re_added"` GamesChangedTitle map[string]string `json:"games_changed_title"` GamesChangedContent []string `json:"games_changed_content"` GamesRemoved []string `json:"games_removed"` CatchedErrors []string `json:"catched_errors"` TotalTime string `json:"total_time"` } type ProgressResponse struct { Progress string `json:"progress"` TimeSpent string `json:"time_spent"` } type GameStatus int const ( NotChanged GameStatus = iota TitleChanged GameChanged NewGame ) var statusName = map[GameStatus]string{ NotChanged: "Not changed", TitleChanged: "Title changed", GameChanged: "Game changed", NewGame: "New game", } func (gs GameStatus) String() string { return statusName[gs] } func ResetDB() { repo.ClearSongs(db.Ctx) repo.ClearGames(db.Ctx) } func SyncProgress() ProgressResponse { progress := int((foldersSynced / numberOfFoldersToSync) * 100) currentTime := time.Now() timeSpent = currentTime.Sub(start) out := time.Time{}.Add(timeSpent) logging.GetLogger().Debug("Sync progress", zap.Int("progress_percent", progress), zap.Int("folders_synced", int(foldersSynced)), zap.Int("total_folders", int(numberOfFoldersToSync)), zap.String("time_spent", out.Format("15:04:05.00000"))) return ProgressResponse{ Progress: fmt.Sprintf("%v", progress), TimeSpent: out.Format("15:04:05"), } } func SyncResult() SyncResponse { logging.GetLogger().Info("Sync completed", zap.Int("games_before", len(gamesBeforeSync)), zap.Int("games_after", len(gamesAfterSync))) if len(gamesAdded) > 0 { logging.GetLogger().Debug("Games added", zap.Strings("games", gamesAdded)) } if len(gamesReAdded) > 0 { logging.GetLogger().Debug("Games readded", zap.Strings("games", gamesReAdded)) } if len(gamesChangedTitle) > 0 { logging.GetLogger().Debug("Games with changed title", zap.Any("changes", gamesChangedTitle)) } if len(gamesChangedContent) > 0 { logging.GetLogger().Debug("Games with changed content", zap.Strings("games", gamesChangedContent)) } var gamesRemovedTemp []string for _, beforeGame := range gamesBeforeSync { var found = false for _, afterGame := range gamesAfterSync { if beforeGame.GameName == afterGame.GameName { found = true break } } if !found { gamesRemovedTemp = append(gamesRemovedTemp, beforeGame.GameName) } } for _, game := range gamesRemovedTemp { var found bool = false for key := range gamesChangedTitle { if game == key { found = true break } } if !found { gamesRemoved = append(gamesRemoved, game) } } if len(gamesRemoved) > 0 { logging.GetLogger().Debug("Games removed", zap.Strings("games", gamesRemoved)) } if len(catchedErrors) > 0 { logging.GetLogger().Error("Errors caught during sync", zap.Strings("errors", catchedErrors)) } out := time.Time{}.Add(totalTime) logging.GetLogger().Info("Sync completed", zap.String("total_time", out.Format("15:04:05.00000"))) return SyncResponse{ GamesAdded: gamesAdded, GamesReAdded: gamesReAdded, GamesChangedTitle: gamesChangedTitle, GamesChangedContent: gamesChangedContent, GamesRemoved: gamesRemoved, CatchedErrors: catchedErrors, TotalTime: out.Format("15:04:05"), } } func SyncGamesNewFull() { syncGamesNew(true) Reset() } func SyncGamesNewOnlyChanges() { syncGamesNew(false) Reset() } func syncGamesNew(full bool) { Syncing = true musicPath := os.Getenv("MUSIC_PATH") fmt.Printf("dir: %s\n", musicPath) logging.GetLogger().Debug("Folder to sync", zap.String("MUSIC_PATH", musicPath)) if !strings.HasSuffix(musicPath, "/") { musicPath += "/" } var syncWg sync.WaitGroup initRepo() start = time.Now() foldersToSkip := []string{".sync", "dist", "old", "characters"} logging.GetLogger().Debug("Folders to skip during sync", zap.Strings("folders", foldersToSkip)) var err error gamesAdded = nil gamesReAdded = nil gamesChangedTitle = nil gamesChangedContent = nil gamesRemoved = nil catchedErrors = nil brokenSongs = nil gamesBeforeSync, err = repo.FindAllGames(db.Ctx) handleError("FindAllGames Before", err, "") logging.GetLogger().Info("Starting sync", zap.Int("games_before", len(gamesBeforeSync))) allGames, err = repo.GetAllGamesIncludingDeleted(db.Ctx) handleError("GetAllGamesIncludingDeleted", err, "") err = repo.SetGameDeletionDate(db.Ctx) handleError("SetGameDeletionDate", err, "") directories, err := os.ReadDir(musicPath) if err != nil { logging.GetLogger().Fatal("Failed to read music directory", zap.String("path", musicPath), zap.String("error", err.Error())) } pool, _ = ants.NewPool(10, ants.WithPreAlloc(true)) poolSong, _ = ants.NewPool(10, ants.WithPreAlloc(true)) defer pool.Release() defer poolSong.Release() foldersSynced = 0 numberOfFoldersToSync = float32(len(directories)) syncWg.Add(int(numberOfFoldersToSync)) for _, dir := range directories { pool.Submit(func() { defer syncWg.Done() syncGameNew(dir, foldersToSkip, musicPath, full) }) } syncWg.Wait() checkBrokenSongsNew() gamesAfterSync, err = repo.FindAllGames(db.Ctx) handleError("FindAllGames After", err, "") finished := time.Now() totalTime = finished.Sub(start) out := time.Time{}.Add(totalTime) logging.GetLogger().Info("Sync completed", zap.Duration("total_time", totalTime), zap.String("formatted_time", out.Format("15:04:05.00000"))) Syncing = false } func checkBrokenSongsNew() { allSongs, err := repo.FetchAllSongs(db.Ctx) handleError("FetchAllSongs", err, "") var brokenWg sync.WaitGroup poolBroken, _ := ants.NewPool(200, ants.WithPreAlloc(true)) defer poolBroken.Release() brokenWg.Add(len(allSongs)) for _, song := range allSongs { poolBroken.Submit(func() { defer brokenWg.Done() checkBrokenSongNew(song) }) } brokenWg.Wait() err = repo.RemoveBrokenSongs(db.Ctx, brokenSongs) handleError("RemoveBrokenSongs", err, "") } func checkBrokenSongNew(song repository.Song) { //Check if file exists and open openFile, err := os.Open(song.Path) if err != nil { //File not found brokenSongs = append(brokenSongs, song.Path) logging.GetLogger().Warn("Broken song found", zap.String("path", song.Path)) } else { err = openFile.Close() if err != nil { logging.GetLogger().Error("Failed to close file", zap.String("path", song.Path), zap.String("error", err.Error())) } } } func syncGameNew(file os.DirEntry, foldersToSkip []string, baseDir string, full bool) { if file.IsDir() && !contains(foldersToSkip, file.Name()) { logging.GetLogger().Debug("Syncing game", zap.String("game", file.Name())) gameDir := baseDir + file.Name() + "/" dirHash := getHashForDir(gameDir) var status GameStatus = NewGame var oldGame repository.Game var id int32 = -1 //fmt.Printf("Games before: %d\n", len(gamesBeforeSync)) for _, currentGame := range allGames { oldGame = currentGame //fmt.Printf("%s | %s\n", oldGame.GameName, oldGame.Hash) if oldGame.GameName == file.Name() && oldGame.Hash == dirHash { status = NotChanged id = oldGame.ID //fmt.Printf("Game not changed\n") break } else if oldGame.GameName == file.Name() && oldGame.Hash != dirHash { status = GameChanged id = oldGame.ID //fmt.Printf("Game changed\n") break } else if oldGame.GameName != file.Name() && oldGame.Hash == dirHash { status = TitleChanged id = oldGame.ID //fmt.Printf("GameName changed\n") break } } if full { status = TitleChanged } entries, err := os.ReadDir(gameDir) if err != nil { logging.GetLogger().Error("Failed to read game directory", zap.String("path", gameDir), zap.String("error", err.Error())) } switch status { case NewGame: if id != -1 { for _, entry := range entries { fileInfo, err := entry.Info() if err != nil { logging.GetLogger().Error("Failed to get file info", zap.String("error", err.Error())) continue } id = getIdFromFileNew(fileInfo) if id != -1 { break } } err = repo.InsertGameWithExistingId(db.Ctx, repository.InsertGameWithExistingIdParams{ID: id, GameName: file.Name(), Path: gameDir, Hash: dirHash}) handleError("InsertGameWithExistingId", err, "") if err != nil { logging.GetLogger().Debug("Game already exists, removing old ID file", zap.Int32("id", id), zap.String("game_dir", gameDir)) fileName := gameDir + "/." + strconv.Itoa(int(id)) + ".id" logging.GetLogger().Debug("Removing ID file", zap.String("filename", fileName)) err := os.Remove(fileName) if err != nil { logging.GetLogger().Error("Failed to remove ID file", zap.String("filename", fileName), zap.String("error", err.Error())) } newDirHash := getHashForDir(gameDir) id = insertGameNew(file.Name(), gameDir, newDirHash) } } else { id = insertGameNew(file.Name(), gameDir, dirHash) } logging.GetLogger().Debug("New game detected", zap.Int32("id", id), zap.String("game", file.Name()), zap.String("hash", dirHash), zap.String("status", status.String())) gamesAdded = append(gamesAdded, file.Name()) newCheckSongs(entries, gameDir, id) case GameChanged: logging.GetLogger().Debug("Game changed", zap.Int32("id", id), zap.String("game", file.Name()), zap.String("hash", dirHash), zap.String("status", status.String())) err = repo.UpdateGameHash(db.Ctx, repository.UpdateGameHashParams{Hash: dirHash, ID: id}) handleError("UpdateGameHash", err, "") gamesChangedContent = append(gamesChangedContent, file.Name()) newCheckSongs(entries, gameDir, id) case TitleChanged: logging.GetLogger().Debug("Game title changed", zap.Int32("id", id), zap.String("oldName", oldGame.GameName), zap.String("newName", file.Name()), zap.String("hash", dirHash), zap.String("status", status.String())) err = repo.UpdateGameName(db.Ctx, repository.UpdateGameNameParams{Name: file.Name(), Path: gameDir, ID: id}) handleError("UpdateGameName", err, "") newCheckSongs(entries, gameDir, id) if gamesChangedTitle == nil { gamesChangedTitle = make(map[string]string) } gamesChangedTitle[oldGame.GameName] = file.Name() case NotChanged: var found bool = false for _, beforeGame := range gamesBeforeSync { if dirHash == beforeGame.Hash { found = true logging.GetLogger().Debug("Game not changed", zap.Int32("id", id), zap.String("newName", file.Name()), zap.String("hash", dirHash), zap.String("status", status.String())) } } if !found { newCheckSongs(entries, gameDir, id) gamesReAdded = append(gamesReAdded, file.Name()) logging.GetLogger().Debug("Game added again", zap.Int32("id", id), zap.String("newName", file.Name()), zap.String("hash", dirHash), zap.String("status", status.String())) } } logging.GetLogger().Debug("Game sync status", zap.Int32("id", id), zap.String("game", file.Name()), zap.String("hash", dirHash), zap.String("status", status.String())) err = repo.RemoveDeletionDate(db.Ctx, id) handleError("RemoveDeletionDate", err, "") } foldersSynced++ logging.GetLogger().Debug("Sync progress", zap.Int("folders_synced", int(foldersSynced)), zap.Int("total_folders", int(numberOfFoldersToSync)), zap.Int("percent", int((foldersSynced/numberOfFoldersToSync)*100))) } func insertGameNew(name string, path string, hash string) int32 { var duplicateError = errors.New("ERROR: duplicate key value violates unique") id, err := repo.InsertGame(db.Ctx, repository.InsertGameParams{GameName: name, Path: path, Hash: hash}) handleError("InsertGame", err, "") if err != nil { logging.GetLogger().Warn("ID collision detected, resetting sequence") if strings.HasPrefix(err.Error(), duplicateError.Error()) { logging.GetLogger().Debug("Resetting game ID sequence") _, err = repo.ResetGameIdSeq(db.Ctx) handleError("ResetGameIdSeq", err, "") id = insertGameNew(name, path, hash) } } return id } func newCheckSongs(entries []os.DirEntry, gameDir string, id int32) int32 { //hasher := md5.New() var numberOfSongs int32 numberOfFiles := len(entries) var songWg sync.WaitGroup songWg.Add(numberOfFiles) for _, entry := range entries { poolSong.Submit(func() { defer songWg.Done() if newCheckSong(entry, gameDir, id) { numberOfSongs++ } }) } songWg.Wait() return numberOfSongs } func newCheckSong(entry os.DirEntry, gameDir string, id int32) bool { fileInfo, err := entry.Info() if err != nil { logging.GetLogger().Error("Failed to get file info", zap.String("filename", entry.Name()), zap.String("error", err.Error())) return false } if isSong(fileInfo) { path := gameDir + entry.Name() songHash := getHashForFile(path) //numberOfSongs++ fileName := entry.Name() songName, _ := strings.CutSuffix(fileName, ".mp3") song, err := repo.GetSongWithHash(db.Ctx, songHash) handleError("GetSongWithHash", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s", id, path, entry.Name(), songHash)) if err == nil { if song.SongName == songName && song.Path == path { return false } } logging.GetLogger().Debug("Song changed", zap.Int32("game_id", id), zap.String("path", path), zap.String("song_name", songName), zap.String("song_hash", songHash)) count, err := repo.CheckSongWithHash(db.Ctx, songHash) handleError("CheckSongWithHash", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash)) if err != nil { count2, err := repo.CheckSong(db.Ctx, path) handleError("CheckSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s\n", id, path, entry.Name(), songHash)) if count2 > 0 { err = repo.AddHashToSong(db.Ctx, repository.AddHashToSongParams{Hash: songHash, Path: path}) handleError("AddHashToSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s", id, path, entry.Name(), songHash)) count, err = repo.CheckSongWithHash(db.Ctx, songHash) handleError("CheckSongWithHash 2", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s", id, path, entry.Name(), songHash)) } } //count, _ := repo.CheckSong(ctx, path) if count > 0 { err = repo.UpdateSong(db.Ctx, repository.UpdateSongParams{SongName: songName, FileName: &fileName, Path: path, Hash: songHash}) handleError("UpdateSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s", id, path, entry.Name(), songHash)) } else { count2, err := repo.CheckSong(db.Ctx, path) handleError("CheckSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s", id, path, entry.Name(), songHash)) if count2 > 0 { err = repo.AddHashToSong(db.Ctx, repository.AddHashToSongParams{Hash: songHash, Path: path}) handleError("AddHashToSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s", id, path, entry.Name(), songHash)) } else { err = repo.AddSong(db.Ctx, repository.AddSongParams{GameID: id, SongName: songName, Path: path, FileName: &fileName, Hash: songHash}) handleError("AddSong", err, fmt.Sprintf("GameID: %d | Path: %s | SongName: %s | SongHash: %s", id, path, entry.Name(), songHash)) } } return true } else if isCoverImage(fileInfo) { //TODO: Later add cover art image here in db } return false } func handleError(funcName string, err error, msg string) { var compareError = errors.New("no rows in result set") if err != nil { if compareError.Error() != err.Error() { logging.GetLogger().Error("Database error", zap.String("function", funcName), zap.String("error", err.Error())) if msg != "" { logging.GetLogger().Debug("Error context", zap.String("message", msg)) catchedErrors = append(catchedErrors, fmt.Sprintf("Func: %s\nError message: %s\nDebug message: %s", funcName, err, msg)) } else { catchedErrors = append(catchedErrors, fmt.Sprintf("Func: %s\nError message: %s", funcName, err)) } } } } func getHashForDir(gameDir string) string { directory, _ := directory_checksum.ScanDirectory(gameDir, afero.NewOsFs()) hash, _ := directory.ComputeDirectoryChecksums() return hash } func getHashForFile(path string) string { hasher := md5.New() readFile, err := os.Open(path) if err != nil { logging.GetLogger().Fatal("Failed to open file for hashing", zap.String("path", path), zap.String("error", err.Error())) } defer readFile.Close() hasher.Reset() _, err = io.Copy(hasher, readFile) if err != nil { logging.GetLogger().Fatal("Failed to hash file", zap.String("path", path), zap.String("error", err.Error())) } return hex.EncodeToString(hasher.Sum(nil)) } func getIdFromFileNew(file os.FileInfo) int32 { name := file.Name() if !file.IsDir() && strings.HasSuffix(name, ".id") { name = strings.Replace(name, ".id", "", 1) name = strings.Replace(name, ".", "", 1) i, _ := strconv.Atoi(name) return int32(i) } return -1 } func isSong(entry fs.FileInfo) bool { return !entry.IsDir() && strings.HasSuffix(entry.Name(), ".mp3") } func isCoverImage(entry fs.FileInfo) bool { return !entry.IsDir() && strings.Contains(entry.Name(), "cover") && (strings.HasSuffix(entry.Name(), ".jpg") || strings.HasSuffix(entry.Name(), ".png")) } func contains(s []string, searchTerm string) bool { i := sort.SearchStrings(s, searchTerm) return i < len(s) && s[i] == searchTerm }