#9 Major routes refactor and expansion, #15 Added API for statistics, Added search cache backend and functionality tests, Backend sync improvements

internal/backend/search_cache.go

@@ -0,0 +1,100 @@
+package backend
+
+import (
+	"regexp"
+	"strings"
+	"sync"
+)
+
+// SoundtrackData holds precomputed search data for a soundtrack
+type SoundtrackData struct {
+	Original  string
+	Lowercase string
+	Cleaned   string
+}
+
+var (
+	searchCache           []SoundtrackData
+	searchCacheMutex      sync.Mutex
+	searchRegexCache      = make(map[string]*regexp.Regexp)
+	searchRegexCacheMutex sync.Mutex
+)
+
+// GetSearchCache returns the precomputed search cache, initializing it if needed
+func GetSearchCache() []SoundtrackData {
+	searchCacheMutex.Lock()
+	defer searchCacheMutex.Unlock()
+
+	if len(searchCache) == 0 {
+		loadSearchCache()
+	}
+	return searchCache
+}
+
+// ReloadSearchCache clears and reloads the search cache.
+// This should be called after a sync operation to ensure search results are up to date.
+func ReloadSearchCache() {
+	searchCacheMutex.Lock()
+	defer searchCacheMutex.Unlock()
+	searchCache = nil
+	loadSearchCache()
+}
+
+// loadSearchCache loads the soundtrack data into the cache
+func loadSearchCache() {
+	soundtracks := GetAllSoundtracks()
+	searchCache = make([]SoundtrackData, len(soundtracks))
+
+	for i, soundtrack := range soundtracks {
+		cleaned := cleanSearchTerm(soundtrack)
+		lowercase := strings.ToLower(soundtrack)
+		searchCache[i] = SoundtrackData{
+			Original:  soundtrack,
+			Lowercase: lowercase,
+			Cleaned:   cleaned,
+		}
+	}
+}
+
+// CleanSearchTermForMatch cleans a search term for better matching
+func CleanSearchTermForMatch(term string) string {
+	term = strings.ReplaceAll(term, " ", "")
+	term = strings.ReplaceAll(term, "é", "e")
+	term = strings.ReplaceAll(term, "+", "plus")
+	term = strings.ReplaceAll(term, "&", "and")
+	term = strings.ReplaceAll(term, "'n", "and")
+	return strings.ToLower(term)
+}
+
+// cleanSearchTerm cleans a search term for better matching (internal)
+func cleanSearchTerm(term string) string {
+	return CleanSearchTermForMatch(term)
+}
+
+// MatchRegexCached checks if a search term matches using cached compiled regex
+func MatchRegexCached(searchTerm string, soundtrackName string) bool {
+	if searchTerm == "" {
+		return true
+	}
+
+	// Check cache first
+	searchRegexCacheMutex.Lock()
+	re, exists := searchRegexCache[searchTerm]
+	if !exists {
+		re = compileSearchRegex(searchTerm)
+		searchRegexCache[searchTerm] = re
+	}
+	searchRegexCacheMutex.Unlock()
+
+	return re.MatchString(soundtrackName)
+}
+
+// compileSearchRegex compiles a search term into a regex pattern
+func compileSearchRegex(searchTerm string) *regexp.Regexp {
+	regText := ".*"
+	for _, letter := range searchTerm {
+		regText += string(letter) + ".*"
+	}
+	re, _ := regexp.Compile(regText)
+	return re
+}

internal/backend/sync.go

@@ -16,6 +16,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/panjf2000/ants/v2"
 
 	"github.com/MShekow/directory-checksum/directory_checksum"
@@ -178,11 +179,13 @@ func SyncResult() SyncResponse {
 func SyncSoundtracksFull() {
 	syncSoundtracks(true)
 	Reset()
+	ReloadSearchCache()
 }
 
 func SyncSoundtracksOnlyChanges() {
 	syncSoundtracks(false)
 	Reset()
+	ReloadSearchCache()
 }
 
 func syncSoundtracks(full bool) {
@@ -295,6 +298,7 @@ func syncSoundtrack(file os.DirEntry, foldersToSkip []string, baseDir string, fu
 		var status SoundtrackStatus = NewSoundtrack
 		var oldSoundtrack repository.Soundtrack
 		var id int32 = -1
+		var soundtrackUuid uuid.UUID
 
 		//fmt.Printf("Soundtracks before: %d\n", len(soundtracksBeforeSync))
 
@@ -304,16 +308,19 @@ func syncSoundtrack(file os.DirEntry, foldersToSkip []string, baseDir string, fu
 			if oldSoundtrack.SoundtrackName == file.Name() && oldSoundtrack.Hash == dirHash {
 				status = NotChanged
 				id = oldSoundtrack.ID
+				soundtrackUuid = oldSoundtrack.Uuid
 				//fmt.Printf("Soundtrack not changed\n")
 				break
 			} else if oldSoundtrack.SoundtrackName == file.Name() && oldSoundtrack.Hash != dirHash {
 				status = SoundtrackChanged
 				id = oldSoundtrack.ID
+				soundtrackUuid = oldSoundtrack.Uuid
 				//fmt.Printf("Soundtrack changed\n")
 				break
 			} else if oldSoundtrack.SoundtrackName != file.Name() && oldSoundtrack.Hash == dirHash {
 				status = TitleChanged
 				id = oldSoundtrack.ID
+				soundtrackUuid = oldSoundtrack.Uuid
 				//fmt.Printf("SoundtrackName changed\n")
 				break
 			}
@@ -328,34 +335,37 @@ func syncSoundtrack(file os.DirEntry, foldersToSkip []string, baseDir string, fu
 		}
 		switch status {
 		case NewSoundtrack:
-			id = insertSoundtrack(file.Name(), soundtrackDir, dirHash)
+			id, soundtrackUuid = insertSoundtrack(file.Name(), soundtrackDir, dirHash)
 			logging.GetLogger().Debug("New soundtrack detected",
 				zap.Int32("id", id),
+				zap.String("uuid", soundtrackUuid.String()),
 				zap.String("soundtrack", file.Name()),
 				zap.String("hash", dirHash),
 				zap.String("status", status.String()))
 			soundtracksAdded = append(soundtracksAdded, file.Name())
-			checkSongs(entries, soundtrackDir, id)
+			checkSongs(entries, soundtrackDir, id, soundtrackUuid)
 		case SoundtrackChanged:
 			logging.GetLogger().Debug("Soundtrack changed",
 				zap.Int32("id", id),
+				zap.String("uuid", soundtrackUuid.String()),
 				zap.String("soundtrack", file.Name()),
 				zap.String("hash", dirHash),
 				zap.String("status", status.String()))
 			err = repo.UpdateSoundtrackHash(BackendCtx(), repository.UpdateSoundtrackHashParams{Hash: dirHash, ID: id})
 			handleError("UpdateSoundtrackHash", err, "")
 			soundtracksChangedContent = append(soundtracksChangedContent, file.Name())
-			checkSongs(entries, soundtrackDir, id)
+			checkSongs(entries, soundtrackDir, id, soundtrackUuid)
 		case TitleChanged:
 			logging.GetLogger().Debug("Soundtrack title changed",
 				zap.Int32("id", id),
+				zap.String("uuid", soundtrackUuid.String()),
 				zap.String("oldName", oldSoundtrack.SoundtrackName),
 				zap.String("newName", file.Name()),
 				zap.String("hash", dirHash),
 				zap.String("status", status.String()))
 			err = repo.UpdateSoundtrackName(BackendCtx(), repository.UpdateSoundtrackNameParams{Name: file.Name(), Path: soundtrackDir, ID: id})
 			handleError("UpdateSoundtrackName", err, "")
-			checkSongs(entries, soundtrackDir, id)
+			checkSongs(entries, soundtrackDir, id, soundtrackUuid)
 			if soundtracksChangedTitle == nil {
 				soundtracksChangedTitle = make(map[string]string)
 			}
@@ -373,10 +383,11 @@ func syncSoundtrack(file os.DirEntry, foldersToSkip []string, baseDir string, fu
 				}
 			}
 			if !found {
-				checkSongs(entries, soundtrackDir, id)
+				checkSongs(entries, soundtrackDir, id, soundtrackUuid)
 				soundtracksReAdded = append(soundtracksReAdded, file.Name())
 				logging.GetLogger().Debug("Soundtrack added again",
 					zap.Int32("id", id),
+					zap.String("uuid", soundtrackUuid.String()),
 					zap.String("newName", file.Name()),
 					zap.String("hash", dirHash),
 					zap.String("status", status.String()))
@@ -390,6 +401,15 @@ func syncSoundtrack(file os.DirEntry, foldersToSkip []string, baseDir string, fu
 			zap.String("status", status.String()))
 		err = repo.RemoveSoundtrackDeletionDate(BackendCtx(), id)
 		handleError("RemoveSoundtrackDeletionDate", err, "")
+
+		// Remove .id files and update hash if any were removed
+		if removeIdFilesInSoundtrack(soundtrackDir) {
+			newHash := getHashForDir(soundtrackDir)
+			if newHash != dirHash {
+				err = repo.UpdateSoundtrackHash(BackendCtx(), repository.UpdateSoundtrackHashParams{Hash: newHash, ID: id})
+				handleError("UpdateSoundtrackHash for .id cleanup", err, "")
+			}
+		}
 	}
 	foldersSynced++
 	logging.GetLogger().Debug("Sync progress",
@@ -398,24 +418,31 @@ func syncSoundtrack(file os.DirEntry, foldersToSkip []string, baseDir string, fu
 		zap.Int("percent", int((foldersSynced/numberOfFoldersToSync)*100)))
 }
 
-func insertSoundtrack(name string, path string, hash string) int32 {
+func insertSoundtrack(name string, path string, hash string) (int32, uuid.UUID) {
 	var duplicateError = errors.New("ERROR: duplicate key value violates unique")
-	id, err := repo.InsertSoundtrack(BackendCtx(), repository.InsertSoundtrackParams{SoundtrackName: name, Path: path, Hash: hash})
+	soundtrackUuid := uuid.New()
+	result, err := repo.InsertSoundtrack(BackendCtx(), repository.InsertSoundtrackParams{
+		SoundtrackName: name,
+		Path:           path,
+		Hash:           hash,
+		Uuid:           soundtrackUuid,
+	})
 	handleError("InsertSoundtrack", err, "")
 	if err != nil {
-		logging.GetLogger().Warn("ID collision detected, resetting sequence")
+		logging.GetLogger().Warn("ID collision or UUID duplicate detected, resetting sequence")
 		if strings.HasPrefix(err.Error(), duplicateError.Error()) {
 			logging.GetLogger().Debug("Resetting soundtrack ID sequence")
 			_, err = repo.ResetSoundtrackIdSeq(BackendCtx())
 			handleError("ResetSoundtrackIdSeq", err, "")
-			id = insertSoundtrack(name, path, hash)
+			id, uuid := insertSoundtrack(name, path, hash)
+			return id, uuid
 		}
 	}
-	return id
+	return result.ID, result.Uuid
 
 }
 
-func checkSongs(entries []os.DirEntry, soundtrackDir string, id int32) int32 {
+func checkSongs(entries []os.DirEntry, soundtrackDir string, id int32, soundtrackUuid uuid.UUID) int32 {
 	//hasher := md5.New()
 	var numberOfSongs int32
 	numberOfFiles := len(entries)
@@ -425,7 +452,7 @@ func checkSongs(entries []os.DirEntry, soundtrackDir string, id int32) int32 {
 	for _, entry := range entries {
 		poolSong.Submit(func() {
 			defer songWg.Done()
-			if checkSong(entry, soundtrackDir, id) {
+			if checkSong(entry, soundtrackDir, id, soundtrackUuid) {
 				numberOfSongs++
 			}
 		})
@@ -434,7 +461,7 @@ func checkSongs(entries []os.DirEntry, soundtrackDir string, id int32) int32 {
 	return numberOfSongs
 }
 
-func checkSong(entry os.DirEntry, soundtrackDir string, id int32) bool {
+func checkSong(entry os.DirEntry, soundtrackDir string, id int32, soundtrackUuid uuid.UUID) bool {
 	fileInfo, err := entry.Info()
 	if err != nil {
 		logging.GetLogger().Error("Failed to get file info", zap.String("filename", entry.Name()), zap.String("error", err.Error()))
@@ -487,8 +514,17 @@ func checkSong(entry os.DirEntry, soundtrackDir string, id int32) bool {
 				err = repo.AddHashToSong(BackendCtx(), repository.AddHashToSongParams{Hash: songHash, SoundtrackID: id, Path: path})
 				handleError("AddHashToSong", err, fmt.Sprintf("SoundtrackID: %d | Path: %s | SongName: %s | SongHash: %s", id, path, entry.Name(), songHash))
 			} else {
-				err = repo.AddSong(BackendCtx(), repository.AddSongParams{SoundtrackID: id, SongName: songName, Path: path, FileName: &fileName, Hash: songHash})
-				handleError("AddSong", err, fmt.Sprintf("SoundtrackID: %d | Path: %s | SongName: %s | SongHash: %s", id, path, entry.Name(), songHash))
+				songUuid := uuid.New()
+				err = repo.AddSong(BackendCtx(), repository.AddSongParams{
+					SoundtrackID:   id,
+					SongName:       songName,
+					Path:           path,
+					FileName:       &fileName,
+					Hash:           songHash,
+					Uuid:           songUuid,
+					SoundtrackUuid: soundtrackUuid,
+				})
+				handleError("AddSong", err, fmt.Sprintf("SoundtrackID: %d | UUID: %s | Path: %s | SongName: %s | SongHash: %s", id, songUuid.String(), path, entry.Name(), songHash))
 
 			}
 		}
@@ -558,6 +594,33 @@ func isCoverImage(entry fs.FileInfo) bool {
 		(strings.HasSuffix(entry.Name(), ".jpg") || strings.HasSuffix(entry.Name(), ".png"))
 }
 
+func removeIdFilesInSoundtrack(soundtrackDir string) bool {
+	entries, err := os.ReadDir(soundtrackDir)
+	if err != nil {
+		logging.GetLogger().Error("Failed to read soundtrack directory for .id file cleanup",
+			zap.String("path", soundtrackDir),
+			zap.String("error", err.Error()))
+		return false
+	}
+
+	anyRemoved := false
+	for _, entry := range entries {
+		if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".id") {
+			filePath := soundtrackDir + entry.Name()
+			err := os.Remove(filePath)
+			if err != nil {
+				logging.GetLogger().Error("Failed to remove .id file",
+					zap.String("path", filePath),
+					zap.String("error", err.Error()))
+			} else {
+				logging.GetLogger().Info("Removed .id file", zap.String("path", filePath))
+				anyRemoved = true
+			}
+		}
+	}
+	return anyRemoved
+}
+
 func contains(s []string, searchTerm string) bool {
 	i := sort.SearchStrings(s, searchTerm)
 	return i < len(s) && s[i] == searchTerm

internal/backend/version.go

@@ -7,8 +7,10 @@ type VersionData struct {
 
 var data = []VersionData{
 	{
-		Version:   "5.0.0-Beta",
+		Version: "5.0.0",
 		Changelog: []string{
+			"#9 - Major routes refactor and expansion",
+			"#15 - Added API for statistics",
 			"#16 - Upgrade Echo framework from v4 to v5",
 			"#17 - Add Zap structured logging framework",
 			"#18 - Add OpenAPI/Swagger documentation",
@@ -17,6 +19,8 @@ var data = []VersionData{
 			"#21 - Refactor handlers into domain-specific files",
 			"#22 - Change VersionData Changelog from string to string array",
 			"#23 - Update all dependencies to latest versions",
+			"Added search cache backend",
+			"Backend sync improvements",
 		},
 	},
 	{