build spotify-to-navidrome migrator with recovery flow
This commit is contained in:
438
internal/recovery/recovery.go
Normal file
438
internal/recovery/recovery.go
Normal file
@@ -0,0 +1,438 @@
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"navimigrate/internal/model"
|
||||
"navimigrate/internal/qobuz"
|
||||
)
|
||||
|
||||
type Searcher interface {
|
||||
SearchTracks(ctx context.Context, query string, limit int) ([]qobuz.Track, error)
|
||||
}
|
||||
|
||||
type TrackMatcher interface {
|
||||
MatchTrack(ctx context.Context, src model.Track) model.MatchedTrack
|
||||
}
|
||||
|
||||
type PlaylistAdder interface {
|
||||
AddTracksToPlaylist(ctx context.Context, playlistID string, trackIDs []string) error
|
||||
}
|
||||
|
||||
type ProgressFunc func(message string)
|
||||
|
||||
type BuildOptions struct {
|
||||
DownloadMissing bool
|
||||
Downloader QobuzDL
|
||||
Progress ProgressFunc
|
||||
}
|
||||
|
||||
type ReaddOptions struct {
|
||||
Force bool
|
||||
Progress ProgressFunc
|
||||
}
|
||||
|
||||
type ReaddSummary struct {
|
||||
Playlists int
|
||||
Candidates int
|
||||
Matched int
|
||||
Added int
|
||||
Errors int
|
||||
}
|
||||
|
||||
func BuildManifestFromReport(ctx context.Context, rep model.TransferReport, searcher Searcher, opts BuildOptions) (Manifest, error) {
|
||||
manifest := Manifest{
|
||||
Version: ManifestVersion,
|
||||
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
DownloadDir: opts.Downloader.OutputDir,
|
||||
QobuzDLPath: opts.Downloader.Path,
|
||||
Playlists: make([]PlaylistManifest, 0, len(rep.Results)),
|
||||
}
|
||||
|
||||
type albumJob struct {
|
||||
ID string
|
||||
}
|
||||
albums := map[string]albumJob{}
|
||||
totalUnmatched := 0
|
||||
for _, pl := range rep.Results {
|
||||
totalUnmatched += len(pl.Unmatched)
|
||||
}
|
||||
processed := 0
|
||||
|
||||
for _, pl := range rep.Results {
|
||||
if len(pl.Unmatched) == 0 {
|
||||
continue
|
||||
}
|
||||
playlistEntry := PlaylistManifest{
|
||||
Name: pl.Name,
|
||||
TargetID: pl.TargetID,
|
||||
Tracks: make([]TrackManifest, 0, len(pl.Unmatched)),
|
||||
}
|
||||
|
||||
for _, um := range pl.Unmatched {
|
||||
processed++
|
||||
notify(opts.Progress, fmt.Sprintf("Qobuz lookup %d/%d: %s - %s", processed, totalUnmatched, short(pl.Name, 36), short(um.Source.Title, 44)))
|
||||
entry := TrackManifest{Source: um.Source}
|
||||
cand, ok, err := findQobuzCandidate(ctx, searcher, um.Source)
|
||||
if err != nil {
|
||||
entry.LookupError = err.Error()
|
||||
} else if ok {
|
||||
entry.QobuzQuery = cand.Query
|
||||
entry.QobuzScore = cand.Score
|
||||
entry.QobuzTrackID = cand.TrackID
|
||||
entry.QobuzAlbumID = cand.AlbumID
|
||||
entry.QobuzAlbumTitle = cand.AlbumTitle
|
||||
entry.QobuzAlbumArtist = cand.AlbumArtist
|
||||
if strings.TrimSpace(cand.AlbumID) != "" {
|
||||
albums[cand.AlbumID] = albumJob{ID: cand.AlbumID}
|
||||
}
|
||||
}
|
||||
playlistEntry.Tracks = append(playlistEntry.Tracks, entry)
|
||||
}
|
||||
manifest.Playlists = append(manifest.Playlists, playlistEntry)
|
||||
}
|
||||
|
||||
if opts.DownloadMissing {
|
||||
albumIDs := make([]string, 0, len(albums))
|
||||
for id := range albums {
|
||||
albumIDs = append(albumIDs, id)
|
||||
}
|
||||
sort.Strings(albumIDs)
|
||||
|
||||
downloadState := map[string]error{}
|
||||
for i, albumID := range albumIDs {
|
||||
notify(opts.Progress, fmt.Sprintf("Qobuz album download %d/%d: %s", i+1, len(albumIDs), albumID))
|
||||
err := opts.Downloader.DownloadAlbum(ctx, albumID)
|
||||
downloadState[albumID] = err
|
||||
}
|
||||
|
||||
for pIdx := range manifest.Playlists {
|
||||
for tIdx := range manifest.Playlists[pIdx].Tracks {
|
||||
entry := &manifest.Playlists[pIdx].Tracks[tIdx]
|
||||
if strings.TrimSpace(entry.QobuzAlbumID) == "" {
|
||||
continue
|
||||
}
|
||||
entry.DownloadAttempted = true
|
||||
if err := downloadState[entry.QobuzAlbumID]; err != nil {
|
||||
entry.Downloaded = false
|
||||
entry.DownloadError = err.Error()
|
||||
} else {
|
||||
entry.Downloaded = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
notify(opts.Progress, "Qobuz missing-track processing complete")
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
func ReaddDownloadedToPlaylists(ctx context.Context, manifest *Manifest, matcher TrackMatcher, adder PlaylistAdder, opts ReaddOptions) (ReaddSummary, error) {
|
||||
summary := ReaddSummary{}
|
||||
for pIdx := range manifest.Playlists {
|
||||
pl := &manifest.Playlists[pIdx]
|
||||
if strings.TrimSpace(pl.TargetID) == "" {
|
||||
continue
|
||||
}
|
||||
summary.Playlists++
|
||||
|
||||
toAdd := make([]string, 0)
|
||||
entryIdxs := make([]int, 0)
|
||||
for tIdx := range pl.Tracks {
|
||||
entry := &pl.Tracks[tIdx]
|
||||
if !opts.Force && entry.Added {
|
||||
continue
|
||||
}
|
||||
if !entry.Downloaded {
|
||||
continue
|
||||
}
|
||||
|
||||
summary.Candidates++
|
||||
entry.RematchAttempted = true
|
||||
res := matcher.MatchTrack(ctx, entry.Source)
|
||||
if !res.Matched || strings.TrimSpace(res.TargetID) == "" {
|
||||
entry.Rematched = false
|
||||
entry.RematchReason = res.Reason
|
||||
if entry.RematchReason == "" {
|
||||
entry.RematchReason = "not found in Navidrome after rescan"
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
entry.Rematched = true
|
||||
entry.RematchReason = ""
|
||||
entry.RematchTrackID = res.TargetID
|
||||
toAdd = append(toAdd, res.TargetID)
|
||||
entryIdxs = append(entryIdxs, tIdx)
|
||||
summary.Matched++
|
||||
}
|
||||
|
||||
if len(toAdd) == 0 {
|
||||
continue
|
||||
}
|
||||
toAdd = uniqueIDs(toAdd)
|
||||
notify(opts.Progress, fmt.Sprintf("Re-add to playlist %s: %d track(s)", pl.Name, len(toAdd)))
|
||||
err := adder.AddTracksToPlaylist(ctx, pl.TargetID, toAdd)
|
||||
if err != nil {
|
||||
summary.Errors++
|
||||
for _, idx := range entryIdxs {
|
||||
entry := &pl.Tracks[idx]
|
||||
entry.AddAttempted = true
|
||||
entry.Added = false
|
||||
entry.AddError = err.Error()
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
for _, idx := range entryIdxs {
|
||||
entry := &pl.Tracks[idx]
|
||||
entry.AddAttempted = true
|
||||
entry.Added = true
|
||||
entry.AddError = ""
|
||||
summary.Added++
|
||||
}
|
||||
}
|
||||
|
||||
manifest.UpdatedAt = time.Now().UTC().Format(time.RFC3339)
|
||||
notify(opts.Progress, "Re-add from downloaded manifest complete")
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
type qobuzCandidate struct {
|
||||
TrackID string
|
||||
AlbumID string
|
||||
AlbumTitle string
|
||||
AlbumArtist string
|
||||
Score float64
|
||||
Query string
|
||||
}
|
||||
|
||||
func findQobuzCandidate(ctx context.Context, searcher Searcher, src model.Track) (qobuzCandidate, bool, error) {
|
||||
queries := buildQueries(src)
|
||||
if len(queries) == 0 {
|
||||
return qobuzCandidate{}, false, nil
|
||||
}
|
||||
|
||||
best := qobuzCandidate{Score: -999}
|
||||
seenTrack := map[string]struct{}{}
|
||||
firstErr := error(nil)
|
||||
errCount := 0
|
||||
for _, q := range queries {
|
||||
res, err := searcher.SearchTracks(ctx, q, 20)
|
||||
if err != nil {
|
||||
errCount++
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
for _, tr := range res {
|
||||
if strings.TrimSpace(tr.ID) == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seenTrack[tr.ID]; ok {
|
||||
continue
|
||||
}
|
||||
seenTrack[tr.ID] = struct{}{}
|
||||
sc := score(src, tr)
|
||||
if sc > best.Score {
|
||||
best = qobuzCandidate{
|
||||
TrackID: tr.ID,
|
||||
AlbumID: tr.AlbumID,
|
||||
AlbumTitle: tr.Album,
|
||||
AlbumArtist: tr.AlbumArtist,
|
||||
Score: sc,
|
||||
Query: q,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if errCount == len(queries) && firstErr != nil {
|
||||
return qobuzCandidate{}, false, fmt.Errorf("qobuz search failed for all queries: %w", firstErr)
|
||||
}
|
||||
|
||||
if strings.TrimSpace(best.TrackID) == "" || strings.TrimSpace(best.AlbumID) == "" {
|
||||
return qobuzCandidate{}, false, nil
|
||||
}
|
||||
if best.Score < 45 {
|
||||
return qobuzCandidate{}, false, nil
|
||||
}
|
||||
return best, true, nil
|
||||
}
|
||||
|
||||
func buildQueries(src model.Track) []string {
|
||||
title := strings.TrimSpace(src.Title)
|
||||
if title == "" {
|
||||
return nil
|
||||
}
|
||||
artist := ""
|
||||
if len(src.Artists) > 0 {
|
||||
artist = strings.TrimSpace(src.Artists[0])
|
||||
}
|
||||
queries := make([]string, 0, 4)
|
||||
if strings.TrimSpace(src.ISRC) != "" {
|
||||
queries = append(queries, strings.ToUpper(strings.TrimSpace(src.ISRC)))
|
||||
}
|
||||
queries = append(queries, strings.TrimSpace(title+" "+artist))
|
||||
queries = append(queries, title)
|
||||
if strings.TrimSpace(src.Album) != "" {
|
||||
queries = append(queries, strings.TrimSpace(title+" "+src.Album+" "+artist))
|
||||
}
|
||||
|
||||
uniq := map[string]struct{}{}
|
||||
out := make([]string, 0, len(queries))
|
||||
for _, q := range queries {
|
||||
q = strings.TrimSpace(q)
|
||||
if q == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := uniq[q]; ok {
|
||||
continue
|
||||
}
|
||||
uniq[q] = struct{}{}
|
||||
out = append(out, q)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func score(src model.Track, dst qobuz.Track) float64 {
|
||||
s := 0.0
|
||||
if strings.TrimSpace(src.ISRC) != "" && strings.EqualFold(strings.TrimSpace(src.ISRC), strings.TrimSpace(dst.ISRC)) {
|
||||
s += 60
|
||||
}
|
||||
s += 25 * similarity(normalize(src.Title), normalize(dst.Title))
|
||||
if len(src.Artists) > 0 {
|
||||
s += 20 * similarity(normalize(src.Artists[0]), normalize(dst.Artist))
|
||||
}
|
||||
if src.DurationMS > 0 && dst.Duration > 0 {
|
||||
delta := math.Abs(float64(src.DurationMS/1000 - dst.Duration))
|
||||
switch {
|
||||
case delta <= 2:
|
||||
s += 10
|
||||
case delta <= 5:
|
||||
s += 7
|
||||
case delta <= 10:
|
||||
s += 4
|
||||
case delta > 25:
|
||||
s -= 6
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func normalize(s string) string {
|
||||
s = strings.ToLower(strings.TrimSpace(s))
|
||||
repl := strings.NewReplacer("&", " and ", "'", "", "-", " ")
|
||||
s = repl.Replace(s)
|
||||
parts := strings.Fields(s)
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
func similarity(a, b string) float64 {
|
||||
if a == "" || b == "" {
|
||||
return 0
|
||||
}
|
||||
if a == b {
|
||||
return 1
|
||||
}
|
||||
ta := tokenSet(a)
|
||||
tb := tokenSet(b)
|
||||
inter := 0
|
||||
for t := range ta {
|
||||
if _, ok := tb[t]; ok {
|
||||
inter++
|
||||
}
|
||||
}
|
||||
if inter == 0 {
|
||||
return 0
|
||||
}
|
||||
jaccard := float64(inter) / float64(len(ta)+len(tb)-inter)
|
||||
lev := levenshteinRatio(a, b)
|
||||
return (jaccard * 0.6) + (lev * 0.4)
|
||||
}
|
||||
|
||||
func tokenSet(s string) map[string]struct{} {
|
||||
set := map[string]struct{}{}
|
||||
for _, p := range strings.Fields(s) {
|
||||
set[p] = struct{}{}
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
func levenshteinRatio(a, b string) float64 {
|
||||
ar := []rune(a)
|
||||
br := []rune(b)
|
||||
if len(ar) == 0 || len(br) == 0 {
|
||||
return 0
|
||||
}
|
||||
d := levenshtein(ar, br)
|
||||
maxLen := len(ar)
|
||||
if len(br) > maxLen {
|
||||
maxLen = len(br)
|
||||
}
|
||||
return 1 - float64(d)/float64(maxLen)
|
||||
}
|
||||
|
||||
func levenshtein(a, b []rune) int {
|
||||
dp := make([]int, len(b)+1)
|
||||
for j := 0; j <= len(b); j++ {
|
||||
dp[j] = j
|
||||
}
|
||||
for i := 1; i <= len(a); i++ {
|
||||
prev := dp[0]
|
||||
dp[0] = i
|
||||
for j := 1; j <= len(b); j++ {
|
||||
tmp := dp[j]
|
||||
cost := 0
|
||||
if a[i-1] != b[j-1] {
|
||||
cost = 1
|
||||
}
|
||||
dp[j] = min3(dp[j]+1, dp[j-1]+1, prev+cost)
|
||||
prev = tmp
|
||||
}
|
||||
}
|
||||
return dp[len(b)]
|
||||
}
|
||||
|
||||
func min3(a, b, c int) int {
|
||||
arr := []int{a, b, c}
|
||||
sort.Ints(arr)
|
||||
return arr[0]
|
||||
}
|
||||
|
||||
func uniqueIDs(ids []string) []string {
|
||||
seen := map[string]struct{}{}
|
||||
out := make([]string, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
continue
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
out = append(out, id)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func notify(fn ProgressFunc, msg string) {
|
||||
if fn != nil {
|
||||
fn(msg)
|
||||
}
|
||||
}
|
||||
|
||||
func short(s string, max int) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if max <= 3 || len(s) <= max {
|
||||
return s
|
||||
}
|
||||
return s[:max-3] + "..."
|
||||
}
|
||||
Reference in New Issue
Block a user