feat(mixtape): add dedup_by_similarity and unique_track_count

- unique_track_count returns the count of distinct songs after grouping near-duplicate versions; deterministic, no RNG. - dedup_by_similarity collapses each duplicate group to one survivor picked at random via the supplied RNG. Output preserves input order of survivors. Both share a single bucketed Union-Find pass: tracks are first grouped by exact normalized artist, then unioned within each artist bucket when their normalized titles score >= 0.80 on token_set_ratio. Cross- artist comparisons are skipped to avoid false positives like "Yesterday" by different artists. 12 new tests (30 total in this module) including a 200-seed statistical test confirming every duplicate version can be selected as the winner. cc
2026-04-25 05:34:11 -06:00
1 parent f048ee7645
commit 6649ee0759
@@ -5,6 +5,10 @@
 //!
 //! See spec: qbz-nix-docs/superpowers/specs/2026-04-25-track-shuffle-mix-design.md

+use qbz_models::QueueTrack as CoreQueueTrack;
+use rand::RngExt;
+use std::collections::{BTreeMap, BTreeSet};
+
 /// Tracks whose normalized titles score at or above this Jaro/token-set
 /// threshold are considered the same song (within the same normalized artist
 /// bucket).
@@ -38,6 +42,112 @@ pub fn normalize_artist(s: &str) -> String {
    collapse_whitespace(&unaccented)
 }

+/// Count of distinct songs in `tracks` after similarity-based grouping.
+/// Deterministic — does not use an RNG. Same input always yields the same
+/// count.
+pub fn unique_track_count(tracks: &[CoreQueueTrack]) -> usize {
+    if tracks.is_empty() {
+        return 0;
+    }
+    let groups = build_similarity_groups(tracks);
+    groups.iter().copied().collect::<BTreeSet<usize>>().len()
+}
+
+/// Removes near-duplicate tracks from `tracks`. Two tracks are considered the
+/// same song when their normalized artists match exactly AND their normalized
+/// titles score at or above [`SIMILARITY_THRESHOLD`]. From each duplicate
+/// group, one survivor is picked at random via `rng`.
+///
+/// The output preserves the original input order of the surviving tracks.
+pub fn dedup_by_similarity<R: rand::Rng>(
+    tracks: Vec<CoreQueueTrack>,
+    rng: &mut R,
+) -> Vec<CoreQueueTrack> {
+    if tracks.is_empty() {
+        return tracks;
+    }
+
+    let groups = build_similarity_groups(&tracks);
+
+    // Bucket original indices by their group representative.
+    let mut by_group: BTreeMap<usize, Vec<usize>> = BTreeMap::new();
+    for (i, &g) in groups.iter().enumerate() {
+        by_group.entry(g).or_default().push(i);
+    }
+
+    // Pick one random survivor index per group.
+    let mut survivors: BTreeSet<usize> = BTreeSet::new();
+    for indices in by_group.values() {
+        let chosen = if indices.len() == 1 {
+            indices[0]
+        } else {
+            indices[rng.random_range(0..indices.len())]
+        };
+        survivors.insert(chosen);
+    }
+
+    tracks
+        .into_iter()
+        .enumerate()
+        .filter(|(i, _)| survivors.contains(i))
+        .map(|(_, t)| t)
+        .collect()
+}
+
+/// For each track index, returns the index of the group representative under
+/// the artist-bucketed token-set similarity grouping. Tracks in different
+/// artist buckets always end up in different groups.
+fn build_similarity_groups(tracks: &[CoreQueueTrack]) -> Vec<usize> {
+    let n = tracks.len();
+    let mut parent: Vec<usize> = (0..n).collect();
+
+    // Bucket indices by normalized artist.
+    let mut by_artist: BTreeMap<String, Vec<usize>> = BTreeMap::new();
+    for (i, t) in tracks.iter().enumerate() {
+        by_artist
+            .entry(normalize_artist(&t.artist))
+            .or_default()
+            .push(i);
+    }
+
+    // Within each artist bucket, union pairs whose normalized titles are
+    // similar enough.
+    for indices in by_artist.values() {
+        let titles: Vec<String> = indices
+            .iter()
+            .map(|&i| normalize_title(&tracks[i].title))
+            .collect();
+        for a in 0..indices.len() {
+            for b in (a + 1)..indices.len() {
+                if token_set_ratio(&titles[a], &titles[b]) >= SIMILARITY_THRESHOLD {
+                    uf_union(&mut parent, indices[a], indices[b]);
+                }
+            }
+        }
+    }
+
+    (0..n).map(|i| uf_find(&mut parent, i)).collect()
+}
+
+fn uf_find(parent: &mut [usize], mut x: usize) -> usize {
+    while parent[x] != x {
+        parent[x] = parent[parent[x]]; // path compression (halving)
+        x = parent[x];
+    }
+    x
+}
+
+fn uf_union(parent: &mut [usize], a: usize, b: usize) {
+    let ra = uf_find(parent, a);
+    let rb = uf_find(parent, b);
+    if ra != rb {
+        // Smaller index becomes parent so behavior is order-stable for tests
+        // that don't shuffle.
+        let (root, child) = if ra < rb { (ra, rb) } else { (rb, ra) };
+        parent[child] = root;
+    }
+}
+
 /// Token-set similarity in `[0.0, 1.0]`, modeled on RapidFuzz's
 /// `token_set_ratio`. Inputs are expected to be already normalized.
 pub fn token_set_ratio(a: &str, b: &str) -> f32 {
@@ -152,6 +262,32 @@ fn join_with_intersection(t1: &str, diff: &[&str]) -> String {
 #[cfg(test)]
 mod tests {
    use super::*;
+    use rand::SeedableRng;
+
+    fn mk_track(id: u64, title: &str, artist: &str, album_id: Option<&str>) -> CoreQueueTrack {
+        CoreQueueTrack {
+            id,
+            title: title.to_string(),
+            artist: artist.to_string(),
+            album: String::new(),
+            duration_secs: 0,
+            artwork_url: None,
+            hires: false,
+            bit_depth: None,
+            sample_rate: None,
+            is_local: false,
+            album_id: album_id.map(|s| s.to_string()),
+            artist_id: None,
+            streamable: true,
+            source: None,
+            parental_warning: false,
+            source_item_id_hint: None,
+        }
+    }
+
+    fn deterministic_rng() -> rand::rngs::StdRng {
+        rand::rngs::StdRng::seed_from_u64(42)
+    }

    // ──────── normalize_title ────────

@@ -270,4 +406,148 @@ mod tests {
        // One empty → 0.0 (no overlap).
        assert!(token_set_ratio("", "yesterday") < 1e-6);
    }
+
+    // ──────── unique_track_count ────────
+
+    #[test]
+    fn unique_track_count_zero_for_empty() {
+        assert_eq!(unique_track_count(&[]), 0);
+    }
+
+    #[test]
+    fn unique_track_count_distinct_tracks() {
+        let tracks = vec![
+            mk_track(1, "Yesterday", "The Beatles", Some("a1")),
+            mk_track(2, "Hey Jude", "The Beatles", Some("a1")),
+            mk_track(3, "Let It Be", "The Beatles", Some("a1")),
+        ];
+        assert_eq!(unique_track_count(&tracks), 3);
+    }
+
+    #[test]
+    fn unique_track_count_groups_versions() {
+        let tracks = vec![
+            mk_track(1, "Yesterday", "The Beatles", Some("a1")),
+            mk_track(2, "Yesterday (Live)", "The Beatles", Some("a2")),
+            mk_track(3, "Yesterday - 2003 Remaster", "The Beatles", Some("a3")),
+            mk_track(4, "Hey Jude", "The Beatles", Some("a1")),
+            mk_track(5, "Let It Be", "The Beatles", Some("a1")),
+        ];
+        // 3 versions of "Yesterday" collapse to 1, plus 2 distinct → 3 unique.
+        assert_eq!(unique_track_count(&tracks), 3);
+    }
+
+    #[test]
+    fn unique_track_count_respects_artist_buckets() {
+        let tracks = vec![
+            mk_track(1, "Yesterday", "The Beatles", Some("a1")),
+            mk_track(2, "Yesterday", "Boyz II Men", Some("a2")),
+        ];
+        // Same title, different artists → not deduplicated.
+        assert_eq!(unique_track_count(&tracks), 2);
+    }
+
+    #[test]
+    fn unique_track_count_is_deterministic() {
+        let tracks = vec![
+            mk_track(1, "Yesterday", "The Beatles", Some("a1")),
+            mk_track(2, "Yesterday (Live)", "The Beatles", Some("a2")),
+            mk_track(3, "Hey Jude", "The Beatles", Some("a1")),
+        ];
+        let c1 = unique_track_count(&tracks);
+        let c2 = unique_track_count(&tracks);
+        let c3 = unique_track_count(&tracks);
+        assert_eq!(c1, c2);
+        assert_eq!(c2, c3);
+        assert_eq!(c1, 2);
+    }
+
+    // ──────── dedup_by_similarity ────────
+
+    #[test]
+    fn dedup_empty_returns_empty() {
+        let mut rng = deterministic_rng();
+        assert!(dedup_by_similarity(vec![], &mut rng).is_empty());
+    }
+
+    #[test]
+    fn dedup_single_track_passes_through() {
+        let mut rng = deterministic_rng();
+        let out = dedup_by_similarity(
+            vec![mk_track(1, "Yesterday", "The Beatles", Some("a1"))],
+            &mut rng,
+        );
+        assert_eq!(out.len(), 1);
+        assert_eq!(out[0].id, 1);
+    }
+
+    #[test]
+    fn dedup_keeps_distinct_titles() {
+        let mut rng = deterministic_rng();
+        let tracks = vec![
+            mk_track(1, "Yesterday", "The Beatles", Some("a1")),
+            mk_track(2, "Hey Jude", "The Beatles", Some("a1")),
+            mk_track(3, "Let It Be", "The Beatles", Some("a1")),
+            mk_track(4, "Help!", "The Beatles", Some("a1")),
+            mk_track(5, "Imagine", "The Beatles", Some("a1")),
+        ];
+        let out = dedup_by_similarity(tracks, &mut rng);
+        assert_eq!(out.len(), 5);
+    }
+
+    #[test]
+    fn dedup_collapses_versions() {
+        let mut rng = deterministic_rng();
+        let tracks = vec![
+            mk_track(1, "Yesterday", "The Beatles", Some("a1")),
+            mk_track(2, "Yesterday (Live)", "The Beatles", Some("a2")),
+            mk_track(3, "Yesterday - 2003 Remaster", "The Beatles", Some("a3")),
+        ];
+        let out = dedup_by_similarity(tracks, &mut rng);
+        assert_eq!(out.len(), 1);
+        // The survivor must be one of the three input ids.
+        assert!([1u64, 2, 3].contains(&out[0].id));
+    }
+
+    #[test]
+    fn dedup_respects_artist_buckets() {
+        let mut rng = deterministic_rng();
+        let tracks = vec![
+            mk_track(1, "Yesterday", "The Beatles", Some("a1")),
+            mk_track(2, "Yesterday", "Boyz II Men", Some("a2")),
+        ];
+        let out = dedup_by_similarity(tracks, &mut rng);
+        assert_eq!(out.len(), 2);
+    }
+
+    #[test]
+    fn dedup_random_winner_varies_across_seeds() {
+        // Over many seeds, all three versions should be selected at least once.
+        let mut seen: BTreeSet<u64> = BTreeSet::new();
+        for seed in 0..200u64 {
+            let mut rng = rand::rngs::StdRng::seed_from_u64(seed);
+            let tracks = vec![
+                mk_track(1, "Yesterday", "The Beatles", Some("a1")),
+                mk_track(2, "Yesterday (Live)", "The Beatles", Some("a2")),
+                mk_track(3, "Yesterday - 2003 Remaster", "The Beatles", Some("a3")),
+            ];
+            let out = dedup_by_similarity(tracks, &mut rng);
+            assert_eq!(out.len(), 1);
+            seen.insert(out[0].id);
+        }
+        assert_eq!(seen.len(), 3, "over 200 seeds, all 3 versions should win at least once; got {:?}", seen);
+    }
+
+    #[test]
+    fn dedup_preserves_input_order_of_survivors() {
+        let mut rng = deterministic_rng();
+        let tracks = vec![
+            mk_track(10, "Hey Jude", "The Beatles", Some("a1")),
+            mk_track(20, "Yesterday", "The Beatles", Some("a1")),
+            mk_track(30, "Let It Be", "The Beatles", Some("a1")),
+        ];
+        let out = dedup_by_similarity(tracks, &mut rng);
+        // None of these collapse, so all survive in original order.
+        assert_eq!(out.iter().map(|t| t.id).collect::<Vec<_>>(), vec![10, 20, 30]);
+    }
 }