From 855f9bea2405c62cb2c318c9118eac06d216aee0 Mon Sep 17 00:00:00 2001 From: Renamr Scaffold Date: Sat, 29 Aug 2026 03:27:19 +0000 Subject: [PATCH] Disambiguate Fetch Data's TMDB search results by name similarity - New crates/renamr-core/src/similarity module: a partial port of the scoring idea behind FileBot's EpisodeMetrics.java - normalizes titles (lowercase, strips a trailing release-year parenthetical, collapses punctuation/whitespace) and scores them with a Sorensen-Dice coefficient over character bigrams. best_title_match() picks whichever candidate is textually closest to the query, provided it clears a minimum threshold; ties keep the earlier (i.e. TMDB-ranked) candidate. - TmdbClient::lookup_episode now uses best_title_match instead of blindly taking the first search result, so a more popular but differently-named show can no longer beat an exact-but-lower-ranked match to the top - and reports NotFound instead of guessing when nothing in the results is a plausible match at all. - 7 new tests (5 in similarity, 2 in tmdb) including the concrete regression case this fixes: searching "Chuck" with TMDB returning "Chucky" ranked above "Chuck" now correctly picks "Chuck". - Also fixes a gitignore gap from last delivery: the _to_delete*/ scratch-dir pattern used to work around this repo's mounted-folder git-lock quirk had only ever been applied directly on the device, never carried back into the zip this delivery is built from - so the last delivery's cp -rf silently reintroduced the untracked-lock-file risk. Fixed in both places now. --- .gitignore | 3 + README.md | 53 ++++---- crates/renamr-core/src/lib.rs | 1 + crates/renamr-core/src/similarity/mod.rs | 158 +++++++++++++++++++++++ crates/renamr-core/src/web/tmdb.rs | 80 ++++++++++-- 5 files changed, 264 insertions(+), 31 deletions(-) create mode 100644 crates/renamr-core/src/similarity/mod.rs diff --git a/.gitignore b/.gitignore index a5dec15..f432b58 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,7 @@ /src-tauri/gen *.log .DS_Store +# Scratch subfolders used to shuffle stale git lock files out of the way when +# committing from the device_bash mounted-folder bridge (see project memory / +# README delivery notes) - never meant to be tracked. _to_delete*/ diff --git a/README.md b/README.md index 6cb936d..4931333 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,12 @@ fork), built from scratch as its own project: new stack, new architecture, new b engine is ported and tested, the faithfully-recreated two-pane UI is wired to real native file/folder loading and drag-and-drop, local filename-based season/episode detection ("Match") is working, real online metadata lookups against TMDB ("Fetch -Data") are working, and the actual on-disk rename operation is wired up (same-directory -rename, skip-on-conflict, with a confirmation step showing the full old-name/new-name -plan before anything touches disk — there is no undo yet). Full media-metadata -bindings and the fuzzy similarity/matching engine are not yet ported. See "Porting -roadmap" below. +Data") are working — including fuzzy show-name disambiguation, so a search that +returns several similarly-named shows doesn't just take TMDB's top-ranked hit — and the +actual on-disk rename operation is wired up (same-directory rename, skip-on-conflict, +with a confirmation step showing the full old-name/new-name plan before anything +touches disk — there is no undo yet). Full media-metadata bindings are not yet ported. +See "Porting roadmap" below. The naming scheme is fully user-editable from the app (the "Naming Scheme" button next to the New Names list) — the default is `{show name} - {season number}x{episode @@ -81,14 +82,16 @@ rendered output would be empty. ``` crates/renamr-core/ Pure Rust core: naming-expression engine (ported), local - filename-based season/episode matching (ported, `matching/`), a - TMDB metadata client (`web/tmdb.rs` — TV search + episode lookup, - tested against a mocked HTTP server via `mockito`), and the - on-disk rename engine (ported for the same-directory case, - `rename/`) — skip-on-conflict, no move/organize-into-folders yet. - Not yet ported: the fuzzy similarity engine, full media metadata - bindings, other web datasource clients (TVDB/AniDB/etc). No Tauri - dependency, so it builds and unit-tests standalone. + filename-based season/episode matching (ported, `matching/`), + text-similarity scoring for disambiguating search results + (`similarity/`, partial `EpisodeMetrics` port), a TMDB metadata + client (`web/tmdb.rs` — TV search + episode lookup, disambiguated + via `similarity/`, tested against a mocked HTTP server via + `mockito`), and the on-disk rename engine (ported for the + same-directory case, `rename/`) — skip-on-conflict, no + move/organize-into-folders yet. Not yet ported: full media + metadata bindings, other web datasource clients (TVDB/AniDB/etc). + No Tauri dependency, so it builds and unit-tests standalone. src-tauri/ Tauri app shell: exposes renamr-core as commands to the frontend (naming-scheme rendering, filename matching, TMDB metadata fetch, executing a rename plan, native file/folder dialogs, drag-and-drop @@ -160,21 +163,25 @@ ported from, for anyone picking this up: 3. **Matching/similarity engine** — `net.filebot.similarity.*`. Local filename-based season/episode detection is done (`crates/renamr-core/src/matching`, ported from the pattern-matching half of `SeasonEpisodeMatcher.java`, adapted to Rust's `regex` crate - which lacks lookahead/lookbehind — wired to the "Match" button). Now that TMDB is - wired up (see item 5 below), `EpisodeMetrics.java` (849 lines, the fuzzy title-matching - score cascade) is the next thing this actually blocks on: today "Fetch Data" just - takes TMDB's first (TMDB-ranked) search result, which is right most of the time but - has no way to catch a wrong pick among several similarly-named shows. `DateMatcher` - also still not ported. + which lacks lookahead/lookbehind — wired to the "Match" button). Show-name + disambiguation is also done (`crates/renamr-core/src/similarity`, a partial port of + the *scoring* idea behind `EpisodeMetrics.java`'s fuzzy title-matching cascade — + Sorensen-Dice similarity over normalized-title character bigrams): "Fetch Data" now + picks whichever TMDB search result is textually closest to the local guess rather + than just trusting TMDB's relevance/popularity-ranked first hit, and reports "not + found" instead of guessing when nothing in the results is a plausible match at all. + `DateMatcher` (numbering/air-date-based identification, the other half of the + original `EpisodeMetrics`) is not ported — Renamr doesn't do date-based episode + identification. 4. **Media detection** — `net.filebot.media.MediaDetection` (1356 lines): ties file-name parsing + matching together to auto-identify what a file is. 5. **Web datasource clients** — `net.filebot.web.*` (~40 files): TheMovieDB, TheTVDB, AniDB, OMDb, TVmaze, OpenSubtitles, etc. TMDB is done for TV (`crates/renamr-core/src/web/tmdb.rs`, `reqwest` + `serde_json`, Bearer-token auth) — search-by-name plus per-season episode - lookup, wired to "Fetch Data". Picks the first (TMDB-ranked) search result rather - than disambiguating between multiple plausible shows — a real fix needs the - `EpisodeMetrics` fuzzy-matching port above. TMDB movie lookup and all other sources - (TheTVDB, AniDB, OMDb, TVmaze, OpenSubtitles) are not yet started. + lookup, wired to "Fetch Data", disambiguating multi-result searches via the + similarity-scoring pass described in item 3 above rather than blindly trusting + TMDB's top-ranked hit. TMDB movie lookup and all other sources (TheTVDB, AniDB, + OMDb, TVmaze, OpenSubtitles) are not yet started. 6. **Rename engine + CLI** — `net.filebot.cli.CmdlineOperations` (1235 lines) and `ArgumentBean` (415 lines, the full flag surface: `-rename`, `--db`, `--format`, `--action`, `--conflict`, `-get-subtitles`, `-script`, etc.). The core same-directory diff --git a/crates/renamr-core/src/lib.rs b/crates/renamr-core/src/lib.rs index 8e8d8c5..1884199 100644 --- a/crates/renamr-core/src/lib.rs +++ b/crates/renamr-core/src/lib.rs @@ -8,6 +8,7 @@ pub mod format; pub mod matching; pub mod rename; +pub mod similarity; pub mod web; use rhai::Dynamic; diff --git a/crates/renamr-core/src/similarity/mod.rs b/crates/renamr-core/src/similarity/mod.rs new file mode 100644 index 0000000..741adfb --- /dev/null +++ b/crates/renamr-core/src/similarity/mod.rs @@ -0,0 +1,158 @@ +//! Lightweight text-similarity scoring, used to disambiguate between several search +//! results for the same query. A partial port of the *scoring* idea behind FileBot's +//! `net.filebot.similarity.EpisodeMetrics` (which combines several metrics: name, +//! episode numbering, air date, etc). This only ports the show-name half: given a +//! search query and a list of candidate titles, it picks the title that's the closest +//! textual match rather than trusting whichever result an external API happened to +//! rank first - TMDB ranks by relevance/popularity, not by exact name match, so a +//! more popular but differently-named show can otherwise beat an exact-but-less-popular +//! one straight to the top of the list. `DateMatcher` (the other half of the original +//! `EpisodeMetrics`) is not ported - Renamr doesn't do date-based episode identification. + +use std::collections::HashMap; + +/// Lowercases, strips a trailing 4-digit-year parenthetical (e.g. "(2013)" - release +/// metadata that never appears in a real filename, but commonly appears in TMDB's own +/// titles for remakes/reboots), and collapses every run of punctuation/whitespace down +/// to a single space, so two titles that differ only in casing, punctuation, or an +/// appended year compare as equal. +pub fn normalize_title(input: &str) -> String { + let mut without_year = input.to_string(); + if let Some(start) = without_year.rfind('(') { + let inner = &without_year[start + 1..]; + if let Some(end) = inner.find(')') { + let candidate = &inner[..end]; + if candidate.len() == 4 && candidate.chars().all(|c| c.is_ascii_digit()) { + without_year.replace_range(start..start + end + 2, ""); + } + } + } + + let mut out = String::with_capacity(without_year.len()); + let mut last_was_space = true; // avoid a leading space in the output + for c in without_year.chars() { + if c.is_alphanumeric() { + out.extend(c.to_lowercase()); + last_was_space = false; + } else if !last_was_space { + out.push(' '); + last_was_space = true; + } + } + out.trim_end().to_string() +} + +/// Character-bigram multiset of a string, e.g. "abc" -> {ab: 1, bc: 1}. +fn bigrams(s: &str) -> HashMap<(char, char), u32> { + let chars: Vec = s.chars().collect(); + let mut map = HashMap::new(); + if chars.len() < 2 { + // Too short for a real bigram - fall back to a single-char "bigram" so a + // one-letter title still contributes something rather than always scoring 0. + if let Some(&c) = chars.first() { + *map.entry((c, '\0')).or_insert(0) += 1; + } + return map; + } + for w in chars.windows(2) { + *map.entry((w[0], w[1])).or_insert(0) += 1; + } + map +} + +/// Sorensen-Dice coefficient over character bigrams of the normalized titles: 1.0 for +/// an exact match (after normalization), trending toward 0.0 for unrelated strings, +/// while staying tolerant of the small differences (punctuation, word order, a +/// trailing "Part 2") that would sink a naive exact-string comparison. +pub fn title_similarity(a: &str, b: &str) -> f64 { + let na = normalize_title(a); + let nb = normalize_title(b); + if na == nb { + return 1.0; + } + + let ba = bigrams(&na); + let bb = bigrams(&nb); + let total: u32 = ba.values().sum::() + bb.values().sum::(); + if total == 0 { + return 0.0; + } + + let mut overlap = 0u32; + for (bigram, &count) in &ba { + if let Some(&other_count) = bb.get(bigram) { + overlap += count.min(other_count); + } + } + (2.0 * overlap as f64) / total as f64 +} + +/// Picks whichever candidate's title is textually closest to `query`, provided its +/// score clears `threshold`. Below that, none of the candidates are considered a +/// plausible match for the query at all and `None` is returned - it's better to +/// report "not found" than to confidently rename files against the wrong show. Ties +/// keep the earlier candidate, which preserves the caller's own ranking (e.g. TMDB's +/// relevance/popularity order) as the tiebreaker when two titles score identically. +pub fn best_title_match<'a, T>( + query: &str, + candidates: &'a [T], + title_of: impl Fn(&T) -> &str, + threshold: f64, +) -> Option<&'a T> { + let mut best: Option<(&'a T, f64)> = None; + for candidate in candidates { + let score = title_similarity(query, title_of(candidate)); + if score < threshold { + continue; + } + match best { + Some((_, best_score)) if score <= best_score => {} + _ => best = Some((candidate, score)), + } + } + best.map(|(candidate, _)| candidate) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn exact_match_after_case_and_punctuation_normalization_scores_1() { + assert_eq!(title_similarity("Breaking Bad", "breaking-bad"), 1.0); + assert_eq!(title_similarity("The Office (US)", "the office, us!"), 1.0); + } + + #[test] + fn trailing_release_year_parenthetical_is_ignored() { + assert_eq!(title_similarity("Chernobyl", "Chernobyl (2019)"), 1.0); + } + + #[test] + fn unrelated_titles_score_low() { + let score = title_similarity("Zzyzx Nonexistent Show", "Totally Unrelated Program"); + assert!(score < 0.3, "expected a low score, got {score}"); + } + + #[test] + fn best_title_match_prefers_the_closer_textual_match_over_list_order() { + let candidates = vec!["Chucky".to_string(), "Chuck".to_string()]; + let best = best_title_match("Chuck", &candidates, |s| s.as_str(), 0.3); + assert_eq!(best.map(String::as_str), Some("Chuck")); + } + + #[test] + fn best_title_match_keeps_the_earlier_candidate_on_a_tie() { + let candidates = vec!["Show A".to_string(), "Show A".to_string()]; + let best = best_title_match("Show A", &candidates, |s| s.as_str(), 0.3); + // Both score 1.0 - the first one (by identity, not just value) should win. + assert!(best.is_some_and(|s| std::ptr::eq(s, &candidates[0]))); + } + + #[test] + fn best_title_match_returns_none_when_nothing_clears_the_threshold() { + let candidates = vec!["Totally Unrelated Program".to_string()]; + let best = best_title_match("Zzyzx Nonexistent Show", &candidates, |s| s.as_str(), 0.3); + assert_eq!(best, None); + } +} diff --git a/crates/renamr-core/src/web/tmdb.rs b/crates/renamr-core/src/web/tmdb.rs index 32980f7..f1d6e50 100644 --- a/crates/renamr-core/src/web/tmdb.rs +++ b/crates/renamr-core/src/web/tmdb.rs @@ -11,8 +11,17 @@ use serde::Deserialize; use thiserror::Error; +use crate::similarity::best_title_match; + const DEFAULT_BASE_URL: &str = "https://api.themoviedb.org/3"; +/// Minimum name-similarity score (see `similarity::title_similarity`) a TMDB search +/// result must clear to be considered a plausible match for the query at all. Below +/// this, `lookup_episode` reports `NotFound` rather than confidently picking whichever +/// result TMDB happened to rank first - popularity/relevance ranking is not the same +/// thing as an accurate name match. +const NAME_MATCH_THRESHOLD: f64 = 0.35; + #[derive(Debug, Error)] pub enum TmdbError { #[error("network error talking to TMDB: {0}")] @@ -153,10 +162,12 @@ impl TmdbClient { } /// Convenience: search for the best-matching show by name, then fetch a specific - /// episode. Picks the first search result - good enough for v1 since TMDB already - /// ranks by relevance; a proper fuzzy-matching disambiguation pass (porting - /// `EpisodeMetrics`) is still on the roadmap for when a search returns multiple - /// plausible shows. + /// episode. Disambiguates a multi-result search by picking whichever result's name + /// is textually closest to `series_query` (see `similarity::best_title_match`) + /// rather than just trusting TMDB's own top-ranked (relevance/popularity) result - + /// so a more popular but differently-named show doesn't get picked over an exact + /// match that's merely ranked lower. If nothing in the results is a plausible + /// match at all, reports `NotFound` instead of guessing. pub async fn lookup_episode( &self, series_query: &str, @@ -164,14 +175,12 @@ impl TmdbClient { episode: i32, ) -> Result { let results = self.search_tv(series_query).await?; - let best = results - .into_iter() - .next() + let best = best_title_match(series_query, &results, |r| r.name.as_str(), NAME_MATCH_THRESHOLD) .ok_or_else(|| TmdbError::NotFound(series_query.to_string()))?; let info = self.episode(best.id, season, episode).await?; Ok(EpisodeInfo { - series_name: best.name, + series_name: best.name.clone(), episode_title: info.episode_title, air_date: info.air_date, }) @@ -283,6 +292,61 @@ mod tests { assert!(matches!(result, Err(TmdbError::NotFound(_)))); } + #[tokio::test] + async fn picks_the_exact_name_match_over_a_more_popular_but_wrong_result() { + let mut server = mockito::Server::new_async().await; + + // TMDB ranks results by relevance/popularity, not exact-name-match - simulate + // it putting a differently-named (but more popular) show first, with the + // actually-correct exact match ranked second. Before the similarity-based + // disambiguation, `lookup_episode` just took `.next()` and would've picked + // "Chucky" here, which is wrong. + let _search_mock = server + .mock("GET", "/search/tv") + .match_query(mockito::Matcher::UrlEncoded("query".into(), "Chuck".into())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"results":[{"id":1,"name":"Chucky"},{"id":2,"name":"Chuck"}]}"#) + .create_async() + .await; + + let _season_mock = server + .mock("GET", "/tv/2/season/1") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"episodes":[{"episode_number":1,"name":"Chuck Versus the Intersect","air_date":"2007-09-24"}]}"#) + .create_async() + .await; + + let client = TmdbClient::new("test-token").with_base_url(server.url()); + let info = client.lookup_episode("Chuck", 1, 1).await.unwrap(); + + assert_eq!(info.series_name, "Chuck"); + assert_eq!(info.episode_title, "Chuck Versus the Intersect"); + } + + #[tokio::test] + async fn reports_not_found_when_no_search_result_is_a_plausible_name_match() { + let mut server = mockito::Server::new_async().await; + + // TMDB returned *something*, but nothing in the results is actually close to + // the query - should report NotFound rather than confidently renaming against + // an unrelated show. + let _mock = server + .mock("GET", "/search/tv") + .match_query(mockito::Matcher::Any) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"results":[{"id":9,"name":"Totally Unrelated Program"}]}"#) + .create_async() + .await; + + let client = TmdbClient::new("test-token").with_base_url(server.url()); + let result = client.lookup_episode("Zzyzx Nonexistent Show", 1, 1).await; + + assert!(matches!(result, Err(TmdbError::NotFound(_)))); + } + #[tokio::test] async fn surfaces_an_invalid_token_as_an_api_error() { let mut server = mockito::Server::new_async().await;