Add channel database validation and CSV improvements
Enhancements: - Add database validation to settings validation * Checks if at least one database file exists * Validates JSON format of enabled databases * Ensures at least one database is enabled * Reports invalid/malformed database files - Add database information to CSV exports * Show which database was used for each channel match * Add "database_used" column to preview and update CSVs * Include enabled databases list in CSV header comments * Track database source through matching pipeline - Shorten validation success message * Condensed format with key info only * Separate success items and info items * More readable for small notification areas * Shows database count in validation results Changes to _match_streams_to_channel(): - Now returns 5-tuple: (streams, cleaned_name, cleaned_stream_names, match_reason, database_used) - Tracks country_code from channel database entry - Returns "N/A" if channel not found in any database CSV export improvements: - Preview CSV includes database_used column - Update CSV includes database_used column - Header comments show: "Channel Databases Loaded: [list]"
This commit is contained in:
@@ -766,7 +766,11 @@ class Plugin:
|
|||||||
def _match_streams_to_channel(self, channel, all_streams, logger, ignore_tags=None,
|
def _match_streams_to_channel(self, channel, all_streams, logger, ignore_tags=None,
|
||||||
ignore_quality=True, ignore_regional=True, ignore_geographic=True,
|
ignore_quality=True, ignore_regional=True, ignore_geographic=True,
|
||||||
ignore_misc=True, channels_data=None):
|
ignore_misc=True, channels_data=None):
|
||||||
"""Find matching streams for a channel using fuzzy matching when available."""
|
"""Find matching streams for a channel using fuzzy matching when available.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
tuple: (matching_streams, cleaned_channel_name, cleaned_stream_names, match_reason, database_used)
|
||||||
|
"""
|
||||||
if ignore_tags is None:
|
if ignore_tags is None:
|
||||||
ignore_tags = []
|
ignore_tags = []
|
||||||
if channels_data is None:
|
if channels_data is None:
|
||||||
@@ -777,6 +781,9 @@ class Plugin:
|
|||||||
# Get channel info from JSON
|
# Get channel info from JSON
|
||||||
channel_info = self._get_channel_info_from_json(channel_name, channels_data, logger)
|
channel_info = self._get_channel_info_from_json(channel_name, channels_data, logger)
|
||||||
|
|
||||||
|
# Determine which database was used (if any)
|
||||||
|
database_used = channel_info.get('_country_code', 'N/A') if channel_info else 'N/A'
|
||||||
|
|
||||||
# Check if channel name contains "max" (case insensitive) - used for Cinemax handling
|
# Check if channel name contains "max" (case insensitive) - used for Cinemax handling
|
||||||
channel_has_max = 'max' in channel_name.lower()
|
channel_has_max = 'max' in channel_name.lower()
|
||||||
|
|
||||||
@@ -816,7 +823,7 @@ class Plugin:
|
|||||||
) for s in sorted_streams]
|
) for s in sorted_streams]
|
||||||
match_reason = "Callsign match"
|
match_reason = "Callsign match"
|
||||||
|
|
||||||
return sorted_streams, cleaned_channel_name, cleaned_stream_names, match_reason
|
return sorted_streams, cleaned_channel_name, cleaned_stream_names, match_reason, database_used
|
||||||
else:
|
else:
|
||||||
logger.info(f"[Stream-Mapparr] No callsign matches found for {callsign}")
|
logger.info(f"[Stream-Mapparr] No callsign matches found for {callsign}")
|
||||||
# Fall through to fuzzy matching
|
# Fall through to fuzzy matching
|
||||||
@@ -864,11 +871,11 @@ class Plugin:
|
|||||||
) for s in sorted_streams]
|
) for s in sorted_streams]
|
||||||
match_reason = f"Fuzzy match ({match_type}, score: {score})"
|
match_reason = f"Fuzzy match ({match_type}, score: {score})"
|
||||||
|
|
||||||
return sorted_streams, cleaned_channel_name, cleaned_stream_names, match_reason
|
return sorted_streams, cleaned_channel_name, cleaned_stream_names, match_reason, database_used
|
||||||
|
|
||||||
# No fuzzy match found
|
# No fuzzy match found
|
||||||
logger.info(f"[Stream-Mapparr] No fuzzy match found for channel: {channel_name}")
|
logger.info(f"[Stream-Mapparr] No fuzzy match found for channel: {channel_name}")
|
||||||
return [], cleaned_channel_name, [], "No fuzzy match"
|
return [], cleaned_channel_name, [], "No fuzzy match", database_used
|
||||||
|
|
||||||
# Fallback to basic substring matching if fuzzy matcher unavailable
|
# Fallback to basic substring matching if fuzzy matcher unavailable
|
||||||
logger.info(f"[Stream-Mapparr] Using basic substring matching for channel: {channel_name}")
|
logger.info(f"[Stream-Mapparr] Using basic substring matching for channel: {channel_name}")
|
||||||
@@ -876,7 +883,7 @@ class Plugin:
|
|||||||
|
|
||||||
if not all_streams:
|
if not all_streams:
|
||||||
logger.warning("[Stream-Mapparr] No streams available for matching!")
|
logger.warning("[Stream-Mapparr] No streams available for matching!")
|
||||||
return [], cleaned_channel_name, [], "No streams available"
|
return [], cleaned_channel_name, [], "No streams available", database_used
|
||||||
|
|
||||||
# Try exact channel name matching from JSON first
|
# Try exact channel name matching from JSON first
|
||||||
if channel_info and channel_info.get('channel_name'):
|
if channel_info and channel_info.get('channel_name'):
|
||||||
@@ -903,7 +910,7 @@ class Plugin:
|
|||||||
) for s in sorted_streams]
|
) for s in sorted_streams]
|
||||||
match_reason = "Exact match (channels.json)"
|
match_reason = "Exact match (channels.json)"
|
||||||
|
|
||||||
return sorted_streams, cleaned_channel_name, cleaned_stream_names, match_reason
|
return sorted_streams, cleaned_channel_name, cleaned_stream_names, match_reason, database_used
|
||||||
|
|
||||||
# Fallback to basic substring matching
|
# Fallback to basic substring matching
|
||||||
for stream in all_streams:
|
for stream in all_streams:
|
||||||
@@ -926,10 +933,10 @@ class Plugin:
|
|||||||
) for s in sorted_streams]
|
) for s in sorted_streams]
|
||||||
match_reason = "Basic substring match"
|
match_reason = "Basic substring match"
|
||||||
|
|
||||||
return sorted_streams, cleaned_channel_name, cleaned_stream_names, match_reason
|
return sorted_streams, cleaned_channel_name, cleaned_stream_names, match_reason, database_used
|
||||||
|
|
||||||
# No match found
|
# No match found
|
||||||
return [], cleaned_channel_name, [], "No match"
|
return [], cleaned_channel_name, [], "No match", database_used
|
||||||
|
|
||||||
def _get_channel_info_from_json(self, channel_name, channels_data, logger):
|
def _get_channel_info_from_json(self, channel_name, channels_data, logger):
|
||||||
"""Find channel info from channels.json by matching channel name."""
|
"""Find channel info from channels.json by matching channel name."""
|
||||||
@@ -1124,24 +1131,75 @@ class Plugin:
|
|||||||
|
|
||||||
self._initialize_fuzzy_matcher(match_threshold)
|
self._initialize_fuzzy_matcher(match_threshold)
|
||||||
if self.fuzzy_matcher:
|
if self.fuzzy_matcher:
|
||||||
validation_results.append(f"✅ Fuzzy Matcher: SUCCESS - Initialized with threshold {match_threshold}")
|
validation_results.append(f"✅ Fuzzy Matcher: Initialized (threshold: {match_threshold})")
|
||||||
else:
|
else:
|
||||||
validation_results.append("⚠️ Fuzzy Matcher: WARNING - Could not initialize (will use fallback matching)")
|
validation_results.append("⚠️ Fuzzy Matcher: WARNING - Could not initialize (will use fallback matching)")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
validation_results.append(f"⚠️ Fuzzy Matcher: WARNING - {str(e)} (will use fallback matching)")
|
validation_results.append(f"⚠️ Fuzzy Matcher: WARNING - {str(e)} (will use fallback matching)")
|
||||||
|
|
||||||
# 7. Check other settings
|
# 7. Validate Channel Databases
|
||||||
|
logger.info("[Stream-Mapparr] Validating channel databases...")
|
||||||
|
try:
|
||||||
|
databases = self._get_channel_databases()
|
||||||
|
|
||||||
|
if not databases:
|
||||||
|
validation_results.append("❌ Channel Databases: FAILED - No *_channels.json files found in plugin directory")
|
||||||
|
has_errors = True
|
||||||
|
else:
|
||||||
|
# Check which databases are enabled
|
||||||
|
enabled_databases = []
|
||||||
|
invalid_databases = []
|
||||||
|
|
||||||
|
for db_info in databases:
|
||||||
|
db_id = db_info['id']
|
||||||
|
setting_key = f"db_enabled_{db_id}"
|
||||||
|
is_enabled = settings.get(setting_key, db_info['default'])
|
||||||
|
|
||||||
|
if is_enabled:
|
||||||
|
# Validate JSON format
|
||||||
|
try:
|
||||||
|
with open(db_info['file_path'], 'r', encoding='utf-8') as f:
|
||||||
|
file_data = json.load(f)
|
||||||
|
|
||||||
|
# Check format
|
||||||
|
if isinstance(file_data, dict):
|
||||||
|
if 'channels' not in file_data:
|
||||||
|
invalid_databases.append(f"{db_info['label']} (missing 'channels' key)")
|
||||||
|
elif not isinstance(file_data['channels'], list):
|
||||||
|
invalid_databases.append(f"{db_info['label']} ('channels' must be an array)")
|
||||||
|
else:
|
||||||
|
enabled_databases.append(db_info['label'])
|
||||||
|
elif isinstance(file_data, list):
|
||||||
|
enabled_databases.append(db_info['label'])
|
||||||
|
else:
|
||||||
|
invalid_databases.append(f"{db_info['label']} (invalid format)")
|
||||||
|
except json.JSONDecodeError as e:
|
||||||
|
invalid_databases.append(f"{db_info['label']} (JSON error: {str(e)[:50]})")
|
||||||
|
except Exception as e:
|
||||||
|
invalid_databases.append(f"{db_info['label']} (error: {str(e)[:50]})")
|
||||||
|
|
||||||
|
if invalid_databases:
|
||||||
|
validation_results.append(f"❌ Channel Databases: FAILED - Invalid database(s): {', '.join(invalid_databases)}")
|
||||||
|
has_errors = True
|
||||||
|
elif not enabled_databases:
|
||||||
|
validation_results.append("❌ Channel Databases: FAILED - No databases enabled. Enable at least one database in settings.")
|
||||||
|
has_errors = True
|
||||||
|
else:
|
||||||
|
validation_results.append(f"✅ Channel Databases: {len(enabled_databases)} enabled")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
validation_results.append(f"❌ Channel Databases: FAILED - {str(e)}")
|
||||||
|
has_errors = True
|
||||||
|
|
||||||
|
# 8. Check other settings
|
||||||
overwrite_streams = settings.get('overwrite_streams', True)
|
overwrite_streams = settings.get('overwrite_streams', True)
|
||||||
if isinstance(overwrite_streams, str):
|
if isinstance(overwrite_streams, str):
|
||||||
overwrite_streams = overwrite_streams.lower() in ('true', 'yes', '1')
|
overwrite_streams = overwrite_streams.lower() in ('true', 'yes', '1')
|
||||||
validation_results.append(f"ℹ️ Overwrite Existing Streams: {'Enabled' if overwrite_streams else 'Disabled'}")
|
|
||||||
|
|
||||||
ignore_tags_str = settings.get("ignore_tags", "").strip()
|
ignore_tags_str = settings.get("ignore_tags", "").strip()
|
||||||
if ignore_tags_str:
|
if ignore_tags_str:
|
||||||
ignore_tags = self._parse_tags(ignore_tags_str)
|
ignore_tags = self._parse_tags(ignore_tags_str)
|
||||||
validation_results.append(f"ℹ️ Ignore Tags: {len(ignore_tags)} tag(s) configured: {', '.join(repr(tag) for tag in ignore_tags)}")
|
validation_results.append(f"ℹ️ {len(ignore_tags)} ignore tag(s) configured")
|
||||||
else:
|
|
||||||
validation_results.append("ℹ️ Ignore Tags: None configured")
|
|
||||||
|
|
||||||
# Return validation results
|
# Return validation results
|
||||||
return has_errors, validation_results, token
|
return has_errors, validation_results, token
|
||||||
@@ -1162,8 +1220,14 @@ class Plugin:
|
|||||||
message += "\n\nPlease fix the errors above before proceeding."
|
message += "\n\nPlease fix the errors above before proceeding."
|
||||||
return {"status": "error", "message": message}
|
return {"status": "error", "message": message}
|
||||||
else:
|
else:
|
||||||
message = "All settings validated successfully!\n\n" + "\n".join(validation_results)
|
# Condensed success message - only show key items
|
||||||
message += "\n\nYou can now proceed with 'Load/Process Channels'."
|
success_items = [item for item in validation_results if item.startswith("✅")]
|
||||||
|
info_items = [item for item in validation_results if item.startswith("ℹ️")]
|
||||||
|
|
||||||
|
message = "Settings validated! " + " | ".join(success_items)
|
||||||
|
if info_items:
|
||||||
|
message += "\n" + " | ".join(info_items)
|
||||||
|
message += "\n\nReady to proceed with 'Load/Process Channels'."
|
||||||
return {"status": "success", "message": message}
|
return {"status": "success", "message": message}
|
||||||
|
|
||||||
def load_process_channels_action(self, settings, logger):
|
def load_process_channels_action(self, settings, logger):
|
||||||
@@ -1422,6 +1486,21 @@ class Plugin:
|
|||||||
overwrite_streams = overwrite_streams.lower() in ('true', 'yes', '1')
|
overwrite_streams = overwrite_streams.lower() in ('true', 'yes', '1')
|
||||||
fuzzy_match_threshold = settings.get('fuzzy_match_threshold', 85)
|
fuzzy_match_threshold = settings.get('fuzzy_match_threshold', 85)
|
||||||
|
|
||||||
|
# Get enabled databases
|
||||||
|
try:
|
||||||
|
databases = self._get_channel_databases()
|
||||||
|
enabled_dbs = []
|
||||||
|
for db_info in databases:
|
||||||
|
db_id = db_info['id']
|
||||||
|
setting_key = f"db_enabled_{db_id}"
|
||||||
|
is_enabled = settings.get(setting_key, db_info['default'])
|
||||||
|
if is_enabled:
|
||||||
|
enabled_dbs.append(db_info['label'])
|
||||||
|
|
||||||
|
db_info_str = ', '.join(enabled_dbs) if enabled_dbs else 'None'
|
||||||
|
except Exception:
|
||||||
|
db_info_str = 'Unknown'
|
||||||
|
|
||||||
# Build header lines
|
# Build header lines
|
||||||
header_lines = [
|
header_lines = [
|
||||||
f"# Stream-Mapparr Export",
|
f"# Stream-Mapparr Export",
|
||||||
@@ -1435,6 +1514,7 @@ class Plugin:
|
|||||||
f"# Channel Groups: {', '.join(selected_groups) if selected_groups else 'All groups'}",
|
f"# Channel Groups: {', '.join(selected_groups) if selected_groups else 'All groups'}",
|
||||||
f"# Ignore Tags: {', '.join(ignore_tags) if ignore_tags else 'None'}",
|
f"# Ignore Tags: {', '.join(ignore_tags) if ignore_tags else 'None'}",
|
||||||
f"# Visible Channel Limit: {visible_channel_limit}",
|
f"# Visible Channel Limit: {visible_channel_limit}",
|
||||||
|
f"# Channel Databases Loaded: {db_info_str}",
|
||||||
f"#",
|
f"#",
|
||||||
f"# Statistics:",
|
f"# Statistics:",
|
||||||
f"# Total Visible Channels: {total_visible_channels}",
|
f"# Total Visible Channels: {total_visible_channels}",
|
||||||
@@ -1548,7 +1628,7 @@ class Plugin:
|
|||||||
sorted_channels = self._sort_channels_by_priority(group_channels)
|
sorted_channels = self._sort_channels_by_priority(group_channels)
|
||||||
|
|
||||||
# Match streams for this channel group (using first channel as representative)
|
# Match streams for this channel group (using first channel as representative)
|
||||||
matched_streams, cleaned_channel_name, cleaned_stream_names, match_reason = self._match_streams_to_channel(
|
matched_streams, cleaned_channel_name, cleaned_stream_names, match_reason, database_used = self._match_streams_to_channel(
|
||||||
sorted_channels[0], streams, logger, ignore_tags,
|
sorted_channels[0], streams, logger, ignore_tags,
|
||||||
ignore_quality, ignore_regional, ignore_geographic, ignore_misc,
|
ignore_quality, ignore_regional, ignore_geographic, ignore_misc,
|
||||||
channels_data
|
channels_data
|
||||||
@@ -1569,6 +1649,7 @@ class Plugin:
|
|||||||
"stream_names": [s['name'] for s in matched_streams],
|
"stream_names": [s['name'] for s in matched_streams],
|
||||||
"stream_names_cleaned": cleaned_stream_names,
|
"stream_names_cleaned": cleaned_stream_names,
|
||||||
"match_reason": match_reason,
|
"match_reason": match_reason,
|
||||||
|
"database_used": database_used,
|
||||||
"will_update": True
|
"will_update": True
|
||||||
}
|
}
|
||||||
all_matches.append(match_info)
|
all_matches.append(match_info)
|
||||||
@@ -1590,6 +1671,7 @@ class Plugin:
|
|||||||
"stream_names": [s['name'] for s in matched_streams],
|
"stream_names": [s['name'] for s in matched_streams],
|
||||||
"stream_names_cleaned": cleaned_stream_names,
|
"stream_names_cleaned": cleaned_stream_names,
|
||||||
"match_reason": f"Skipped (exceeds limit of {visible_channel_limit})",
|
"match_reason": f"Skipped (exceeds limit of {visible_channel_limit})",
|
||||||
|
"database_used": database_used,
|
||||||
"will_update": False
|
"will_update": False
|
||||||
}
|
}
|
||||||
all_matches.append(match_info)
|
all_matches.append(match_info)
|
||||||
@@ -1626,6 +1708,7 @@ class Plugin:
|
|||||||
'channel_number',
|
'channel_number',
|
||||||
'matched_streams',
|
'matched_streams',
|
||||||
'match_reason',
|
'match_reason',
|
||||||
|
'database_used',
|
||||||
'stream_names'
|
'stream_names'
|
||||||
]
|
]
|
||||||
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
|
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
|
||||||
@@ -1640,6 +1723,7 @@ class Plugin:
|
|||||||
'channel_number': match.get('channel_number', 'N/A'),
|
'channel_number': match.get('channel_number', 'N/A'),
|
||||||
'matched_streams': match['matched_streams'],
|
'matched_streams': match['matched_streams'],
|
||||||
'match_reason': match['match_reason'],
|
'match_reason': match['match_reason'],
|
||||||
|
'database_used': match['database_used'],
|
||||||
'stream_names': '; '.join(match['stream_names']) # Show all streams
|
'stream_names': '; '.join(match['stream_names']) # Show all streams
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -1767,7 +1851,7 @@ class Plugin:
|
|||||||
sorted_channels = self._sort_channels_by_priority(group_channels)
|
sorted_channels = self._sort_channels_by_priority(group_channels)
|
||||||
|
|
||||||
# Match streams for this channel group
|
# Match streams for this channel group
|
||||||
matched_streams, cleaned_channel_name, cleaned_stream_names, match_reason = self._match_streams_to_channel(
|
matched_streams, cleaned_channel_name, cleaned_stream_names, match_reason, database_used = self._match_streams_to_channel(
|
||||||
sorted_channels[0], streams, logger, ignore_tags,
|
sorted_channels[0], streams, logger, ignore_tags,
|
||||||
ignore_quality, ignore_regional, ignore_geographic, ignore_misc,
|
ignore_quality, ignore_regional, ignore_geographic, ignore_misc,
|
||||||
channels_data
|
channels_data
|
||||||
@@ -1821,7 +1905,8 @@ class Plugin:
|
|||||||
update_details.append({
|
update_details.append({
|
||||||
'channel_name': channel_name,
|
'channel_name': channel_name,
|
||||||
'stream_names': stream_names_list,
|
'stream_names': stream_names_list,
|
||||||
'matched_streams': len(matched_streams)
|
'matched_streams': len(matched_streams),
|
||||||
|
'database_used': database_used
|
||||||
})
|
})
|
||||||
|
|
||||||
if overwrite_streams:
|
if overwrite_streams:
|
||||||
@@ -1882,7 +1967,7 @@ class Plugin:
|
|||||||
csvfile.write(header_comment)
|
csvfile.write(header_comment)
|
||||||
|
|
||||||
# Write CSV data
|
# Write CSV data
|
||||||
fieldnames = ['channel_name', 'stream_names', 'matched_streams']
|
fieldnames = ['channel_name', 'stream_names', 'matched_streams', 'database_used']
|
||||||
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
|
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
|
||||||
writer.writeheader()
|
writer.writeheader()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user