From c3e5a7135ac2c351b101ac71843c619536d32aeb Mon Sep 17 00:00:00 2001 From: joren Date: Sun, 5 Apr 2026 22:49:25 +0200 Subject: [PATCH] refactor: performance and code quality improvements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Move `import traceback` to module level (was duplicated in 9 method bodies) - Cache `_get_channel_databases()` result to avoid re-reading JSON files on every call - `_get_all_streams()` now fetches `width`/`height` so dead-stream filtering needs no extra DB queries - `_filter_working_streams()`: replace N individual ORM queries with one bulk `filter(id__in=...)` query - `_sort_streams_by_quality()`: add explicit `prioritize_quality` parameter instead of hidden instance state - `sort_streams_action()`: read and pass `prioritize_quality` from settings - Extract `_parse_channel_file()` helper in fuzzy_matcher.py to eliminate duplicated parsing loop - Simplify `REGIONAL_PATTERNS` — removed verbose `[Ee][Aa][Ss][Tt]` style since `re.IGNORECASE` is always applied Co-Authored-By: Claude Sonnet 4.6 --- Stream-Mapparr/fuzzy_matcher.py | 182 ++++++++++++-------------------- Stream-Mapparr/plugin.py | 179 +++++++++++++++---------------- 2 files changed, 153 insertions(+), 208 deletions(-) diff --git a/Stream-Mapparr/fuzzy_matcher.py b/Stream-Mapparr/fuzzy_matcher.py index 4e99baf..9fab135 100644 --- a/Stream-Mapparr/fuzzy_matcher.py +++ b/Stream-Mapparr/fuzzy_matcher.py @@ -51,31 +51,20 @@ QUALITY_PATTERNS = [ ] # Regional indicator patterns: East, West, Pacific, Central, Mountain, Atlantic +# All patterns are applied with re.IGNORECASE, so no need to spell out both cases. REGIONAL_PATTERNS = [ - # Regional: " East" or " east" (word with space prefix) - r'\s[Ee][Aa][Ss][Tt]', - # Regional: " West" or " west" (word with space prefix) - r'\s[Ww][Ee][Ss][Tt]', - # Regional: " Pacific" or " pacific" (word with space prefix) - r'\s[Pp][Aa][Cc][Ii][Ff][Ii][Cc]', - # Regional: " Central" or " central" (word with space prefix) - r'\s[Cc][Ee][Nn][Tt][Rr][Aa][Ll]', - # Regional: " Mountain" or " mountain" (word with space prefix) - r'\s[Mm][Oo][Uu][Nn][Tt][Aa][Ii][Nn]', - # Regional: " Atlantic" or " atlantic" (word with space prefix) - r'\s[Aa][Tt][Ll][Aa][Nn][Tt][Ii][Cc]', - # Regional: (East) or (EAST) (parenthesized format) - r'\s*\([Ee][Aa][Ss][Tt]\)\s*', - # Regional: (West) or (WEST) (parenthesized format) - r'\s*\([Ww][Ee][Ss][Tt]\)\s*', - # Regional: (Pacific) or (PACIFIC) (parenthesized format) - r'\s*\([Pp][Aa][Cc][Ii][Ff][Ii][Cc]\)\s*', - # Regional: (Central) or (CENTRAL) (parenthesized format) - r'\s*\([Cc][Ee][Nn][Tt][Rr][Aa][Ll]\)\s*', - # Regional: (Mountain) or (MOUNTAIN) (parenthesized format) - r'\s*\([Mm][Oo][Uu][Nn][Tt][Aa][Ii][Nn]\)\s*', - # Regional: (Atlantic) or (ATLANTIC) (parenthesized format) - r'\s*\([Aa][Tt][Ll][Aa][Nn][Tt][Ii][Cc]\)\s*', + r'\sEast', + r'\sWest', + r'\sPacific', + r'\sCentral', + r'\sMountain', + r'\sAtlantic', + r'\s*\(East\)\s*', + r'\s*\(West\)\s*', + r'\s*\(Pacific\)\s*', + r'\s*\(Central\)\s*', + r'\s*\(Mountain\)\s*', + r'\s*\(Atlantic\)\s*', ] # Geographic prefix patterns: US:, USA:, etc. @@ -142,63 +131,68 @@ class FuzzyMatcher: if self.plugin_dir: self._load_channel_databases() + def _parse_channel_file(self, channel_file): + """Parse a single *_channels.json file and append entries to instance collections. + + Returns: + Tuple of (broadcast_count, premium_count) for the file, or (0, 0) on error. + """ + try: + with open(channel_file, 'r', encoding='utf-8') as f: + data = json.load(f) + channels_list = data.get('channels', []) if isinstance(data, dict) else data + + broadcast_count = 0 + premium_count = 0 + + for channel in channels_list: + channel_type = channel.get('type', '').lower() + + if 'broadcast' in channel_type or channel_type == 'broadcast (ota)': + self.broadcast_channels.append(channel) + broadcast_count += 1 + + callsign = channel.get('callsign', '').strip() + if callsign: + self.channel_lookup[callsign] = channel + base_callsign = re.sub(r'-(?:TV|CD|LP|DT|LD)$', '', callsign) + if base_callsign != callsign: + self.channel_lookup[base_callsign] = channel + else: + channel_name = channel.get('channel_name', '').strip() + if channel_name: + self.premium_channels.append(channel_name) + self.premium_channels_full.append(channel) + premium_count += 1 + + self.logger.info( + f"Loaded from {os.path.basename(channel_file)}: " + f"{broadcast_count} broadcast, {premium_count} premium channels" + ) + return broadcast_count, premium_count + + except Exception as e: + self.logger.error(f"Error loading {channel_file}: {e}") + return 0, 0 + def _load_channel_databases(self): """Load all *_channels.json files from the plugin directory.""" pattern = os.path.join(self.plugin_dir, "*_channels.json") channel_files = glob(pattern) - + if not channel_files: self.logger.warning(f"No *_channels.json files found in {self.plugin_dir}") return False - + self.logger.info(f"Found {len(channel_files)} channel database file(s): {[os.path.basename(f) for f in channel_files]}") - + total_broadcast = 0 total_premium = 0 - for channel_file in channel_files: - try: - with open(channel_file, 'r', encoding='utf-8') as f: - data = json.load(f) - # Extract the channels array from the JSON structure - channels_list = data.get('channels', []) if isinstance(data, dict) else data + b, p = self._parse_channel_file(channel_file) + total_broadcast += b + total_premium += p - file_broadcast = 0 - file_premium = 0 - - for channel in channels_list: - channel_type = channel.get('type', '').lower() - - if 'broadcast' in channel_type or channel_type == 'broadcast (ota)': - # Broadcast channel with callsign - self.broadcast_channels.append(channel) - file_broadcast += 1 - - # Create lookup by callsign - callsign = channel.get('callsign', '').strip() - if callsign: - self.channel_lookup[callsign] = channel - - # Also store base callsign without suffix for easier matching - base_callsign = re.sub(r'-(?:TV|CD|LP|DT|LD)$', '', callsign) - if base_callsign != callsign: - self.channel_lookup[base_callsign] = channel - else: - # Premium/cable/national channel - channel_name = channel.get('channel_name', '').strip() - if channel_name: - self.premium_channels.append(channel_name) - self.premium_channels_full.append(channel) - file_premium += 1 - - total_broadcast += file_broadcast - total_premium += file_premium - - self.logger.info(f"Loaded from {os.path.basename(channel_file)}: {file_broadcast} broadcast, {file_premium} premium channels") - - except Exception as e: - self.logger.error(f"Error loading {channel_file}: {e}") - self.logger.info(f"Total channels loaded: {total_broadcast} broadcast, {total_premium} premium") return True @@ -218,13 +212,9 @@ class FuzzyMatcher: self.premium_channels = [] self.premium_channels_full = [] self.channel_lookup = {} - - # Update country_codes tracking self.country_codes = country_codes - # Determine which files to load if country_codes: - # Load only specified country databases channel_files = [] for code in country_codes: file_path = os.path.join(self.plugin_dir, f"{code}_channels.json") @@ -233,7 +223,6 @@ class FuzzyMatcher: else: self.logger.warning(f"Channel database not found: {code}_channels.json") else: - # Load all available databases pattern = os.path.join(self.plugin_dir, "*_channels.json") channel_files = glob(pattern) @@ -245,49 +234,10 @@ class FuzzyMatcher: total_broadcast = 0 total_premium = 0 - for channel_file in channel_files: - try: - with open(channel_file, 'r', encoding='utf-8') as f: - data = json.load(f) - # Extract the channels array from the JSON structure - channels_list = data.get('channels', []) if isinstance(data, dict) else data - - file_broadcast = 0 - file_premium = 0 - - for channel in channels_list: - channel_type = channel.get('type', '').lower() - - if 'broadcast' in channel_type or channel_type == 'broadcast (ota)': - # Broadcast channel with callsign - self.broadcast_channels.append(channel) - file_broadcast += 1 - - # Create lookup by callsign - callsign = channel.get('callsign', '').strip() - if callsign: - self.channel_lookup[callsign] = channel - - # Also store base callsign without suffix for easier matching - base_callsign = re.sub(r'-(?:TV|CD|LP|DT|LD)$', '', callsign) - if base_callsign != callsign: - self.channel_lookup[base_callsign] = channel - else: - # Premium/cable/national channel - channel_name = channel.get('channel_name', '').strip() - if channel_name: - self.premium_channels.append(channel_name) - self.premium_channels_full.append(channel) - file_premium += 1 - - total_broadcast += file_broadcast - total_premium += file_premium - - self.logger.info(f"Loaded from {os.path.basename(channel_file)}: {file_broadcast} broadcast, {file_premium} premium channels") - - except Exception as e: - self.logger.error(f"Error loading {channel_file}: {e}") + b, p = self._parse_channel_file(channel_file) + total_broadcast += b + total_premium += p self.logger.info(f"Total channels loaded: {total_broadcast} broadcast, {total_premium} premium") return True diff --git a/Stream-Mapparr/plugin.py b/Stream-Mapparr/plugin.py index dfa4f49..3525ccf 100644 --- a/Stream-Mapparr/plugin.py +++ b/Stream-Mapparr/plugin.py @@ -8,6 +8,7 @@ import json import csv import os import re +import traceback import urllib.request import urllib.error import time @@ -652,6 +653,7 @@ class Plugin: self.channel_stream_matches = [] self.fuzzy_matcher = None self.saved_settings = {} + self._channel_databases_cache = None LOGGER.info(f"[Stream-Mapparr] {self.name} Plugin v{self.version} initialized") @@ -778,7 +780,6 @@ class Plugin: } except Exception as e: logger.error(f"Error cleaning up periodic tasks: {e}") - import traceback logger.error(f"Traceback: {traceback.format_exc()}") return {"status": "error", "message": f"Error cleaning up periodic tasks: {e}"} @@ -915,8 +916,7 @@ class Plugin: except Exception as e: LOGGER.error(f"[Stream-Mapparr] Error in scheduled scan: {e}") - import traceback - LOGGER.error(f"[Stream-Mapparr] Traceback: {traceback.format_exc()}") + LOGGER.error(f"[Stream-Mapparr] Traceback: {traceback.format_exc()}") # Mark as executed for today's date last_run[scheduled_time] = current_date @@ -1005,7 +1005,9 @@ class Plugin: return None def _get_channel_databases(self): - """Scan for channel database files and return metadata for each.""" + """Scan for channel database files and return metadata for each. Result is cached.""" + if self._channel_databases_cache is not None: + return self._channel_databases_cache plugin_dir = os.path.dirname(__file__) databases = [] try: @@ -1033,6 +1035,7 @@ class Plugin: databases[0]['default'] = True except Exception as e: LOGGER.error(f"[Stream-Mapparr] Error scanning for channel databases: {e}") + self._channel_databases_cache = databases return databases def _resolve_match_threshold(self, settings): @@ -1131,9 +1134,15 @@ class Plugin: def _get_all_streams(self, logger): """Fetch all streams via Django ORM, returning dicts compatible with existing processing logic.""" - return list(Stream.objects.all().values( - 'id', 'name', 'm3u_account', 'channel_group', 'channel_group__name' - )) + fields = ['id', 'name', 'm3u_account', 'channel_group', 'channel_group__name'] + # Include width/height for dead-stream filtering (populated by IPTV Checker) + for field in ('width', 'height'): + try: + Stream._meta.get_field(field) + fields.append(field) + except Exception: + pass + return list(Stream.objects.all().values(*fields)) def _get_all_m3u_accounts(self, logger): """Fetch all M3U accounts via Django ORM.""" @@ -1286,137 +1295,126 @@ class Plugin: return tag return "" - def _sort_streams_by_quality(self, streams): + def _sort_streams_by_quality(self, streams, prioritize_quality=None): """Sort streams by M3U priority first, then by quality using stream_stats (resolution + FPS). - + + Args: + streams: List of stream dicts + prioritize_quality: If True, sort quality before M3U source priority. + Defaults to self._prioritize_quality when None. + Priority: 1. M3U source priority (if specified - lower priority number = higher precedence) 2. Quality tier (High > Medium > Low > Unknown > Dead) 3. Resolution (higher = better) 4. FPS (higher = better) - + Quality tiers: - Tier 0: High quality (>=1280x720 and >=30 FPS) - Tier 1: Medium quality (either HD or good FPS) - Tier 2: Low quality (below HD and below 30 FPS) - Tier 3: Dead streams (0x0 resolution) """ + if prioritize_quality is None: + prioritize_quality = getattr(self, '_prioritize_quality', False) + def get_stream_quality_score(stream): - """Calculate quality score for sorting. - Returns tuple: (m3u_priority, tier, -resolution_pixels, -fps) - Lower values = higher priority - Negative resolution/fps for descending sort - """ - # Get M3U priority (0 = highest, 999 = lowest/unspecified) m3u_priority = stream.get('_m3u_priority', 999) - stats = stream.get('stats', {}) - - # Check for dead stream (0x0) width = stats.get('width', 0) height = stats.get('height', 0) - + if width == 0 or height == 0: - # Tier 3: Dead streams (0x0) - lowest priority - return (m3u_priority, 3, 0, 0) - - # Calculate total pixels + return (m3u_priority, 3, 0, 0) if not prioritize_quality else (3, m3u_priority, 0, 0) + resolution_pixels = width * height - - # Get FPS fps = stats.get('source_fps', 0) - - # Determine quality tier is_hd = width >= 1280 and height >= 720 is_good_fps = fps >= 30 - + if is_hd and is_good_fps: - # Tier 0: High quality (HD + good FPS) tier = 0 elif is_hd or is_good_fps: - # Tier 1: Medium quality (either HD or good FPS) tier = 1 else: - # Tier 2: Low quality (below HD and below 30 FPS) tier = 2 - - # Return tuple for sorting. Behavior depends on user preference: - # - Default: (m3u_priority, tier, -pixels, -fps) -> source first, then quality - # - If prioritize quality first: (tier, m3u_priority, -pixels, -fps) - if getattr(self, '_prioritize_quality', False): + + if prioritize_quality: return (tier, m3u_priority, -resolution_pixels, -fps) else: return (m3u_priority, tier, -resolution_pixels, -fps) - - # Sort streams by M3U priority first, then quality score + return sorted(streams, key=get_stream_quality_score) def _filter_working_streams(self, streams, logger): """ Filter out dead streams (0x0 resolution) based on IPTV Checker metadata. - + + Uses width/height already present in stream dicts (from _get_all_streams) when + available; otherwise falls back to a single bulk ORM query rather than one query + per stream. + Args: streams: List of stream dictionaries to filter logger: Logger instance for output - + Returns: List of working streams (excluding dead ones) """ working_streams = [] dead_count = 0 no_metadata_count = 0 - + + # Check if width/height were already fetched (by _get_all_streams) + sample = streams[0] if streams else {} + has_inline_dims = 'width' in sample and 'height' in sample + + if not has_inline_dims: + # Bulk-fetch resolution data in one query instead of N individual queries + stream_ids = [s['id'] for s in streams] + try: + dim_map = { + obj['id']: (obj.get('width'), obj.get('height')) + for obj in Stream.objects.filter(id__in=stream_ids).values('id', 'width', 'height') + } + except Exception as e: + logger.warning(f"[Stream-Mapparr] Could not bulk-fetch stream dimensions: {e} — including all streams") + return list(streams) + else: + dim_map = None + for stream in streams: stream_id = stream['id'] stream_name = stream.get('name', 'Unknown') - - try: - # Query Stream model for IPTV Checker metadata - stream_obj = Stream.objects.filter(id=stream_id).first() - - if not stream_obj: - # Stream not in database - include it (benefit of doubt) + + if dim_map is not None: + if stream_id not in dim_map: working_streams.append(stream) no_metadata_count += 1 continue - - # Check if stream has been marked dead by IPTV Checker - # IPTV Checker stores width and height as 0 for dead streams - width = getattr(stream_obj, 'width', None) - height = getattr(stream_obj, 'height', None) - - # If width or height is None, IPTV Checker hasn't checked this stream yet - if width is None or height is None: - # No metadata yet - include it (benefit of doubt) - working_streams.append(stream) - no_metadata_count += 1 - continue - - # Check if stream is dead (0x0 resolution) - if width == 0 or height == 0: - # Dead stream - skip it - dead_count += 1 - logger.debug(f"[Stream-Mapparr] Filtered dead stream: '{stream_name}' (ID: {stream_id}, resolution: {width}x{height})") - continue - - # Working stream - include it + width, height = dim_map[stream_id] + else: + width = stream.get('width') + height = stream.get('height') + + if width is None or height is None: working_streams.append(stream) - logger.debug(f"[Stream-Mapparr] Working stream: '{stream_name}' (ID: {stream_id}, resolution: {width}x{height})") - - except Exception as e: - # Error checking stream - include it (benefit of doubt) - logger.warning(f"[Stream-Mapparr] Error checking stream {stream_id} health: {e}, including stream") - working_streams.append(stream) - - # Log summary + no_metadata_count += 1 + continue + + if width == 0 or height == 0: + dead_count += 1 + logger.debug(f"[Stream-Mapparr] Filtered dead stream: '{stream_name}' (ID: {stream_id}, resolution: {width}x{height})") + continue + + working_streams.append(stream) + if dead_count > 0: logger.info(f"[Stream-Mapparr] Filtered out {dead_count} dead streams with 0x0 resolution") - if no_metadata_count > 0: logger.info(f"[Stream-Mapparr] {no_metadata_count} streams have no IPTV Checker metadata (included by default)") - logger.info(f"[Stream-Mapparr] {len(working_streams)} working streams available for matching") - + return working_streams def _wait_for_iptv_checker_completion(self, settings, logger, max_wait_hours=None): @@ -1654,7 +1652,6 @@ class Plugin: except Exception as e: logger.error(f"[Stream-Mapparr] Error building US callsign database: {e}") - import traceback logger.error(traceback.format_exc()) return {} @@ -2283,8 +2280,7 @@ class Plugin: result.get('message', 'Operation failed'), context) except Exception as e: logger.error(f"[Stream-Mapparr] Operation failed: {str(e)}") - import traceback - logger.error(traceback.format_exc()) + logger.error(traceback.format_exc()) error_msg = f'Error: {str(e)}' self._send_progress_update(action, 'error', 0, error_msg, context) finally: @@ -2353,7 +2349,6 @@ class Plugin: except Exception as e: LOGGER.error(f"[Stream-Mapparr] Error in plugin run: {str(e)}") - import traceback LOGGER.error(traceback.format_exc()) return {"status": "error", "message": str(e)} @@ -2476,7 +2471,6 @@ class Plugin: except Exception as e: logger.error(f"[Stream-Mapparr] Error validating settings: {str(e)}") - import traceback logger.error(f"Traceback: {traceback.format_exc()}") validation_results.append(f"❌ Validation error: {str(e)}") has_errors = True @@ -3737,7 +3731,6 @@ class Plugin: except Exception as e: logger.error(f"[Stream-Mapparr] Error in US OTA matching: {str(e)}") - import traceback logger.error(traceback.format_exc()) return {"status": "error", "message": f"Error in US OTA matching: {str(e)}"} @@ -3756,8 +3749,12 @@ class Plugin: profile_name = settings.get('profile_name', '').strip() if not profile_name: return {"status": "error", "message": "Profile name is required"} - + selected_groups_str = settings.get('selected_groups', '').strip() + prioritize_quality = settings.get('prioritize_quality', PluginConfig.DEFAULT_PRIORITIZE_QUALITY) + if isinstance(prioritize_quality, str): + prioritize_quality = prioritize_quality.lower() in ('true', 'yes', '1') + prioritize_quality = bool(prioritize_quality) # Fetch all channels for the profile via ORM logger.info(f"[Stream-Mapparr] Fetching channels for profile: {profile_name}") @@ -3874,9 +3871,9 @@ class Plugin: channel_id = channel['id'] channel_name = channel['name'] streams = channel.get('streams', []) - + # Sort streams by quality - sorted_streams = self._sort_streams_by_quality(streams) + sorted_streams = self._sort_streams_by_quality(streams, prioritize_quality=prioritize_quality) # Check if order changed original_ids = [s['id'] for s in streams] @@ -3990,7 +3987,6 @@ class Plugin: except Exception as e: logger.error(f"[Stream-Mapparr] Error in sort_streams_action: {str(e)}") - import traceback logger.error(f"Traceback: {traceback.format_exc()}") return {"status": "error", "message": f"Error sorting streams: {str(e)}"} @@ -4090,7 +4086,6 @@ class Plugin: except Exception as e: logger.error(f"[Stream-Mapparr] Error managing visibility: {str(e)}") - import traceback logger.error(traceback.format_exc()) return {"status": "error", "message": f"Error: {str(e)}"}