From df941720e360637b2dca158a5bc8e7d3841a6f42 Mon Sep 17 00:00:00 2001 From: Bram Date: Wed, 11 Feb 2026 21:52:16 +0100 Subject: [PATCH] prefetcher --- Dockers/self-hosted-iptv/iptv/config.py | 4 + Dockers/self-hosted-iptv/iptv/epg_builder.py | 15 +- .../iptv/metadata_prefetcher.py | 137 ++++++++++++++++++ Dockers/self-hosted-iptv/iptv/tmdb_client.py | 37 +++++ Dockers/self-hosted-iptv/main.py | 7 +- 5 files changed, 197 insertions(+), 3 deletions(-) create mode 100644 Dockers/self-hosted-iptv/iptv/metadata_prefetcher.py diff --git a/Dockers/self-hosted-iptv/iptv/config.py b/Dockers/self-hosted-iptv/iptv/config.py index e2a071c..8960542 100644 --- a/Dockers/self-hosted-iptv/iptv/config.py +++ b/Dockers/self-hosted-iptv/iptv/config.py @@ -51,6 +51,10 @@ TMDB_CACHE_TTL_SEC = 24 * 3600 TMDB_RATE_LIMIT_REQUESTS = 40 TMDB_RATE_LIMIT_WINDOW_SEC = 10.0 TMDB_CACHE_DIR = DATA_ROOT / "tmdb_cache" +try: + TMDB_PREFETCH_INTERVAL_SEC = int(os.environ.get("TMDB_PREFETCH_INTERVAL_SEC", str(6 * 3600))) +except (TypeError, ValueError): + TMDB_PREFETCH_INTERVAL_SEC = 6 * 3600 TMDB_IMAGE_BASE = "https://image.tmdb.org/t/p" TMDB_IMAGE_STILL_SIZE = "w500" diff --git a/Dockers/self-hosted-iptv/iptv/epg_builder.py b/Dockers/self-hosted-iptv/iptv/epg_builder.py index e8e7d7e..2215a2e 100644 --- a/Dockers/self-hosted-iptv/iptv/epg_builder.py +++ b/Dockers/self-hosted-iptv/iptv/epg_builder.py @@ -48,11 +48,12 @@ def schedule_epoch_utc() -> datetime: class EPGBuilder: """Builds XMLTV EPG for all channels.""" - def __init__(self, schedule_builder, tmdb_client, channel_repo, cache_manager=None): + def __init__(self, schedule_builder, tmdb_client, channel_repo, cache_manager=None, prefetcher=None): self._schedule = schedule_builder self._tmdb = tmdb_client self._channels = channel_repo self._cache = cache_manager + self._prefetcher = prefetcher def build_xml(self) -> str: """ @@ -86,6 +87,7 @@ class EPGBuilder: ET.SubElement(chan_el, "display-name").text = name total_programmes = 0 + scheduled: set[tuple[Path, str | None, str | None, int | None]] = set() for ch in channels: cid = ch.get("id", ch.get("name", "")) ch_start = time.monotonic() @@ -113,6 +115,14 @@ class EPGBuilder: if start_dt >= end_epoch: break channel_programmes += 1 + cfg = path_configs[i] if i < len(path_configs) else {} + if SERIES_ROOT in path.parents: + scheduled.add(( + path, + cfg.get("display_name"), + cfg.get("tmdb_search"), + cfg.get("tmdb_id"), + )) prog = ET.SubElement( root, "programme", @@ -120,7 +130,6 @@ class EPGBuilder: stop=format_xmltv_time(stop_dt), channel=cid, ) - cfg = path_configs[i] if i < len(path_configs) else {} programme_name = get_programme_name_from_path(path) meta = self._tmdb.get_episode_metadata( programme_name, @@ -194,4 +203,6 @@ class EPGBuilder: ) if self._cache: self._cache.set_epg(xml) + if self._prefetcher: + self._prefetcher.run_once(scheduled) return xml diff --git a/Dockers/self-hosted-iptv/iptv/metadata_prefetcher.py b/Dockers/self-hosted-iptv/iptv/metadata_prefetcher.py new file mode 100644 index 0000000..09b55f9 --- /dev/null +++ b/Dockers/self-hosted-iptv/iptv/metadata_prefetcher.py @@ -0,0 +1,137 @@ +""" +Background metadata prefetcher: periodically scans schedules for programmes +missing TMDB metadata and fetches them, respecting TMDB rate limits. +""" + +import logging +import os +import threading +import time +from pathlib import Path + +from .config import SERIES_ROOT, TMDB_PREFETCH_INTERVAL_SEC +from .epg_builder import get_programme_name_from_path + +logger = logging.getLogger(__name__) + + +class MetadataPrefetcher: + """ + Runs in a background thread, periodically fetching TMDB metadata for + series episodes that are not yet cached locally. + Respects TMDB rate limits via the shared TMDBClient. + """ + + def __init__(self, schedule_builder, tmdb_client, channel_repo): + self._schedule = schedule_builder + self._tmdb = tmdb_client + self._channels = channel_repo + self._interval_sec = TMDB_PREFETCH_INTERVAL_SEC + self._stop = threading.Event() + self._thread: threading.Thread | None = None + + def start(self) -> None: + """Start the background prefetch thread.""" + if not os.environ.get("TMDB_API_KEY", "").strip(): + logger.debug("TMDB_PREFETCH: skipped (no TMDB_API_KEY)") + return + if self._thread is not None and self._thread.is_alive(): + return + self._stop.clear() + self._thread = threading.Thread(target=self._run_loop, daemon=True) + self._thread.start() + logger.info("TMDB metadata prefetcher started (interval: %.1fh)", self._interval_sec / 3600) + + def stop(self) -> None: + """Signal the prefetcher to stop.""" + self._stop.set() + + def _run_loop(self) -> None: + """Background loop: wait, then run a scan.""" + while not self._stop.is_set(): + self._stop.wait(timeout=min(60, self._interval_sec)) + if self._stop.is_set(): + break + if self._stop.wait(timeout=max(0, self._interval_sec - 60)): + break + self._run_scan() + + def _run_scan(self, scheduled: set | None = None) -> None: + """ + Prefetch TMDB metadata for series episodes that are not already + scheduled (not in the current EPG) and not in the TMDB cache. + This builds the cache so the next EPG build will be faster. + TMDBClient rate limiting applies to API calls. + """ + channels = self._channels.get_all() + if not channels: + return + + scheduled = scheduled or set() + seen: set[tuple[Path, str | None, str | None, int | None]] = set() + fetched = 0 + + for ch in channels: + cid = ch.get("id", ch.get("name", "")) + try: + schedule = self._schedule.get_or_build(cid) + except Exception as e: + logger.debug("Prefetch channel %s: %s", cid, e) + continue + if not schedule: + continue + paths, _, path_configs = schedule + + for i, path in enumerate(paths): + if SERIES_ROOT not in path.parents: + continue + cfg = path_configs[i] if i < len(path_configs) else {} + key = ( + path, + cfg.get("display_name"), + cfg.get("tmdb_search"), + cfg.get("tmdb_id"), + ) + if key in seen: + continue + seen.add(key) + if key in scheduled: + continue + + programme_name = get_programme_name_from_path(path) + if self._tmdb.has_episode_metadata_cached( + programme_name, + path, + tmdb_search=cfg.get("tmdb_search"), + tmdb_id=cfg.get("tmdb_id"), + series_root=SERIES_ROOT, + ): + continue + + self._tmdb.get_episode_metadata( + programme_name, + path, + display_name=cfg.get("display_name"), + tmdb_search=cfg.get("tmdb_search"), + tmdb_id=cfg.get("tmdb_id"), + series_root=SERIES_ROOT, + ) + fetched += 1 + + if fetched: + logger.info( + "TMDB prefetch: fetched metadata for %s unscheduled episodes (of %s total)", + fetched, + len(seen), + ) + + def run_once(self, scheduled: set | None = None) -> None: + """ + Run a single prefetch scan immediately (e.g. after EPG build). + scheduled: set of (path, display_name, tmdb_search, tmdb_id) for + programmes already in the current EPG; only fetch for episodes NOT + in this set, so the next scheduling process will be faster. + """ + if not os.environ.get("TMDB_API_KEY", "").strip(): + return + threading.Thread(target=self._run_scan, args=(scheduled or set(),), daemon=True).start() diff --git a/Dockers/self-hosted-iptv/iptv/tmdb_client.py b/Dockers/self-hosted-iptv/iptv/tmdb_client.py index 30719cc..b98f9bb 100644 --- a/Dockers/self-hosted-iptv/iptv/tmdb_client.py +++ b/Dockers/self-hosted-iptv/iptv/tmdb_client.py @@ -266,6 +266,43 @@ class TMDBClient: self._write_disk_cache("episode_credits", key, cred) return cred + def has_episode_metadata_cached( + self, + programme_name: str, + path: Path, + *, + tmdb_search: str | None = None, + tmdb_id: int | None = None, + series_root: Path, + ) -> bool: + """ + Check if episode metadata is already on disk (no API calls). + Used by prefetcher to skip episodes that don't need fetching. + """ + if series_root not in path.parents: + return True + se = parse_season_episode(path.stem) + if not se: + return True + _, episode_num = se + + if tmdb_id is not None: + series_id = tmdb_id + else: + search_term = (tmdb_search or programme_name).strip() or programme_name + key = _sanitize_key(search_term) + cached = self._read_disk_cache("series_search", key) + if cached is None: + return False + sid = cached.get("series_id") + if sid is None: + return False + series_id = int(sid) + + ep_key = f"{series_id}_{se[0]}_{episode_num}" + ep_path = self._cache_path("episode", ep_key) + return ep_path.exists() + def get_episode_metadata( self, programme_name: str, diff --git a/Dockers/self-hosted-iptv/main.py b/Dockers/self-hosted-iptv/main.py index 2f474df..4b58341 100644 --- a/Dockers/self-hosted-iptv/main.py +++ b/Dockers/self-hosted-iptv/main.py @@ -12,6 +12,7 @@ from iptv.cache import CacheManager from iptv.channel import ChannelRepository from iptv.schedule import ScheduleBuilder from iptv.tmdb_client import TMDBClient +from iptv.metadata_prefetcher import MetadataPrefetcher from iptv.epg_builder import EPGBuilder from iptv.server import create_server @@ -39,7 +40,11 @@ def main(): schedule_builder.set_channel_repo(channel_repo) tmdb_client = TMDBClient() cache_manager = CacheManager(schedule_builder=schedule_builder) - epg_builder = EPGBuilder(schedule_builder, tmdb_client, channel_repo, cache_manager) + prefetcher = MetadataPrefetcher(schedule_builder, tmdb_client, channel_repo) + prefetcher.start() + epg_builder = EPGBuilder( + schedule_builder, tmdb_client, channel_repo, cache_manager, prefetcher + ) port = 8080 server = create_server(