From 77ef3168930b00d619f857a98afe56d7ef4cd5fb Mon Sep 17 00:00:00 2001 From: Hayden <154503486+groupthinking@users.noreply.github.com> Date: Mon, 3 Aug 2026 17:53:18 -0500 Subject: [PATCH] perf: bound L1 cache access-history retention InMemoryCacheLayer appended one timestamp per hit to an unbounded list and only discarded it when the key left the cache, so a resident hot key's history grew with total reads served. The memory was invisible to CacheStats, which counts CacheEntry.size_bytes only, so no budget could reclaim it. Switch access_patterns to deque(maxlen=ACCESS_HISTORY_WINDOW). Retention now scales with resident keys, which the cache already bounds. maxlen evicts in O(1) on append, so the hot path is unchanged. The sole consumer, _calculate_adaptive_ttl, reads only len/[0]/[-1], all of which deque supports. A 500-key x 2000-hit probe drops retained history from 30.6 MiB to 1.35 MiB with resident entries, total_size_bytes and the hot-key TTL all unchanged. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../backend/services/intelligent_cache.py | 18 ++- tests/unit/test_intelligent_cache.py | 117 ++++++++++++++++++ 2 files changed, 133 insertions(+), 2 deletions(-) diff --git a/src/youtube_extension/backend/services/intelligent_cache.py b/src/youtube_extension/backend/services/intelligent_cache.py index d4de006be..6090eb808 100644 --- a/src/youtube_extension/backend/services/intelligent_cache.py +++ b/src/youtube_extension/backend/services/intelligent_cache.py @@ -22,7 +22,7 @@ import statistics import threading import time -from collections import OrderedDict, defaultdict +from collections import OrderedDict, defaultdict, deque from dataclasses import asdict, dataclass # pickle removed for security @@ -56,6 +56,16 @@ def _resolve_tag_write_limit(max_connections: int) -> int: """ return max(1, min(TAG_WRITE_CONCURRENCY, max_connections - TAG_WRITE_POOL_RESERVE)) + +# Hit timestamps retained per key to drive _calculate_adaptive_ttl(). That +# consumer reads only the first element, the last element and the length, so the +# window only has to be long enough for the ratio between them to be a stable +# frequency estimate rather than a two-sample artefact. Bounding it keeps L1's +# footprint proportional to the number of resident keys instead of to the total +# number of reads the process has ever served. +ACCESS_HISTORY_WINDOW = 64 + + class DateTimeEncoder(json.JSONEncoder): """JSON encoder that handles datetime objects""" def default(self, obj): @@ -138,7 +148,11 @@ def __init__(self, name: str = "L1_Memory", max_size: int = 10000, max_size_byte super().__init__(name, max_size) self.max_size_bytes = max_size_bytes self.cache: OrderedDict[str, CacheEntry] = OrderedDict() - self.access_patterns = defaultdict(list) # Track access patterns for intelligent TTL + # Bounded per key: see ACCESS_HISTORY_WINDOW. The whole entry is dropped + # when the key leaves the cache, via _release_entry(). + self.access_patterns = defaultdict( + lambda: deque(maxlen=ACCESS_HISTORY_WINDOW) + ) # Track access patterns for intelligent TTL async def get(self, key: str) -> Optional[Any]: """Get value from in-memory cache""" diff --git a/tests/unit/test_intelligent_cache.py b/tests/unit/test_intelligent_cache.py index d1c7a36c3..98330dc38 100644 --- a/tests/unit/test_intelligent_cache.py +++ b/tests/unit/test_intelligent_cache.py @@ -2123,3 +2123,120 @@ async def test_adaptive_ttl_does_not_treat_reused_key_as_hot(self): "TTL instead of the 3600s base; it inherited its expired " "predecessor's access timestamps and was scored as a hot key" ) + + +# --------------------------------------------------------------------------- +# InMemoryCacheLayer — access-history retention is bounded +# --------------------------------------------------------------------------- + + +class TestAccessHistoryRetentionIsBounded: + """A key that is read repeatedly must not accumulate one timestamp per hit + forever. Before this was bounded, ``access_patterns[key]`` was a plain list + appended to on every cache hit and never trimmed, so a hot key's history + grew without limit for as long as the key stayed resident.""" + + async def test_hot_key_history_stays_within_window(self): + from youtube_extension.backend.services.intelligent_cache import ( + ACCESS_HISTORY_WINDOW, + ) + + layer = InMemoryCacheLayer("hot", max_size=100, max_size_bytes=1024 * 1024) + await layer.set("hot", "value") + + hits = ACCESS_HISTORY_WINDOW * 20 + for _ in range(hits): + await layer.get("hot") + + # Precondition: the key is still resident, so nothing released its + # history behind our back and the count below is the retention policy. + assert "hot" in layer.cache, "key was evicted; this is not a retention test" + + retained = len(layer.access_patterns["hot"]) + assert retained <= ACCESS_HISTORY_WINDOW, ( + f"history for a single resident key grew to {retained} entries " + f"after {hits} hits; expected it to stay within " + f"ACCESS_HISTORY_WINDOW={ACCESS_HISTORY_WINDOW}" + ) + + async def test_window_retains_the_most_recent_timestamps(self): + """Bounding must drop the oldest samples, not the newest, so the + retained window still describes current behaviour.""" + from youtube_extension.backend.services.intelligent_cache import ( + ACCESS_HISTORY_WINDOW, + ) + + layer = InMemoryCacheLayer("recent", max_size=100, max_size_bytes=1024 * 1024) + await layer.set("k", "value") + + for _ in range(ACCESS_HISTORY_WINDOW): + await layer.get("k") + boundary = time.time() + for _ in range(ACCESS_HISTORY_WINDOW): + await layer.get("k") + + retained = list(layer.access_patterns["k"]) + assert retained == sorted(retained), "retained timestamps lost their ordering" + assert all(ts >= boundary for ts in retained), ( + "history retained samples recorded before the most recent " + f"{ACCESS_HISTORY_WINDOW} hits; the window is dropping the wrong end" + ) + + +# --------------------------------------------------------------------------- +# Adaptive TTL still consumes the bounded history +# --------------------------------------------------------------------------- + + +class TestAdaptiveTtlOverBoundedHistory: + """``_calculate_adaptive_ttl`` reads ``accesses[0]``, ``accesses[-1]`` and + ``len(accesses)``. Those must keep working over the bounded container, and + a saturated window must still classify a hot key as high-frequency.""" + + def _make_system(self, layer): + from youtube_extension.backend.services.intelligent_cache import ( + IntelligentCacheSystem, + ) + + system = IntelligentCacheSystem.__new__(IntelligentCacheSystem) + system.layers = [layer] + system.adaptive_ttl_enabled = True + system.cache_warming_enabled = False + system.auto_invalidation_enabled = False + system.performance_history = [] + system.optimization_suggestions = [] + return system + + async def test_saturated_window_still_yields_high_frequency_ttl(self): + """The window holds only the tail of a burst, but that tail is itself a + tight burst, so the frequency estimate must still read as hot.""" + from youtube_extension.backend.services.intelligent_cache import ( + ACCESS_HISTORY_WINDOW, + ) + + layer = InMemoryCacheLayer("ttl", max_size=10, max_size_bytes=1024 * 1024) + await layer.set("k", "value") + # Far more hits than the window holds, all in a tight burst. + for _ in range(ACCESS_HISTORY_WINDOW * 5): + await layer.get("k") + + system = self._make_system(layer) + ttl = system._calculate_adaptive_ttl("k") + assert ttl == 3600 * 4, ( + f"a saturated access window produced a {ttl}s TTL; a key read " + f"{ACCESS_HISTORY_WINDOW * 5} times in a burst should still be " + "classified as high-frequency" + ) + + async def test_indexing_and_len_work_over_bounded_history(self): + """Guards the three container operations _calculate_adaptive_ttl uses. + A container that bounded retention but broke indexing would silently + send every key back to the base TTL.""" + layer = InMemoryCacheLayer("idx", max_size=10, max_size_bytes=1024 * 1024) + await layer.set("k", "value") + for _ in range(5): + await layer.get("k") + + accesses = layer.access_patterns["k"] + assert len(accesses) == 5, "len() over the history container is wrong" + assert accesses[0] <= accesses[-1], "first/last indexing is not ordered"