fix(DEPLOY-005): resolve pinned universe sources from the repo, not DATA_DIR

.dockerignore

@@ -29,6 +29,10 @@ Dependencies/all_instrument*.csv
 data/cache/
 data/ipo/
 data/universes/*
+# DEPLOY-005: the pinned Hemant symbol lists must reach the image, or
+# refresh_universe_files() cannot rebuild those universes in a container.
+!data/universes/sources/
+!data/universes/sources/*.csv
 !data/universes/hemant_super_45.csv
 !data/universes/hemant_good_45.csv
 !data/universes/hemant_good_200.csv

.gitignore

@@ -7,6 +7,11 @@ Dependencies/all_instrument*.csv
 data/cache/
 data/ipo/
 data/universes/*
+# DEPLOY-005: the pinned Hemant symbol lists are inputs, not generated state.
+# Git will not look inside an ignored directory, so the directory itself has to
+# be un-ignored before the files in it can be.
+!data/universes/sources/
+!data/universes/sources/*.csv
 !data/universes/hemant_super_45.csv
 !data/universes/hemant_good_45.csv
 !data/universes/hemant_good_200.csv

README.md

@@ -1174,9 +1174,12 @@ Supported universe keys are `nifty_100`, `nifty_500`, `fno`,
 `hemant_super_45`, `hemant_good_45`, `hemant_good_200`, and the composites
 `hemant_super_good_union` (Hemant Super 45 ∪ Good 45) and
 `hemant_super_good_200_union` (Hemant Super 45 ∪ Good 45 ∪ Good 200), both deduped.
-The Hemant lists live in `data/universes/` alongside the other universe CSVs
-and are mapped to Dhan cash-equity IDs when universe files are refreshed; the
-union is assembled from those same source lists at refresh time.
+The pinned Hemant symbol lists live in `data/universes/sources/` and are mapped
+to Dhan cash-equity IDs when universe files are refreshed; the union is assembled
+from those same source lists at refresh time. The generated universe CSVs are
+written to `data/universes/` (which follows `DATA_DIR`), while the pinned sources
+are resolved relative to the repository so a container that redirects `DATA_DIR`
+to a volume can still rebuild them (DEPLOY-005).
 
 ---
 

backend/config/__init__.py

@@ -24,6 +24,7 @@
     PROJECT_ROOT,
     REQUEST_HEADERS,
     SCREENERS_DIR,
+    UNIVERSE_SOURCE_DIR,
     AppSettings,
     DhanCredentials,
     SettingsError,
@@ -90,6 +91,7 @@
     "REQUEST_HEADERS",
     "SCREENERS_DIR",
     "UNIVERSE_DIR",
+    "UNIVERSE_SOURCE_DIR",
     "AppSettings",
     "DhanCredentials",
     "SettingsError",

backend/config/settings.py

@@ -30,6 +30,21 @@
 DEFAULT_DATA_DIR = PROJECT_ROOT / "data"
 SCREENERS_DIR = PROJECT_ROOT / "screeners"
 
+# The pinned universe *source* lists (the Hemant Google Doc snapshots) are code,
+# not runtime state: they ship inside the image and are only ever edited by a
+# reviewed commit. They therefore anchor to PROJECT_ROOT and deliberately do NOT
+# follow DATA_DIR.
+#
+# Beginner note (DEPLOY-005):
+# Generated universe CSVs live under `settings.universe_dir`, which follows
+# DATA_DIR so a deployment can point them at a persistent volume. Before
+# DEPLOY-005 the pinned sources were resolved the same way, so a container that
+# set DATA_DIR=/data looked for its source lists on the (empty) volume instead of
+# at /app/data/universes where COPY had actually put them - and
+# `refresh_universe_files()` died with FileNotFoundError. Keeping the two paths
+# on separate anchors is what makes that impossible rather than merely unlikely.
+UNIVERSE_SOURCE_DIR = DEFAULT_DATA_DIR / "universes" / "sources"
+
 # Default Claude model used by the Claude Agent SDK features (Check
 # Fundamentals, Technical Analysis AI, and 67 Ka Funda AI).
 DEFAULT_FUNDAMENTALS_MODEL = "claude-sonnet-4-6"

backend/universe_builder.py

@@ -23,17 +23,31 @@
     DHAN_SCRIP_MASTER_URL,
     NIFTY_100_URL,
     NIFTY_500_URL,
+    PROJECT_ROOT,
     REQUEST_HEADERS,
     UNIVERSE_DIR,
+    UNIVERSE_SOURCE_DIR,
 )
 
 # The Hemant lists are intentionally local CSVs. Unlike NIFTY 100/500, there is
-# no stable public CSV endpoint to download during startup, so we keep the
-# pinned source lists next to the generated universe files in `data/universes/`.
+# no stable public CSV endpoint to download during startup, so the pinned source
+# lists are committed under `data/universes/sources/`.
+#
+# Beginner note (DEPLOY-005):
+# These paths come from UNIVERSE_SOURCE_DIR, which is anchored to PROJECT_ROOT,
+# NOT from UNIVERSE_DIR, which follows DATA_DIR. That separation matters:
+#   * inputs  - pinned, reviewed, shipped inside the image  -> PROJECT_ROOT
+#   * outputs - regenerated every refresh, deployment state -> DATA_DIR
+# They used to be the same directory, which caused two problems. A deployment
+# that set DATA_DIR (Render, docker-compose) looked for its source lists on an
+# empty volume and `refresh_universe_files()` raised FileNotFoundError, taking
+# the daily-scan cron down with it. And locally, each refresh rewrote the very
+# file it had just read, so a generated artifact was also its own source of
+# truth. Keep these two constants distinct.
 HEMANT_SOURCE_FILES: dict[str, Path] = {
-    "hemant_super_45": UNIVERSE_DIR / "hemant_super_45.csv",
-    "hemant_good_45": UNIVERSE_DIR / "hemant_good_45.csv",
-    "hemant_good_200": UNIVERSE_DIR / "hemant_good_200.csv",
+    "hemant_super_45": UNIVERSE_SOURCE_DIR / "hemant_super_45.csv",
+    "hemant_good_45": UNIVERSE_SOURCE_DIR / "hemant_good_45.csv",
+    "hemant_good_200": UNIVERSE_SOURCE_DIR / "hemant_good_200.csv",
 }
 
 # Some symbols in the Hemant source use the Google Doc's naming, while Dhan's
@@ -539,7 +553,10 @@ def build_symbol_list_universe(
     universe = universe.merge(equity_lookup, on="symbol", how="left")
     universe["universe"] = universe_key
     universe["universe_name"] = UNIVERSE_CONFIG[universe_key]["display_name"]
-    universe["source"] = source or UNIVERSE_CONFIG[universe_key].get("source_file", "")
+    configured_source = UNIVERSE_CONFIG[universe_key].get("source_file", "")
+    universe["source"] = source or (
+        repo_relative_source_label(configured_source) if configured_source else ""
+    )
     # Do not alphabetize custom source lists. Their order comes from the pinned
     # CSV snapshot and may be meaningful to the user reviewing the list.
     return finalize_universe(universe, sort_symbols=False)
@@ -584,6 +601,26 @@ def finalize_universe(universe: pd.DataFrame, sort_symbols: bool = True) -> pd.D
     return result.reset_index(drop=True)
 
 
+def repo_relative_source_label(path: Path | str) -> str:
+    """Return a machine-independent label for a pinned source file.
+
+    Beginner note (DEPLOY-005):
+    This string is written into every row's ``source`` column, and the generated
+    Hemant universes are committed. An absolute path would therefore bake one
+    developer's home directory into the repository - noise in every diff, and
+    meaningless inside the container, where the repo lives at /app. A
+    repo-relative POSIX path says the same thing on every machine.
+    """
+    candidate = Path(path)
+    try:
+        return candidate.resolve().relative_to(PROJECT_ROOT).as_posix()
+    except ValueError:
+        # A test (or an operator) pointed at a file outside the repository.
+        # Fall back to the bare file name rather than leaking an unrelated
+        # absolute path into a committed CSV.
+        return candidate.name
+
+
 def universe_file_path(universe_key: str, universe_dir: Path | str = UNIVERSE_DIR) -> Path:
     """Return where a universe CSV should live on disk."""
     return Path(universe_dir) / UNIVERSE_CONFIG[universe_key]["file_name"]
@@ -645,7 +682,7 @@ def refresh_universe_files(
                 universe_key=key,
                 raw_symbols=load_symbol_list_csv(source_file),
                 equity_lookup=equity_lookup,
-                source=str(source_file),
+                source=repo_relative_source_label(source_file),
             )
         else:
             source_df = index_sources.get(key)