feat(carousel): YouTube account-cookie auth for caption fetching

.env.example

@@ -4,3 +4,16 @@
 # Pexels API key — used by `npm run pexels:fetch`
 # Get one free at https://www.pexels.com/api/
 PEXELS_API_KEY=your_pexels_api_key_here
+
+# YouTube account cookie for caption fetching (scripts/carousel/CaptionExtractor.js).
+# Helps when anonymous access is blocked/rate-limited (bot-detection, 429s).
+# Value: base64 of a Netscape cookies.txt export (what yt-dlp's --cookies reads;
+# e.g. the "Get cookies.txt LOCALLY" browser extension while logged into youtube.com).
+#   macOS/Linux:  base64 -w0 cookies.txt
+#   (base64url is also accepted.)
+YT_COOKIES_B64=
+
+# When to use the cookie:
+#   fallback (default) — try anonymous first, retry with the cookie only on failure
+#   primary            — attach the cookie from the first request
+YT_COOKIES_MODE=fallback

scripts/__tests__/CaptionExtractor.test.js

@@ -11,6 +11,11 @@ describe('CaptionExtractor', () => {
   beforeEach(() => {
     extractor = new CaptionExtractor();
     fetch.mockClear();
+    // Hermeticity: never let a cookie configured in the runner's shell change
+    // the anonymous-path behaviour these tests assert.
+    delete process.env.YT_COOKIES_B64;
+    delete process.env.YT_COOKIE_B64;
+    delete process.env.YT_COOKIES_MODE;
     const { fetchTranscript } = jest.requireMock('youtube-transcript/dist/youtube-transcript.esm.js');
     fetchTranscript.mockClear();
     // Default: youtube-transcript unavailable so tests exercise the HTML-parsing fallback path
@@ -542,3 +547,99 @@ describe('CaptionExtractor', () => {
     });
   });
 });
+
+describe('CaptionExtractor cookie auth (YT_COOKIES_B64)', () => {
+  const nsLine = (name, value, domain = '.youtube.com') =>
+    [domain, 'TRUE', '/', 'TRUE', '9999999999', name, value].join('\t');
+  const encode = (txt) => Buffer.from(txt, 'utf8').toString('base64url');
+  const COOKIE_ENV = ['YT_COOKIES_B64', 'YT_COOKIE_B64', 'YT_COOKIES_MODE'];
+
+  let extractor;
+  let saved;
+  let fetchTranscript;
+
+  beforeEach(() => {
+    extractor = new CaptionExtractor();
+    fetch.mockClear();
+    ({ fetchTranscript } = jest.requireMock('youtube-transcript/dist/youtube-transcript.esm.js'));
+    fetchTranscript.mockReset();
+    saved = {};
+    for (const k of COOKIE_ENV) { saved[k] = process.env[k]; delete process.env[k]; }
+  });
+
+  afterEach(() => {
+    for (const k of COOKIE_ENV) {
+      if (saved[k] === undefined) delete process.env[k];
+      else process.env[k] = saved[k];
+    }
+  });
+
+  // Regression guard: absent a cookie, the primary tier stays anonymous.
+  test('no cookie: primary tier is called anonymously (no fetch override)', async () => {
+    fetchTranscript.mockResolvedValueOnce([{ offset: 0, duration: 1000, text: 'hi' }]);
+    const result = await extractor.fetchAllCaptions('vid');
+    expect(result).toEqual([{ startSec: 0, endSec: 1, text: 'hi' }]);
+    expect(fetchTranscript.mock.calls[0][1].fetch).toBeUndefined();
+  });
+
+  test('fallback mode: retries the primary tier with the cookie when anonymous yields no captions', async () => {
+    process.env.YT_COOKIES_B64 = encode(`${nsLine('SAPISID', 'S1')}\n${nsLine('SID', 'abc')}`);
+    fetchTranscript
+      .mockResolvedValueOnce([]) // anonymous → empty
+      .mockResolvedValueOnce([{ offset: 0, duration: 1000, text: 'hi' }]); // cookie → captions
+    const result = await extractor.fetchAllCaptions('vid');
+    expect(result).toEqual([{ startSec: 0, endSec: 1, text: 'hi' }]);
+    expect(fetchTranscript.mock.calls[0][1].fetch).toBeUndefined();
+    expect(typeof fetchTranscript.mock.calls[1][1].fetch).toBe('function');
+  });
+
+  test('primary mode: uses the cookie on the first primary-tier call', async () => {
+    process.env.YT_COOKIES_B64 = encode(nsLine('SID', 'abc'));
+    process.env.YT_COOKIES_MODE = 'primary';
+    fetchTranscript.mockResolvedValueOnce([{ offset: 0, duration: 1000, text: 'hi' }]);
+    const result = await extractor.fetchAllCaptions('vid');
+    expect(result).toEqual([{ startSec: 0, endSec: 1, text: 'hi' }]);
+    expect(typeof fetchTranscript.mock.calls[0][1].fetch).toBe('function');
+  });
+
+  // Behavioural (not just call-shape): the fetch handed to the primary tier
+  // must actually attach the account Cookie to real YouTube requests, and must
+  // NOT leak it to lookalike hosts.
+  test('the cookie-retry fetch attaches the account Cookie to youtube.com but not to lookalike hosts', async () => {
+    process.env.YT_COOKIES_B64 = encode(`${nsLine('SAPISID', 'S1')}\n${nsLine('SID', 'abc')}`);
+    fetchTranscript.mockResolvedValueOnce([]).mockResolvedValueOnce([{ offset: 0, duration: 1000, text: 'hi' }]);
+    await extractor.fetchAllCaptions('vid');
+    const cookieFetchFn = fetchTranscript.mock.calls[1][1].fetch;
+    expect(typeof cookieFetchFn).toBe('function');
+
+    fetch.mockClear();
+    fetch.mockResolvedValueOnce({ ok: true });
+    await cookieFetchFn('https://www.youtube.com/api/timedtext?v=vid');
+    expect(fetch.mock.calls[0][1].headers.Cookie).toContain('SAPISID=S1');
+
+    fetch.mockClear();
+    fetch.mockResolvedValueOnce({ ok: true });
+    await cookieFetchFn('https://www.youtube.com.evil.com/');
+    expect(fetch.mock.calls[0][1]?.headers?.Cookie).toBeUndefined();
+  });
+
+  test('fetchViaInnertube: authenticated WEB client sends SAPISIDHASH + Cookie, then falls through to mobile clients', async () => {
+    const auth = { cookieHeader: 'SAPISID=S1; SID=abc', cookies: new Map([['SAPISID', 'S1'], ['SID', 'abc']]) };
+    fetch
+      .mockResolvedValueOnce({ ok: true, json: async () => ({}) }) // WEB POST → no caption tracks
+      .mockResolvedValue({ ok: false }); // mobile ANDROID/IOS clients fail
+    const result = await extractor.fetchViaInnertube('vid', auth, (u, i) => fetch(u, i));
+    const webCall = fetch.mock.calls[0];
+    expect(webCall[0]).toContain('/youtubei/v1/player');
+    expect(webCall[1].headers.Authorization).toMatch(/^SAPISIDHASH /);
+    expect(webCall[1].headers.Cookie).toContain('SAPISID=S1');
+    expect(result).toBeNull();
+  });
+
+  test('no cookie: does NOT retry the primary tier when the first call throws', async () => {
+    fetchTranscript.mockRejectedValueOnce(new Error('boom'));
+    fetch.mockResolvedValue({ ok: false, status: 500 }); // page fetch fallback → throws
+    await expect(extractor.fetchAllCaptions('vid')).rejects.toThrow();
+    expect(fetchTranscript).toHaveBeenCalledTimes(1);
+  });
+});

scripts/__tests__/ytCookies.test.js

@@ -0,0 +1,235 @@
+import { createHash } from 'node:crypto';
+import {
+  decodeCookieB64,
+  parseCookies,
+  sapisidHash,
+  getCookieMode,
+  loadYtCookies,
+  makeCookieFetch,
+} from '../carousel/ytCookies.js';
+
+// Helper: build a Netscape cookies.txt line
+const nsLine = (name, value, domain = '.youtube.com') =>
+  [domain, 'TRUE', '/', 'TRUE', '9999999999', name, value].join('\t');
+
+describe('decodeCookieB64', () => {
+  test('decodes a base64url payload back to the original UTF-8 text', () => {
+    const original = 'SID=abc123; HSID=def456';
+    const b64 = Buffer.from(original, 'utf8').toString('base64url');
+    expect(decodeCookieB64(b64)).toBe(original);
+  });
+
+  test('tolerates surrounding whitespace/newlines in the encoded value', () => {
+    const original = 'SID=abc';
+    const b64 = '  ' + Buffer.from(original, 'utf8').toString('base64url') + '\n';
+    expect(decodeCookieB64(b64)).toBe(original);
+  });
+});
+
+describe('parseCookies', () => {
+  test('parses a Netscape cookies.txt into a cookie header string', () => {
+    const txt = ['# Netscape HTTP Cookie File', nsLine('SID', 'abc123'), nsLine('HSID', 'def456')].join('\n');
+    const { cookies, cookieHeader } = parseCookies(txt);
+    expect(cookies.get('SID')).toBe('abc123');
+    expect(cookies.get('HSID')).toBe('def456');
+    expect(cookieHeader).toBe('SID=abc123; HSID=def456');
+  });
+
+  test('parses a raw Cookie header string', () => {
+    const { cookies, cookieHeader } = parseCookies('SID=abc123; HSID=def456');
+    expect(cookies.get('SID')).toBe('abc123');
+    expect(cookies.get('HSID')).toBe('def456');
+    expect(cookieHeader).toBe('SID=abc123; HSID=def456');
+  });
+
+  test('preserves "=" characters inside cookie values (base64 tokens)', () => {
+    const { cookies } = parseCookies(nsLine('__Secure-3PSID', 'aa==bb=='));
+    expect(cookies.get('__Secure-3PSID')).toBe('aa==bb==');
+  });
+
+  test('ignores comment lines and blank lines', () => {
+    const txt = ['# comment', '', nsLine('SID', 'abc'), '   '].join('\n');
+    const { cookies } = parseCookies(txt);
+    expect(cookies.size).toBe(1);
+    expect(cookies.get('SID')).toBe('abc');
+  });
+
+  test('parses #HttpOnly_-prefixed lines as data (HttpOnly cookies are not comments)', () => {
+    const txt = [
+      '# Netscape HTTP Cookie File',
+      nsLine('SAPISID', 'S1'),                 // plain (not HttpOnly)
+      `#HttpOnly_${nsLine('SID', 'sid1')}`,    // HttpOnly-prefixed
+      `#HttpOnly_${nsLine('__Secure-3PSID', 'p3')}`,
+    ].join('\n');
+    const { cookies } = parseCookies(txt);
+    expect(cookies.get('SAPISID')).toBe('S1');
+    expect(cookies.get('SID')).toBe('sid1');
+    expect(cookies.get('__Secure-3PSID')).toBe('p3');
+  });
+
+  test('parses a cookies.txt that is entirely #HttpOnly_ lines', () => {
+    const txt = [`#HttpOnly_${nsLine('SID', 'sid1')}`, `#HttpOnly_${nsLine('HSID', 'h1')}`].join('\n');
+    const { cookies } = parseCookies(txt);
+    expect(cookies.size).toBe(2);
+    expect(cookies.get('SID')).toBe('sid1');
+    expect(cookies.get('HSID')).toBe('h1');
+  });
+
+  test('raw-header branch splits on the first "=" so base64 padding survives', () => {
+    const { cookies } = parseCookies('__Secure-3PSID=aa==bb==; SID=xyz');
+    expect(cookies.get('__Secure-3PSID')).toBe('aa==bb==');
+    expect(cookies.get('SID')).toBe('xyz');
+  });
+});
+
+describe('sapisidHash', () => {
+  const now = 1700000000;
+  const origin = 'https://www.youtube.com';
+  const expected = (sapisid) =>
+    `SAPISIDHASH ${now}_${createHash('sha1').update(`${now} ${sapisid} ${origin}`).digest('hex')}`;
+
+  test('computes the yt-dlp SAPISIDHASH header deterministically from SAPISID', () => {
+    const cookies = new Map([['SAPISID', 'MYSAPISID']]);
+    expect(sapisidHash(cookies, now, origin)).toBe(expected('MYSAPISID'));
+  });
+
+  test('falls back to __Secure-3PAPISID when SAPISID is absent', () => {
+    const cookies = new Map([['__Secure-3PAPISID', 'THREEP']]);
+    expect(sapisidHash(cookies, now, origin)).toBe(expected('THREEP'));
+  });
+
+  test('returns null when no SAPISID-family cookie is present', () => {
+    const cookies = new Map([['SID', 'abc']]);
+    expect(sapisidHash(cookies, now, origin)).toBeNull();
+  });
+});
+
+describe('getCookieMode', () => {
+  test('defaults to "fallback" when unset', () => {
+    expect(getCookieMode({})).toBe('fallback');
+  });
+
+  test('returns "primary" when YT_COOKIES_MODE=primary (case-insensitive)', () => {
+    expect(getCookieMode({ YT_COOKIES_MODE: 'Primary' })).toBe('primary');
+  });
+
+  test('falls back to "fallback" for unrecognized values', () => {
+    expect(getCookieMode({ YT_COOKIES_MODE: 'nonsense' })).toBe('fallback');
+  });
+});
+
+describe('loadYtCookies', () => {
+  const encode = (txt) => Buffer.from(txt, 'utf8').toString('base64url');
+
+  test('loads and parses auth from YT_COOKIES_B64', () => {
+    const env = { YT_COOKIES_B64: encode(nsLine('SAPISID', 'S1') + '\n' + nsLine('SID', 'abc')) };
+    const auth = loadYtCookies(env);
+    expect(auth).not.toBeNull();
+    expect(auth.cookies.get('SAPISID')).toBe('S1');
+    expect(auth.cookieHeader).toContain('SID=abc');
+  });
+
+  test('falls back to the legacy YT_COOKIE_B64 name', () => {
+    const env = { YT_COOKIE_B64: encode(nsLine('SID', 'abc')) };
+    expect(loadYtCookies(env).cookies.get('SID')).toBe('abc');
+  });
+
+  test('returns null when neither env var is set', () => {
+    expect(loadYtCookies({})).toBeNull();
+  });
+
+  test('returns null when the payload contains no parseable cookies', () => {
+    // random binary that yields no name=value pairs
+    const env = { YT_COOKIES_B64: Buffer.from([0x00, 0x8c, 0xda, 0x4b]).toString('base64url') };
+    expect(loadYtCookies(env)).toBeNull();
+  });
+
+  test('loads a realistic mixed export (plain SAPISID + #HttpOnly_ session cookies)', () => {
+    const txt = [
+      '# Netscape HTTP Cookie File',
+      nsLine('SAPISID', 'S1'),
+      `#HttpOnly_${nsLine('SID', 'sid1')}`,
+      `#HttpOnly_${nsLine('__Secure-3PSID', 'p3')}`,
+    ].join('\n');
+    const auth = loadYtCookies({ YT_COOKIES_B64: encode(txt) });
+    expect(auth.cookies.get('SAPISID')).toBe('S1');
+    expect(auth.cookies.get('SID')).toBe('sid1'); // must NOT be dropped as a comment
+    expect(auth.cookieHeader).toContain('__Secure-3PSID=p3');
+  });
+});
+
+describe('makeCookieFetch', () => {
+  const auth = { cookieHeader: 'SID=abc; SAPISID=S1', cookies: new Map([['SAPISID', 'S1'], ['SID', 'abc']]) };
+
+  test('injects the Cookie header on youtube.com requests', async () => {
+    const spy = jest.fn().mockResolvedValue({ ok: true });
+    const cf = makeCookieFetch(auth, { fetchImpl: spy });
+    await cf('https://www.youtube.com/api/timedtext?v=x');
+    expect(spy.mock.calls[0][1].headers.Cookie).toBe('SID=abc; SAPISID=S1');
+  });
+
+  test('does NOT inject the Cookie header on non-youtube requests', async () => {
+    const spy = jest.fn().mockResolvedValue({ ok: true });
+    const cf = makeCookieFetch(auth, { fetchImpl: spy });
+    await cf('https://example.com/thing');
+    expect(spy.mock.calls[0][1].headers?.Cookie).toBeUndefined();
+  });
+
+  test('adds SAPISIDHASH Authorization + Origin only when withAuth is set', async () => {
+    const spy = jest.fn().mockResolvedValue({ ok: true });
+    const withAuth = makeCookieFetch(auth, { fetchImpl: spy, withAuth: true, now: 1700000000 });
+    await withAuth('https://www.youtube.com/youtubei/v1/player');
+    const h1 = spy.mock.calls[0][1].headers;
+    expect(h1.Authorization).toMatch(/^SAPISIDHASH 1700000000_/);
+    expect(h1.Origin).toBe('https://www.youtube.com');
+
+    spy.mockClear();
+    const noAuth = makeCookieFetch(auth, { fetchImpl: spy });
+    await noAuth('https://www.youtube.com/api/timedtext');
+    expect(spy.mock.calls[0][1].headers.Authorization).toBeUndefined();
+  });
+
+  test('preserves caller-supplied headers', async () => {
+    const spy = jest.fn().mockResolvedValue({ ok: true });
+    const cf = makeCookieFetch(auth, { fetchImpl: spy });
+    await cf('https://www.youtube.com/', { headers: { 'User-Agent': 'UA-X' } });
+    const h = spy.mock.calls[0][1].headers;
+    expect(h['User-Agent']).toBe('UA-X');
+    expect(h.Cookie).toBe('SID=abc; SAPISID=S1');
+  });
+
+  test.each([
+    'https://youtube.com.attacker.net/api/timedtext',
+    'https://www.youtube.com.evil.com/',
+    'https://evil.example/?next=.youtube.com',
+    'https://youtube.com@evil.com/',
+    'https://notyoutube.com/',
+    'https://googlevideo.com.evil.com/',
+  ])('does NOT attach the cookie to hostile/lookalike host: %s', async (url) => {
+    const spy = jest.fn().mockResolvedValue({ ok: true });
+    const cf = makeCookieFetch(auth, { fetchImpl: spy });
+    await cf(url);
+    expect(spy.mock.calls[0][1]?.headers?.Cookie).toBeUndefined();
+  });
+
+  test.each([
+    'https://www.youtube.com/api/timedtext?v=x',
+    'https://m.youtube.com/',
+    'https://youtube.com/',
+    'https://r1---sn-abc.googlevideo.com/videoplayback',
+  ])('attaches the cookie to genuine YouTube host: %s', async (url) => {
+    const spy = jest.fn().mockResolvedValue({ ok: true });
+    const cf = makeCookieFetch(auth, { fetchImpl: spy });
+    await cf(url);
+    expect(spy.mock.calls[0][1].headers.Cookie).toBe('SID=abc; SAPISID=S1');
+  });
+
+  test('merges the account cookie with a caller-supplied Cookie header (both survive)', async () => {
+    const spy = jest.fn().mockResolvedValue({ ok: true });
+    const cf = makeCookieFetch(auth, { fetchImpl: spy });
+    await cf('https://www.youtube.com/api/timedtext', { headers: { Cookie: 'YSC=page123' } });
+    const sent = spy.mock.calls[0][1].headers.Cookie;
+    expect(sent).toContain('YSC=page123');
+    expect(sent).toContain('SAPISID=S1');
+  });
+});