From 84759392e1e54cb641f13046160908f4131e428f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 8 Sep 2026 22:49:05 +0000 Subject: [PATCH 1/2] Initial plan From 99230ec84234a1db856f693f417060108e664dec Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 8 Sep 2026 23:56:18 +0000 Subject: [PATCH 2/2] fix: migrate transcript clients to proxy-aware v1 APIs Co-authored-by: groupthinking <154503486+groupthinking@users.noreply.github.com> --- shared/libs/youtube_proxy.py | 26 ++++- src/agents/interactive_metadata_extractor.py | 7 +- src/agents/markdown_video_processor.py | 10 +- src/agents/process_video_with_mcp.py | 19 +++- src/integration/youtube_api.py | 6 +- src/mcp/mcp_video_processor.py | 9 +- .../services/youtube/adapters/official_api.py | 11 ++- .../services/youtube/adapters/robust.py | 26 ++++- .../processors/enhanced_extractor.py | 1 - src/youtube_extension/utils/proxy.py | 14 ++- tests/unit/test_transcript_proxy_migration.py | 98 +++++++++++++++++++ 11 files changed, 202 insertions(+), 25 deletions(-) create mode 100644 tests/unit/test_transcript_proxy_migration.py diff --git a/shared/libs/youtube_proxy.py b/shared/libs/youtube_proxy.py index 1020a8434..97a256f47 100644 --- a/shared/libs/youtube_proxy.py +++ b/shared/libs/youtube_proxy.py @@ -45,8 +45,18 @@ def _get_webshare_proxy_url() -> str | None: url = os.getenv("WEBSHARE_PROXY_URL", "").strip() if not url: return None - parsed = urllib.parse.urlparse(url) - if parsed.scheme not in ("http", "https", "socks5") or not parsed.hostname: + parsed: urllib.parse.ParseResult | None = None + try: + parsed = urllib.parse.urlparse(url) + hostname = parsed.hostname + parsed.port + except ValueError: + hostname = None + if ( + parsed is None + or parsed.scheme not in ("http", "https", "socks5") + or not hostname + ): logger.warning( "WEBSHARE_PROXY_URL is set but malformed — falling back to direct connection" ) @@ -438,7 +448,9 @@ async def _transcript_operation(): logger.info(f"✅ Direct transcript extraction: {len(transcript)} segments") return transcript except Exception as e: - logger.debug(f"Direct transcript failed: {e}") + logger.debug( + f"Direct transcript failed: {_redact_proxy_credentials(e)}" + ) # Method 2: Alternative language codes # ``list_transcripts`` class method is now the instance ``list``; @@ -464,7 +476,9 @@ async def _transcript_operation(): logger.info(f"✅ Alternative language transcript: {len(transcript)} segments") return transcript except Exception as e: - logger.debug(f"Alternative transcript failed: {e}") + logger.debug( + f"Alternative transcript failed: {_redact_proxy_credentials(e)}" + ) # Method 3: yt-dlp fallback try: @@ -489,7 +503,9 @@ async def _transcript_operation(): # Convert to transcript format return [{'text': 'Transcript extracted via yt-dlp', 'start': 0, 'duration': 1}] except Exception as e: - logger.debug(f"yt-dlp extraction failed: {e}") + logger.debug( + f"yt-dlp extraction failed: {_redact_proxy_credentials(e)}" + ) # CouldNotRetrieveTranscript(>=1.0) takes a bare video_id and builds # its own message/URL; passing a sentence corrupts the generated URL. diff --git a/src/agents/interactive_metadata_extractor.py b/src/agents/interactive_metadata_extractor.py index e739738ca..08b549ce8 100644 --- a/src/agents/interactive_metadata_extractor.py +++ b/src/agents/interactive_metadata_extractor.py @@ -16,6 +16,8 @@ from dotenv import load_dotenv from youtube_transcript_api import YouTubeTranscriptApi +from youtube_extension.utils.proxy import get_transcript_proxy_config + load_dotenv() logger = logging.getLogger(__name__) @@ -84,7 +86,10 @@ async def extract_transcript(self, video_id: str) -> list[dict[str, Any]]: # event loop free. loop = asyncio.get_event_loop() transcript = await loop.run_in_executor( - None, lambda: YouTubeTranscriptApi().fetch(video_id).to_raw_data() + None, + lambda: YouTubeTranscriptApi( + proxy_config=get_transcript_proxy_config() + ).fetch(video_id).to_raw_data(), ) for i, entry in enumerate(transcript): diff --git a/src/agents/markdown_video_processor.py b/src/agents/markdown_video_processor.py index 7b507b4f9..97d2979ba 100644 --- a/src/agents/markdown_video_processor.py +++ b/src/agents/markdown_video_processor.py @@ -15,6 +15,8 @@ import aiohttp from dotenv import load_dotenv +from youtube_extension.utils.proxy import redact_proxy_credentials + # Load environment variables load_dotenv() @@ -54,10 +56,14 @@ async def get_video_metadata(self, video_id: str) -> dict[str, Any]: try: transcript_data = await proxy.get_transcript(video_id) except Exception as e: - logger.warning(f"Transcript not available: {e}") + logger.warning( + "Transcript not available: %s", redact_proxy_credentials(e) + ) except Exception as e: - logger.warning(f"MCP proxy failed, using direct API: {e}") + logger.warning( + "MCP proxy failed, using direct API: %s", redact_proxy_credentials(e) + ) # Fallback to direct API url = "https://www.googleapis.com/youtube/v3/videos" params = { diff --git a/src/agents/process_video_with_mcp.py b/src/agents/process_video_with_mcp.py index 700d212c8..d8a806344 100644 --- a/src/agents/process_video_with_mcp.py +++ b/src/agents/process_video_with_mcp.py @@ -32,6 +32,8 @@ from pathlib import Path from typing import Any +from youtube_extension.utils.proxy import get_proxy_url, get_transcript_proxy_config + # Load environment variables from project root .env if present try: from dotenv import load_dotenv # type: ignore @@ -219,7 +221,10 @@ async def _extract_transcript_with_rotation(self, video_id: str) -> list[dict[st if YouTubeTranscriptApi is not None: # youtube-transcript-api >=1.0 instance API transcript = await loop.run_in_executor( - None, lambda: YouTubeTranscriptApi().fetch(video_id).to_raw_data() + None, + lambda: YouTubeTranscriptApi( + proxy_config=get_transcript_proxy_config() + ).fetch(video_id).to_raw_data(), ) if transcript: return transcript @@ -230,7 +235,10 @@ async def _extract_transcript_with_rotation(self, video_id: str) -> list[dict[st try: if YouTubeTranscriptApi is not None: transcript_list = await loop.run_in_executor( - None, lambda: YouTubeTranscriptApi().list(video_id) # type: ignore[union-attr] + None, + lambda: YouTubeTranscriptApi( + proxy_config=get_transcript_proxy_config() + ).list(video_id), # type: ignore[union-attr] ) fetch_tasks = [ loop.run_in_executor(None, lambda t=t: t.fetch().to_raw_data()) @@ -248,7 +256,11 @@ async def _extract_transcript_with_rotation(self, video_id: str) -> list[dict[st # 3) yt-dlp fallback (mocked in tests) try: - with yt_dlp.YoutubeDL({"quiet": True}) as ydl: # type: ignore[attr-defined] + ydl_options: dict[str, Any] = {"quiet": True} + proxy_url = get_proxy_url() + if proxy_url: + ydl_options["proxy"] = proxy_url + with yt_dlp.YoutubeDL(ydl_options) as ydl: # type: ignore[attr-defined] _ = ydl.extract_info(f"https://www.youtube.com/watch?v={video_id}", download=False) return [{"text": "Transcript extracted via yt-dlp", "start": 0.0, "duration": 0.0}] except Exception: @@ -390,4 +402,3 @@ async def main() -> dict[str, Any]: if __name__ == "__main__": asyncio.run(main()) - diff --git a/src/integration/youtube_api.py b/src/integration/youtube_api.py index c5339c990..c54d93f47 100644 --- a/src/integration/youtube_api.py +++ b/src/integration/youtube_api.py @@ -12,6 +12,8 @@ import httpx from youtube_transcript_api import YouTubeTranscriptApi +from youtube_extension.utils.proxy import get_transcript_proxy_config + @dataclass class VideoMetadata: @@ -96,7 +98,9 @@ async def get_transcript( loop = asyncio.get_event_loop() transcript = await loop.run_in_executor( None, - lambda: YouTubeTranscriptApi().fetch(video_id, languages=languages).to_raw_data() + lambda: YouTubeTranscriptApi( + proxy_config=get_transcript_proxy_config() + ).fetch(video_id, languages=languages).to_raw_data() ) return [ diff --git a/src/mcp/mcp_video_processor.py b/src/mcp/mcp_video_processor.py index 7d3162011..a3f0b8dc5 100644 --- a/src/mcp/mcp_video_processor.py +++ b/src/mcp/mcp_video_processor.py @@ -20,6 +20,7 @@ from typing import Any from utils.path_utils import select_readable_file, select_writable_dir +from youtube_extension.utils.proxy import get_transcript_proxy_config # MCP integration imports try: @@ -691,7 +692,9 @@ async def _direct_extraction(): # executor — otherwise it stalls the event loop and defeats the # @timeout_protection / circuit-breaker hanging protection. loop = asyncio.get_event_loop() - yt_api = YouTubeTranscriptApi() + yt_api = YouTubeTranscriptApi( + proxy_config=get_transcript_proxy_config() + ) transcript = await loop.run_in_executor( None, lambda: yt_api.fetch( @@ -715,7 +718,9 @@ async def _routed_extraction(): # These are blocking network calls — run them in an executor to keep # the event loop free and let the timeout protection work. loop = asyncio.get_event_loop() - yt_api = YouTubeTranscriptApi() + yt_api = YouTubeTranscriptApi( + proxy_config=get_transcript_proxy_config() + ) transcript_list = await loop.run_in_executor( None, lambda: yt_api.list(video_id) ) diff --git a/src/youtube_extension/backend/services/youtube/adapters/official_api.py b/src/youtube_extension/backend/services/youtube/adapters/official_api.py index 80d2dd2f3..83a968b35 100644 --- a/src/youtube_extension/backend/services/youtube/adapters/official_api.py +++ b/src/youtube_extension/backend/services/youtube/adapters/official_api.py @@ -17,7 +17,10 @@ import httpx from youtube_extension.utils import extract_video_id -from youtube_extension.utils.proxy import get_transcript_proxy_config +from youtube_extension.utils.proxy import ( + get_transcript_proxy_config, + redact_proxy_credentials, +) # Fallback transcript retrieval try: @@ -262,7 +265,11 @@ async def get_video_transcript(self, video_id_or_url: str, language: str = 'en') except CouldNotRetrieveTranscript as e: logger.error(f"❌ Could not retrieve transcript for {video_id}: {e}") except Exception as e: - logger.warning(f"youtube-transcript-api fetch failed for {video_id}: {e}") + logger.warning( + "youtube-transcript-api fetch failed for %s: %s", + video_id, + redact_proxy_credentials(e), + ) # Fallback via robust service if needed if (not transcript_data) and HAS_ROBUST_TRANSCRIPT_FALLBACK: diff --git a/src/youtube_extension/backend/services/youtube/adapters/robust.py b/src/youtube_extension/backend/services/youtube/adapters/robust.py index 30d9e33ae..6cf908784 100644 --- a/src/youtube_extension/backend/services/youtube/adapters/robust.py +++ b/src/youtube_extension/backend/services/youtube/adapters/robust.py @@ -20,7 +20,11 @@ import httpx -from youtube_extension.utils.proxy import get_proxy_url, get_transcript_proxy_config +from youtube_extension.utils.proxy import ( + get_proxy_url, + get_transcript_proxy_config, + redact_proxy_credentials, +) # Import our cost monitor try: @@ -487,7 +491,10 @@ def _list_and_fetch() -> Any: segments = transcript_data.get("segments", []) return True, len(segments) except Exception as e: - logger.debug(f"Transcript availability check failed: {e}") + logger.debug( + "Transcript availability check failed: %s", + redact_proxy_credentials(e), + ) return False, 0 @@ -521,7 +528,11 @@ async def get_transcript( f"YouTubeTranscriptApi.fetch() returned {len(transcript) if transcript else 0} segments" ) except Exception as fetch_err: - api_error = f"YouTubeTranscriptApi.fetch failed: {type(fetch_err).__name__}: {fetch_err}" + api_error = ( + "YouTubeTranscriptApi.fetch failed: " + f"{type(fetch_err).__name__}: " + f"{redact_proxy_credentials(fetch_err)}" + ) logger.warning(api_error) transcript_errors.append(api_error) # Try instance list() as fallback — reuse the same proxy @@ -543,7 +554,11 @@ def _list_fallback() -> Any: f"YouTubeTranscriptApi.list() returned {len(transcript) if transcript else 0} segments" ) except Exception as list_err: - api_error = f"YouTubeTranscriptApi.list() fallback failed: {type(list_err).__name__}: {list_err}" + api_error = ( + "YouTubeTranscriptApi.list() fallback failed: " + f"{type(list_err).__name__}: " + f"{redact_proxy_credentials(list_err)}" + ) logger.warning(api_error) transcript_errors.append(api_error) transcript = [] @@ -592,7 +607,8 @@ def _list_fallback() -> Any: except Exception as e: error_msg = ( - f"YouTube Transcript API outer exception: {type(e).__name__}: {e}" + "YouTube Transcript API outer exception: " + f"{type(e).__name__}: {redact_proxy_credentials(e)}" ) logger.warning(error_msg) transcript_errors.append(error_msg) diff --git a/src/youtube_extension/processors/enhanced_extractor.py b/src/youtube_extension/processors/enhanced_extractor.py index 395a0c355..ae7899aa7 100644 --- a/src/youtube_extension/processors/enhanced_extractor.py +++ b/src/youtube_extension/processors/enhanced_extractor.py @@ -22,7 +22,6 @@ import yt_dlp from googleapiclient.discovery import build from googleapiclient.errors import HttpError - from youtube_transcript_api import get_transcript from youtube_transcript_api._errors import ( CouldNotRetrieveTranscript, NoTranscriptFound, diff --git a/src/youtube_extension/utils/proxy.py b/src/youtube_extension/utils/proxy.py index f82a9481f..27c437916 100644 --- a/src/youtube_extension/utils/proxy.py +++ b/src/youtube_extension/utils/proxy.py @@ -67,8 +67,18 @@ def get_proxy_url() -> str | None: url = os.getenv(_PROXY_ENV_VAR, "").strip() if not url: return None - parsed = urllib.parse.urlparse(url) - if parsed.scheme not in ("http", "https", "socks5") or not parsed.hostname: + parsed: urllib.parse.ParseResult | None = None + try: + parsed = urllib.parse.urlparse(url) + hostname = parsed.hostname + parsed.port + except ValueError: + hostname = None + if ( + parsed is None + or parsed.scheme not in ("http", "https", "socks5") + or not hostname + ): logger.warning( "%s is set but malformed — falling back to direct connection", _PROXY_ENV_VAR, diff --git a/tests/unit/test_transcript_proxy_migration.py b/tests/unit/test_transcript_proxy_migration.py new file mode 100644 index 000000000..e138ce2bd --- /dev/null +++ b/tests/unit/test_transcript_proxy_migration.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from youtube_extension.utils.proxy import get_proxy_url + + +def test_malformed_proxy_port_falls_back_to_direct_connection(monkeypatch): + monkeypatch.setenv("WEBSHARE_PROXY_URL", "http://proxy.example:invalid") + + assert get_proxy_url() is None + + +@pytest.mark.asyncio +async def test_integration_transcript_client_uses_v1_proxy_config(monkeypatch): + import integration.youtube_api as youtube_api + + captured: dict[str, object] = {} + + class FakeTranscript: + def fetch(self, video_id: str, languages: list[str]): + assert video_id == "auJzb1D-fag" + assert languages == ["en"] + return SimpleNamespace( + to_raw_data=lambda: [ + {"text": "hello", "start": 0.0, "duration": 1.0} + ] + ) + + def fake_api(**kwargs): + captured.update(kwargs) + return FakeTranscript() + + proxy_config = object() + monkeypatch.setattr(youtube_api, "YouTubeTranscriptApi", fake_api) + monkeypatch.setattr( + youtube_api, "get_transcript_proxy_config", lambda: proxy_config + ) + + service = youtube_api.YouTubeAPIService(api_key="test") + try: + result = await service.get_transcript("auJzb1D-fag") + finally: + await service.close() + + assert captured == {"proxy_config": proxy_config} + assert result[0].text == "hello" + + +def test_process_video_yt_dlp_options_include_central_proxy(monkeypatch): + import agents.process_video_with_mcp as processor + + captured: dict[str, object] = {} + + class FailingTranscriptApi: + def __init__(self, **_kwargs): + pass + + def fetch(self, *_args, **_kwargs): + raise RuntimeError("transcript unavailable") + + def list(self, *_args, **_kwargs): + raise RuntimeError("transcript unavailable") + + class FakeYoutubeDL: + def __init__(self, options): + captured.update(options) + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def extract_info(self, *_args, **_kwargs): + return {"title": "video"} + + monkeypatch.setattr( + processor, + "get_proxy_url", + lambda: "******proxy.example:8080", + ) + monkeypatch.setattr(processor, "YouTubeTranscriptApi", FailingTranscriptApi) + monkeypatch.setattr( + processor.yt_dlp, "YoutubeDL", FakeYoutubeDL, raising=False + ) + + import asyncio + + asyncio.run( + processor.RealVideoProcessor()._extract_transcript_with_rotation( + "auJzb1D-fag" + ) + ) + + assert captured["proxy"] == "******proxy.example:8080"