diff --git a/.env.example b/.env.example index c1ba20a..d86c319 100644 --- a/.env.example +++ b/.env.example @@ -38,9 +38,9 @@ GOOGLE_GCS_BUCKET=bucket-name # ============================================================ # == LLM settings == # ============================================================ -MODEL_URI=google_genai:gemini-3-flash-preview -MODEL_TEMPERATURE=0.2 -THINKING_LEVEL=low +OPENAI_API_KEY=openai-api-key +MODEL_URI=gpt-5.6-luna +REASONING_EFFORT=medium # ============================================================ # == LangSmith settings == diff --git a/.github/workflows/test-chatbot.yaml b/.github/workflows/test-chatbot.yaml index 3de158e..a2d1678 100644 --- a/.github/workflows/test-chatbot.yaml +++ b/.github/workflows/test-chatbot.yaml @@ -47,9 +47,9 @@ jobs: GOOGLE_GCS_BUCKET: mock-bucket # Mock LLM configuration + OPENAI_API_KEY: mock-openai-api-key MODEL_URI: mock-model-uri - MODEL_TEMPERATURE: 0.0 - THINKING_LEVEL: low + REASONING_EFFORT: medium # Mock LangSmith configuration LANGSMITH_TRACING: false diff --git a/alembic/versions/4b3d2fa4a75f_add_thread_language.py b/alembic/versions/4b3d2fa4a75f_add_thread_language.py new file mode 100644 index 0000000..c3190c8 --- /dev/null +++ b/alembic/versions/4b3d2fa4a75f_add_thread_language.py @@ -0,0 +1,32 @@ +"""Add language column to threads table. + +Revision ID: 4b3d2fa4a75f +Revises: f6ce7837e023 +Create Date: 2026-08-04 19:50:02.000000 +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "4b3d2fa4a75f" +down_revision: Union[str, Sequence[str], None] = "f6ce7837e023" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # server_default backfills existing threads with the Portuguese default; + # new rows get their value from the application (ThreadPayload.language). + op.add_column( + "thread", + sa.Column("language", sa.String(), nullable=False, server_default="pt"), + ) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_column("thread", "language") diff --git a/app/agent/context.py b/app/agent/context.py new file mode 100644 index 0000000..dc8eae2 --- /dev/null +++ b/app/agent/context.py @@ -0,0 +1,20 @@ +from dataclasses import dataclass + +from app.i18n import LanguageCode + + +@dataclass +class AgentContext: + """Per-run context for an agent run. + + Read by: + - The system-prompt middleware (`language`) + - The tools (`language` for localized metadata; `thread_id` and `user_id` for BigQuery job labels). + + `thread_id` is *also* kept in `config["configurable"]` because the langgraph + checkpointer keys persistence on it there; on this context it is app metadata. + """ + + thread_id: str + user_id: str + language: LanguageCode diff --git a/app/agent/middleware.py b/app/agent/middleware.py new file mode 100644 index 0000000..9f6fbe1 --- /dev/null +++ b/app/agent/middleware.py @@ -0,0 +1,9 @@ +from datetime import date + +from langchain.agents.middleware import ModelRequest, dynamic_prompt + + +@dynamic_prompt +def system_prompt_middleware(request: ModelRequest) -> str: + """Render the system prompt template, filling `{current_date}`.""" + return request.system_message.content.format(current_date=date.today().isoformat()) diff --git a/app/agent/prompts.py b/app/agent/prompts.py index ce712e6..c8e219d 100644 --- a/app/agent/prompts.py +++ b/app/agent/prompts.py @@ -1,143 +1,87 @@ SYSTEM_PROMPT = """\ -# Persona -You are a research assistant specialized in the Base dos Dados (BD) platform. Your goal is to help users analyze Brazilian public data, answering questions based on the available data and using the provided tools. +You are the research assistant for the Base dos Dados (BD) platform. You help users analyze Brazilian public data by querying it with the provided tools. -Current date: {current_date} +Current date is {current_date}. Your training cutoff predates it, so the table metadata — not your prior knowledge — is the authority on what data exists and how recent it is. ---- +# Capabilities -# Essential Brazilian Data -Main data sources available: -- **IBGE**: Census, demographics, economic surveys (`censo`, `pnad`, `pib`, `pof`). -- **INEP**: Education data (`ideb`, `censo escolar`, `enem`, `saeb`). -- **Ministério da Saúde (MS)**: Health data (`pns`, `sinasc`, `sinan`, `sim`). -- **Ministério da Economia (ME)**: Employment and economic data (`rais`, `caged`). -- **Tribunal Superior Eleitoral (TSE)**: Electoral data (`eleicoes`). -- **Banco Central do Brasil (BCB)**: Financial data (`taxa selic`, `cambio`, `ipca`). +You can search and explore Base dos Dados's datasets and tables, query and analyze the data, translate coded values, and explain the platform and how you work. You have no other abilities — in particular, you cannot generate charts, plots, maps, visualizations, or files. Never offer, promise, or imply an action beyond these, in your prose answer or in the follow-up prompts. Downloads of query results are attached automatically by the interface — you do not generate them and need not offer them. -Common patterns across data sources: -- Geographic: `sigla_uf` (state), `id_municipio` (municipality — 7-digit IBGE code). -- Temporal: `ano` (year), `period_start` / `period_end` fields from the table metadata. -- Identifiers: `id_*`, `codigo_*`, `sigla_*`. +# When to act vs. ask ---- +- **The question is specific** (a metric, a named entity, a period you can resolve from metadata): run the workflow through to an answer. Metadata calls, exploratory queries, and retries after a tool error are safe — run them without asking. +- **The question is a bare topic** ("Economia", "dados sobre educação"): explore the metadata, then describe what data exists. Do not call `execute_bigquery_sql`. +- **An entity is referenced but not named** (municipality, state, company, sector, etc.): ask which one before querying. Never substitute a likely, common, or well-known value; listing options as examples is fine. -# Available Tools -- **search_datasets**: Search datasets by keyword. -- **get_dataset_details**: Get detailed information about a dataset, with an overview of its tables. -- **get_table_details**: Get detailed information about a table, with columns, coverage period, and partitioning. -- **execute_bigquery_sql**: Execute SQL queries on BigQuery. -- **decode_table_values**: Return the key/value dictionary to decode a column. +Prefer a single round of clarification: ask for everything you need at once, then work with what the user gives you. ---- +# Workflow -# Execution Rules -**First**, apply the **Query Clarification Protocol**: if the question is broad or has unspecified entities/filters, **stop and clarify** — do not follow the flow below. Proceed only when the question is specific enough. +1. `search_datasets` — find candidate datasets by keyword. +2. `get_dataset_details` — see the dataset's tables and pick the relevant ones. +3. `get_table_details` — read the columns, `period_start`/`period_end`, `partitioned_by`, `reference_table_id`, and `needs_decoding`. +4. Query — write the query that answers the question, following the SQL rules below. Run a preliminary query only to learn something the metadata cannot give you (a column's distinct values, whether a filter matches any rows), not to preview or refine a result you could already write directly. -Follow this flow when answering data questions: -1. **Search datasets**: Use `search_datasets` to find datasets related to the question. -2. **Explore the datasets**: Use `get_dataset_details` to get an overview of the available tables and identify the most relevant ones. -3. **Examine the tables**: Use `get_table_details` to get a table's details. Pay attention to the coverage period (`period_start` and `period_end`), the partitioned columns (`partitioned_by`), and identify which columns need translation (`reference_table_id` and `needs_decoding`). -4. **Build and run the SQL query**: Run exploratory queries freely, then build the query behind your answer following the **SQL Query Protocol**, which details how to handle table coverage periods and coded columns. -5. If a tool fails, analyze the error, adjust the strategy, and try again. +When a tool fails, read the error, adjust the strategy, and try again. ---- +# Grounding rules -# Grounding Rules (CRITICAL) -**EVERY** statement about specific data (numbers, statistics, dataset/table/column names, coverage periods, coded values) **must** be grounded in tool results obtained in this conversation. **NEVER** answer by citing specific data from your prior knowledge, nor invent plausible values to fill gaps. This is **essential** for the user to trust you. +Every specific claim — numbers, statistics, dataset/table/column names, coverage periods, coded values — must come from a tool result in this conversation. Never fill a gap with prior knowledge or a plausible-looking value; say what you could not find instead. The user's trust depends on this. -Your training cutoff predates the current date. Trust the `period_start` / `period_end` fields returned by `get_table_details` for the data's coverage period — do **not** assume that dates after your training cutoff are invalid. +Answer without tools only to explain the platform or your own capabilities, to ask for clarification, or to reuse data you already retrieved earlier in this conversation. -You may answer without calling tools **only** when: -- You are explaining the Base dos Dados platform or your own capabilities. -- You are asking the user to clarify (see **Query Clarification Protocol**) — e.g. when an entity/filter is unnamed, you may ask without using any tool. -- You are referencing **data already obtained successfully via tools** in earlier turns of this same conversation. +# Brazilian data landscape ---- +Main data sources available and starting keywords for the search: -# Query Clarification Protocol -Before using any tool, assess whether the question is specific enough to start a data search (e.g. "Qual foi o IDEB médio por estado em 2021?"). If so, proceed to the search. +- **IBGE** (census, demographics, economic surveys): `censo`, `pnad`, `pib`, `pof`. +- **INEP** (education): `ideb`, `censo escolar`, `enem`, `saeb`. +- **Ministério da Saúde** (health): `pns`, `sinasc`, `sinan`, `sim`. +- **Ministério da Economia** (labor and economy): `rais`, `caged`. +- **TSE** (elections): `eleicoes`. +- **Banco Central** (financial series): `taxa selic`, `cambio`, `ipca`. -If the question is broad or exploratory (e.g. a single topic, like "Economia" or "Dados sobre educação"), **explore** with `search_datasets`, `get_dataset_details`, and `get_table_details` to discover the available data — but **stop at that step** and do **NOT** call `execute_bigquery_sql`. Based on what you found, describe to the user which data is available and guide them to refine the question (metric, period, geographic level, purpose), suggesting examples of specific questions. +Recurring column patterns: `sigla_uf` (state), `id_municipio` (7-digit IBGE code), `ano` / `mes` / `data` (time), `id_*` / `codigo_*` / `sigla_*` (identifiers). -If the question references an entity without identifying it (of any type: municipality, state, company, school, sector, etc.), **ask which one before querying**. **NEVER** assume a value the user did not provide — not even the most likely, most common, or most well-known. You may suggest options as examples, but do **not** run a query for any of them. +# SQL rules -Whenever you have **any doubt** about what to search for, ask the user for more detail. +- Reference tables by their full `gcp_id` (`project.dataset.table`); name the columns you need instead of `SELECT *`; `SELECT` statements only. +- Filter every partitioned table on one of its `partitioned_by` columns — including each partitioned table in a `JOIN`, since an unfiltered one is scanned whole and can exceed the processing limit. +- Match the query population to what the question implies. Do not add a filter the user did not ask for that drops rows along a dimension the answer is not about (e.g. `sigla_uf IS NOT NULL`, excluding a category) — it silently biases the totals and shares. If a filter is genuinely needed for correctness (e.g. dropping a pre-aggregated total row to avoid double-counting), keep it and state it in `response`. +- Filtering a plain text/categorical column by a value whose exact stored spelling you are unsure of (wording, case, accents, abbreviation): confirm the column's real values first with a quick `GROUP BY`/`DISTINCT`, since the metadata does not list them. A guessed literal that matches nothing yields silent NULLs or empty results, not an error. +- Answer in one query, and decide its output shape before you run — each `execute_bigquery_sql` call scans the table against the processing limit. Use CTEs and window functions (`SUM(...) OVER ()` for totals and shares, `ROW_NUMBER()` for ranking) to get every metric in a single scan, instead of one query per metric or a second query that only adds a column, computes a share, or trims the `LIMIT`. +- `ORDER BY` what matters for reading the result, and comment non-obvious logic with `--`. ---- +## Coverage period -# SQL Query Protocol -- **Reference full IDs:** `project.dataset.table`. -- **Select specific columns**: Do not use `SELECT *`. -- **Read-only access**: Only `SELECT` statements are allowed. -- **Partitioning**: Check the `partitioned_by` field from the `get_table_details` result. If the table is partitioned, always include a filter on at least one of the partitioned columns. This is **mandatory** to reduce processed bytes — queries without such a filter tend to scan the entire table and may exceed the processing limit. In `JOIN` queries, **each** partitioned table referenced needs its own partition filter — filtering only the main table is not enough, as the others will be scanned in full. -- **Style**: Use specific column names, `ORDER BY`, and SQL comments (`--`). -- **Consolidate**: Compute related metrics in a single query using conditional aggregation (`SUM(CASE WHEN ...)`) and CTEs (`WITH ...`) instead of one query per metric, for fewer scans and cleaner results. Explore freely; just don't fragment an answer's data across many small queries when one would do. +`period_start` and `period_end` from `get_table_details` are generated from the table itself and are authoritative. They override the dataset usage guide and your prior knowledge — including any claim that recent periods are partial, incomplete, or unstable. Never probe the period with `MIN`, `MAX`, or `DISTINCT`. -## Temporal Coverage -For any query involving a temporal dimension (columns like `ano`, `mes`, `data`, `semestre`), use the `period_start` and `period_end` fields from the `get_table_details` result as the authoritative source of the available period. +Their format varies by table (`2026`, `'2026-01-01'`, etc.). Use the value verbatim in the matching temporal column: a year against `ano`, a date against `data`. -These fields are generated automatically and reflect what **actually** exists in the table today. They take **precedence over the usage guide**, which is written manually: **ignore** statements from the guide (or from your prior knowledge) that recent periods have partial, incomplete, or unstable data when they contradict `period_end`. +- The user asked for a period outside `[period_start, period_end]`: report the available period and ask how they want to proceed. Do not silently query a different one. +- The user asked for no period: pick the window that best answers the question, anchored to the most recent data and always named in `response`. + - A **flow** (a quantity accumulated over time, summed across periods): default to the most recent complete year of coverage. A single sub-annual period is a volatile snapshot that usually misleads for these. + - A **stock** (a level that exists at a point in time, not summed across periods), a current/latest value, or a dated event: use `period_end`, the latest snapshot. + - Use a different window only when the question or the data makes it clearly more representative, and say which and why. Any narrower or older window is a choice about representativeness only — never a judgment that recent data is partial or unstable (see the precedence rule above). -The format of the values **varies by table** — it may be a year (`2024`), a date (`'2026-04-12'`), etc. Use the value **exactly** as returned, in the filter of the corresponding temporal column (year for years, date for dates, etc.). +## Coded columns -- **If the user specified a period**: validate that it is within `[period_start, period_end]`. If it is not, inform the user of the available period and ask how they would like to proceed — do not silently query a different period. -- **If the user did NOT specify a period**: **always** use `period_end` as the default filter and inform the user that you used the most recent period available. **NEVER** select a year earlier than `period_end` because you judge — based on the usage guide or prior knowledge — that the most recent data is partial or incomplete (see the precedence rule above). +A column holding opaque values (IDs, numeric codes, acronyms) must be translated before it appears in a query that filters, groups, or displays it: +- `reference_table_id` present: call `get_table_details` on that table and `JOIN` it. +- `needs_decoding: true`: call `decode_table_values` for the code-to-label dictionary. -**NEVER** run `SELECT MIN/MAX/DISTINCT` on temporal columns to discover the period — `period_start`/`period_end` already contain that information. +Filter and present the readable names (`WHERE nome_regiao = 'Nordeste'`, not `WHERE id_regiao = '2'`). Coded columns your query never touches need no translation. -## Coded Columns -Some columns store opaque values (IDs, numeric codes, acronyms, etc.) that must be translated to readable names before appearing in **any** query. The metadata defines how to translate them: +## Empty or unmatched results -- **`reference_table_id` present**: Call `get_table_details` with that ID and `JOIN` with the reference table. Filter, aggregate, and display values by their readable names (e.g. `WHERE nome_regiao = 'Nordeste'` instead of `WHERE id_regiao = '2'`). -- **`needs_decoding: true`**: Call `decode_table_values` to get the key/value dictionary and translate the values. +When a query returns 0 rows, or returns rows whose expected values come back all NULL because a join or subquery matched nothing, do not conclude the data is absent before checking the filters: coded values (join or decode to see the keys actually stored), temporal filters (against `period_start` / `period_end`), and string literals (case, accents, leading zeros like `'1'` vs `'01'`, whitespace). Rewrite the query with verified values. If the slice is genuinely empty, say so and stop trying. -Coded columns not used in the query do not need translation. +# Final Answer -**NEVER** write SQL queries that filter, aggregate, or display coded columns without translating them first. Coded values without context make the result incomprehensible and lead to incorrect filters. +`response` is prose Markdown in the user's language. Lead with the direct answer and the figures behind it, then the analysis and context that make them usable. Use tables for rankings, comparisons, and numeric series; prose for summary and interpretation. When a query returns many rows, report the shape of the result — top N, extremes, averages, trend — and a representative slice, not every row. -## Empty Result -When `execute_bigquery_sql` returns 0 rows, review the filters: -1. For filters on a categorical/coded column: - - If the column has `reference_table_id`, JOIN with the reference table. - - If the column has `needs_decoding: true`, use `decode_table_values` to check the key/value pairs. -2. For temporal filters: re-validate against `period_start` / `period_end`. -3. For string filters: consider case, accents, leading zeros (e.g. `'1'` vs `'01'`), whitespace. +State findings directly. Flag only the caveats that change how a number should be read: a narrowed period, an incomplete coverage, an excluded category. No preamble, no restating the question, no generic sign-off. Write for a reader who knows their domain but not this dataset. -Only after reviewing the filters, rewrite the query with verified values. -If after review the empty result is legitimate (the data really does not exist for the requested slice), **stop trying and inform the user**. +Keep out of `response`: the source tables and links, the follow-up prompts, and the SQL itself — each has its own field or its own place in the interface. ---- - -# Final Response -Your final response is **structured**: besides the prose text (`response` field), you return dedicated fields (data source, coverage period, and suggestions). - -## `response` Field (prose) -Write the answer as **flowing, continuous text**, without splitting it into named sections. Present the data in the most readable format possible: use Markdown tables for rankings, comparisons, numeric series; use prose for summaries, context, and analysis. The `response` field must contain: -- The direct answer to the question, with the data obtained. -- Relevant analysis and context about the data. - -If the query returns many rows, do **not** present all the data in the prose. Summarize the main findings (top N, extremes, averages, trends, etc.) and present only a representative slice of the data. - -Do **NOT** include in the prose: the list of source tables/links, the coverage period, the exploration suggestions — these elements go in the structured fields below. - -## Structured Fields -Fill them **only** based on the tool results obtained in this conversation: -- **`data_sources`**: the tables the answer draws on — those you **queried**, or specific tables you **recommend when clarifying/guiding**. Each with `dataset_id` (UUID from the `dataset_id` field of `get_table_details`, or the `id` field of `get_dataset_details`), `table_id` (UUID from the `id` field of `get_table_details`, or a table's `id` from `get_dataset_details`; **never** the dataset UUID), and a readable name. **Never** use the `gcp_id` or the BigQuery name of the dataset/table. Leave empty only when no table is relevant (e.g. explaining the platform). -- **`temporal_coverage`**: the interval your SQL query **actually filtered** — which may be narrower than the table's full coverage. E.g.: if `ano = 2010`, then `{{period_start: '2010', period_end: '2010'}}`; if `ano BETWEEN 2010 AND 2012`, then `{{period_start: '2010', period_end: '2012'}}`. Leave empty when there is no temporal dimension. -- **`follow_up_questions`**: 3 suggestions for exploring the data further. - -## Constraints -- Do **NOT** use Markdown headers (# or ##) or section titles in the response. -- Use only flowing text, bold for emphasis, lists, tables, and code blocks. -- Keep a professional yet accessible tone. -- Always respond in the user's language. - ---- - -# Compliance Checklist -Before writing the final response, perform a **strictly internal** review, checking that all the constraints mentioned in the instructions were met. Reflect: - -1. **Critical Failure — Grounding**: Is my answer grounded in results obtained through the available tools? -2. **Critical Failure — SQL Queries**: Did I run the SQL queries in compliance with the **SQL Query Protocol**, respecting the tables' coverage periods, JOINing with reference tables, and translating coded columns? -3. **Critical Failure — Final Response**: Is the `response` prose free of source/period/SQL/suggestions, and are the structured fields (`data_sources`, `temporal_coverage`, `follow_up_questions`) filled from the tool results?""" +Fill `data_sources` and `follow_up_prompts` from the tool results of this conversation, following each field's own description.""" diff --git a/app/agent/schemas.py b/app/agent/schemas.py index f0aebd6..fa680eb 100644 --- a/app/agent/schemas.py +++ b/app/agent/schemas.py @@ -1,43 +1,6 @@ -from enum import Enum - from pydantic import BaseModel, Field -class TemporalGranularity(str, Enum): - """Granularity of the data's temporal coverage.""" - - DAY = "day" - MONTH = "month" - YEAR = "year" - - -class TemporalCoverage(BaseModel): - """The interval the SQL query actually filtered on in the answer.""" - - period_start: str = Field( - description=( - "Start of the interval filtered by the SQL query (e.g. '2010' for " - "`ano = 2010` or `ano BETWEEN 2010 AND 2012`). Format it to match `granularity`: " - "YYYY (year), YYYY-MM (month) or YYYY-MM-DD (day) — e.g. '2010', '2010-01', '2010-01-01'. " - "May be narrower than the table's full coverage." - ) - ) - period_end: str = Field( - description=( - "End of the interval filtered by the SQL query (e.g. '2010' for " - "`ano = 2010`; '2012' for `ano BETWEEN 2010 AND 2012`). Format it to match `granularity`: " - "YYYY (year), YYYY-MM (month) or YYYY-MM-DD (day) — e.g. '2012', '2012-01', '2012-01-01'. " - "May be narrower than the table's full coverage." - ) - ) - granularity: TemporalGranularity = Field( - description=( - "Granularity of `period_start`/`period_end`, matching their format: " - "YYYY (year), YYYY-MM (month), YYYY-MM-DD (day)." - ) - ) - - class DataSource(BaseModel): """A Base dos Dados table the answer draws on or points the user to.""" @@ -73,16 +36,11 @@ class StructuredResponse(BaseModel): "on clarification turns. Leave empty (None) when no table is relevant." ), ) - temporal_coverage: TemporalCoverage | None = Field( - default=None, - description=( - "The interval your SQL query actually filtered. Leave empty (None) when no query " - "was run (e.g. a clarification turn) or the answer has no temporal dimension." - ), - ) - follow_up_questions: list[str] | None = Field( + follow_up_prompts: list[str] | None = Field( default=None, description=( - "3 suggested follow-up questions (in the user's language) to explore the data further." + "3 next prompts the user could send you to explore the data further, each written " + "in the user's own voice — a message the user types to you, never a question you ask " + "the user. In the user's language." ), ) diff --git a/app/agent/tools/__init__.py b/app/agent/tools/__init__.py index 656700c..1e2d5ca 100644 --- a/app/agent/tools/__init__.py +++ b/app/agent/tools/__init__.py @@ -1,10 +1,7 @@ from langchain_core.tools import BaseTool from app.agent.tools.api import get_dataset_details, get_table_details, search_datasets -from app.agent.tools.bigquery import ( - decode_table_values, - execute_bigquery_sql, -) +from app.agent.tools.bigquery import decode_table_values, execute_bigquery_sql class BDToolkit: diff --git a/app/agent/tools/api.py b/app/agent/tools/api.py index f08e22a..d25af2d 100644 --- a/app/agent/tools/api.py +++ b/app/agent/tools/api.py @@ -1,8 +1,10 @@ import json import httpx +from langchain.tools import ToolRuntime from langchain_core.tools import tool +from app.agent.context import AgentContext from app.agent.tools.exceptions import handle_tool_errors from app.agent.tools.models import ( Column, @@ -12,6 +14,7 @@ TableOverview, ) from app.agent.tools.queries import DATASET_DETAILS_QUERY, TABLE_DETAILS_QUERY +from app.i18n import DEFAULT_LANGUAGE, LanguageCode, localized_field from app.settings import settings # httpx default timeout @@ -32,38 +35,60 @@ # URL for fetching dataset details GRAPHQL_URL = f"{settings.BASEDOSDADOS_BASE_URL}/graphql" -# URL for fetching usage guides -BASE_USAGE_GUIDE_URL = "https://raw.githubusercontent.com/basedosdados/website/refs/heads/main/next/content/userGuide/pt" +# Base URL for fetching usage guides; the language subpath (pt/en/es) is appended +# per request, falling back to pt when a localized guide does not exist yet. +BASE_USAGE_GUIDE_URL = "https://raw.githubusercontent.com/basedosdados/website/refs/heads/main/next/content/userGuide" _client = httpx.AsyncClient(timeout=httpx.Timeout(TIMEOUT, read=READ_TIMEOUT)) -@tool -@handle_tool_errors -async def search_datasets(query: str) -> str: - """Search for datasets in Base dos Dados using keywords. +async def _fetch_usage_guide(gcp_dataset_id: str, language: LanguageCode) -> str | None: + """Fetch a dataset's usage-guide markdown for `language`, falling back to the default. - CRITICAL: Use individual KEYWORDS only, not full sentences. The search engine uses Elasticsearch. + Localized guides may not exist yet (only the default language is populated today), so + a missing localized file falls back to the default-language guide. Args: - query (str): 2-3 keywords maximum. Use Portuguese terms, organization names, or dataset names. - Good Examples: "censo", "rais", "ibge", "inep", "educacao", "saude". - Avoid: "Brazilian population data by municipality". + gcp_dataset_id (str): The BigQuery dataset id. + language (LanguageCode): The thread's language. Returns: - str: JSON array of datasets. If empty/irrelevant results, try different keywords. + str | None: The guide markdown, or None if no guide exists in any language. + """ + filename = gcp_dataset_id.replace("_", "-") + + # Try the requested language, then the default; `fromkeys` preserves order + # and drops the duplicate when `language` is already the default. + locales = dict.fromkeys((language, DEFAULT_LANGUAGE)) + + for locale in locales: + response = await _client.get(f"{BASE_USAGE_GUIDE_URL}/{locale}/{filename}.md") + if response.status_code == httpx.codes.OK: + return response.text.strip() - Strategy: hierarchical funnel — ALWAYS start with a SINGLE keyword and broaden a level only if it returns nothing: - 1. Dataset name ("censo", "rais", "enem") or organization ("ibge", "inep", "tse"). - 2. Core theme ("educacao", "saude", "economia", "emprego"). - 3. English term ("health", "education"). - 4. A 2-3 word combination only if the levels above fail ("saude ms", "censo municipio"). + return None - Next step: Use `get_dataset_details()` with returned dataset IDs. + +@tool +@handle_tool_errors +async def search_datasets(query: str, runtime: ToolRuntime[AgentContext]) -> str: + """Search Base dos Dados datasets (Elasticsearch). + + Args: + query (str): 1-3 keywords, never a sentence — a dataset or organization name, + else a theme. Start with one keyword; broaden only if it returns empty. + + Returns: + JSON array of datasets (id, name, description, organizations, tags, themes). """ response = await _client.get( url=SEARCH_URL, - params={"contains": "tables", "q": query, "page_size": PAGE_SIZE}, + params={ + "contains": "tables", + "q": query, + "page_size": PAGE_SIZE, + "locale": runtime.context.language, + }, ) response.raise_for_status() @@ -89,24 +114,17 @@ async def search_datasets(query: str) -> str: @tool @handle_tool_errors -async def get_dataset_details(dataset_id: str) -> str: - """Get comprehensive details about a specific dataset including all its tables. - - Use AFTER `search_datasets()` to understand data structure before writing queries. +async def get_dataset_details( + dataset_id: str, runtime: ToolRuntime[AgentContext] +) -> str: + """Get a dataset's tables and metadata by its id. Args: - dataset_id (str): Dataset ID obtained from `search_datasets()`. - This is a UUID-like string, not the human-readable name. + dataset_id (str): Dataset UUID from `search_datasets()`. Returns: - str: JSON object with complete dataset information, including: - - Basic metadata (name, description, tags, themes, organizations). - - tables: Array of all tables in the dataset with: - - gcp_id: Full BigQuery table reference (`project.dataset.table`). - - table descriptions explaining what each table contains. - - usage_guide: Provide key information and best practices for using the dataset. - - Next step: Use `get_table_details()` with returned table IDs. + JSON object — dataset metadata, a `usage_guide`, and `tables`, + each with its `gcp_id` (`project.dataset.table`), name, and description. """ response = await _client.post( url=GRAPHQL_URL, @@ -129,29 +147,31 @@ async def get_dataset_details(dataset_id: str) -> str: dataset = dataset_edges[0]["node"] + language = runtime.context.language + dataset_id = dataset["id"].split("DatasetNode:")[-1] - dataset_name = dataset["name"] - dataset_description = dataset.get("description") + dataset_name = localized_field(dataset, "name", language) + dataset_description = localized_field(dataset, "description", language) # Tags dataset_tags = [] for edge in dataset.get("tags", {}).get("edges", []): - if tag := edge.get("node", {}).get("name"): + if tag := localized_field(edge.get("node", {}), "name", language): dataset_tags.append(tag) # Themes dataset_themes = [] for edge in dataset.get("themes", {}).get("edges", []): - if theme := edge.get("node", {}).get("name"): + if theme := localized_field(edge.get("node", {}), "name", language): dataset_themes.append(theme) # Organizations dataset_organizations = [] for edge in dataset.get("organizations", {}).get("edges", []): - if org := edge.get("node", {}).get("name"): + if org := localized_field(edge.get("node", {}), "name", language): dataset_organizations.append(org) # Tables @@ -162,8 +182,8 @@ async def get_dataset_details(dataset_id: str) -> str: table = edge["node"] table_id = table["id"].split("TableNode:")[-1] - table_name = table["name"] - table_description = table.get("description") + table_name = localized_field(table, "name", language) + table_description = localized_field(table, "description", language) cloud_table_edges = table["cloudTables"]["edges"] if cloud_table_edges: @@ -185,16 +205,11 @@ async def get_dataset_details(dataset_id: str) -> str: ) ) - # Fetch usage guide + # Fetch usage guide (localized, pt fallback) usage_guide = None if gcp_dataset_id is not None: - filename = gcp_dataset_id.replace("_", "-") - - response = await _client.get(f"{BASE_USAGE_GUIDE_URL}/{filename}.md") - - if response.status_code == httpx.codes.OK: - usage_guide = response.text.strip() + usage_guide = await _fetch_usage_guide(gcp_dataset_id, language) result = Dataset( id=dataset_id, @@ -212,25 +227,15 @@ async def get_dataset_details(dataset_id: str) -> str: @tool @handle_tool_errors -async def get_table_details(table_id: str) -> str: - """Get comprehensive details about a specific table including all its columns. - - Use AFTER `get_dataset_details()` to understand table structure before writing queries. +async def get_table_details(table_id: str, runtime: ToolRuntime[AgentContext]) -> str: + """Get a table's schema and metadata by its id. Args: - table_id (str): Table ID obtained from `get_dataset_details()`. - This is typically a UUID-like string, not the human-readable name. + table_id (str): Table UUID from `get_dataset_details()`. Returns: - str: JSON object with complete table information, including: - - Basic metadata (name, description). - - gcp_id: Full BigQuery table reference (`project.dataset.table`). - - columns: All column names, types, and descriptions, including - `needs_decoding` and `reference_table_id` for coded columns. - - partitioned_by: Columns to filter on for cost control. - - period_start / period_end: First and last period covered by the table. - Format varies (`2024`, `'2026-04-12'`, etc.) — use the value verbatim, - matched to the appropriate temporal column (`ano`, `data`, etc.). + JSON object — table metadata, `gcp_id`, `period_start`/`period_end`, `partitioned_by`, and + `columns` (name, type, description, and the `needs_decoding` / `reference_table_id` flags). """ response = await _client.post( url=GRAPHQL_URL, @@ -253,9 +258,11 @@ async def get_table_details(table_id: str) -> str: table = table_edges[0]["node"] + language = runtime.context.language + table_id = table["id"].split("TableNode:")[-1] - table_name = table["name"] - table_description = table.get("description") + table_name = localized_field(table, "name", language) + table_description = localized_field(table, "description", language) table_temporal_coverage = table.get("temporalCoverage") or {} cloud_table_edges = table["cloudTables"]["edges"] @@ -293,7 +300,7 @@ async def get_table_details(table_id: str) -> str: Column( name=column["name"], type=column["bigqueryType"]["name"], - description=column.get("description"), + description=localized_field(column, "description", language), unit=column.get("measurementUnit"), reference_table_id=directory_table_id, needs_decoding=column["coveredByDictionary"], diff --git a/app/agent/tools/bigquery.py b/app/agent/tools/bigquery.py index 56b4620..236d773 100644 --- a/app/agent/tools/bigquery.py +++ b/app/agent/tools/bigquery.py @@ -1,4 +1,5 @@ import inspect +import itertools import json import uuid from functools import cache @@ -6,14 +7,20 @@ from google.api_core.exceptions import GoogleAPICallError, NotFound from google.cloud import bigquery as bq -from langchain_core.runnables import RunnableConfig +from langchain.tools import ToolRuntime from langchain_core.tools import tool +from app.agent.context import AgentContext from app.agent.tools.exceptions import handle_tool_errors from app.settings import settings MAX_BYTES_BILLED = 10 * 10**9 +# Cap on how many rows are serialized into the agent's context. The full result set +# is still materialized in BigQuery's destination table, so downloads (via query_ref) +# get every row regardless — this only keeps a large result from blowing up context. +MAX_CONTEXT_ROWS = 1000 + @cache def _bq_client() -> bq.Client: # pragma: no cover @@ -26,36 +33,18 @@ def _bq_client() -> bq.Client: # pragma: no cover @tool(response_format="content_and_artifact") @handle_tool_errors(response_format="content_and_artifact") def execute_bigquery_sql( - sql_query: str, slug: str, config: RunnableConfig + sql_query: str, slug: str, runtime: ToolRuntime[AgentContext] ) -> tuple[str, dict[str, Any] | None]: - """Execute a SQL query against BigQuery tables from the Base dos Dados database. - - PRECONDITION — only call this when the question is already specific enough to - answer with data. For a broad/exploratory question (a bare topic) or one that - references an entity the user did not name, do NOT call this tool: explore the - metadata and ask the user to refine the question first. - - Use AFTER identifying the right datasets and understanding tables structure. - It includes a 10GB processing limit for safety. + """Run one read-only GoogleSQL query against Base dos Dados (10GB scan limit). Args: - sql_query (str): Standard GoogleSQL query. Must reference tables using their full `gcp_id` from `get_dataset_details()`. - slug (str): A short filename-safe slug for this query's result, in the user's language, lowercase with underscores (e.g. "populacao_por_ano"). - - Rules: - - Use fully qualified names: `project.dataset.table`. - - Select only needed columns, don't use `SELECT *`. - - Always filter by partitioned columns when present (see `partitioned_by` in `get_table_details` results). In `JOIN` queries, each partitioned table needs its own partition filter. - - Order by relevant columns. - - Use `LIMIT` for exploration. - - Use appropriate data types in comparisons. - - Only `SELECT` statements are allowed. + sql_query (str): The query. Follow the SQL rules in the system prompt. + slug (str): Short, filename-safe, lowercase_with_underscores name for this result's + download, in the user's language. Must be distinct from the other queries in the + current request — each slug names a separate download. Returns: - str: A JSON object with: - - `row_count`: the number of rows returned. - - `rows`: the rows as a JSON array. - If the query returns no rows, a short message string is returned instead. + JSON object with `row_count` and `rows`. """ client = _bq_client() @@ -69,8 +58,8 @@ def execute_bigquery_sql( ) labels = { - "thread_id": config.get("configurable", {}).get("thread_id", "unknown"), - "user_id": config.get("configurable", {}).get("user_id", "unknown"), + "thread_id": runtime.context.thread_id, + "user_id": runtime.context.user_id, "tool_name": inspect.currentframe().f_code.co_name, } @@ -82,7 +71,9 @@ def execute_bigquery_sql( labels=labels, ), ) - rows = [dict(row) for row in job.result()] + result = job.result() + total_rows = result.total_rows + rows = [dict(row) for row in itertools.islice(result, MAX_CONTEXT_ROWS)] except GoogleAPICallError as e: reason = e.errors[0].get("reason") if getattr(e, "errors", None) else None if reason == "bytesBilledLimitExceeded": @@ -91,20 +82,27 @@ def execute_bigquery_sql( ) from e raise - if not rows: - message = ( - "Query returned 0 rows. Review the filters per the empty-result protocol." + payload = {"row_count": total_rows, "rows": rows} + + # Surface truncation only when it actually happened, so the agent knows the rows + # it sees are a subset — and that the full set is still available for download. + if total_rows > len(rows): + payload["truncated"] = True + payload["truncation_note"] = ( + f"Only the first {len(rows)} of {total_rows} rows are shown here to keep " + "the context small. The full result can still be downloaded from the interface." ) - return json.dumps(message, ensure_ascii=False), None + + content = json.dumps(payload, ensure_ascii=False, default=str) + + # No rows -> nothing to download + if not rows: + return content, None # Server-minted handle for the anonymous result table BigQuery already materialized # (~24h TTL), so a later export hands back exactly these rows without re-running. query_ref = f"qr_{uuid.uuid4().hex}" - content = json.dumps( - {"row_count": len(rows), "rows": rows}, ensure_ascii=False, default=str - ) - artifact = { "type": "query_result", "query_ref": query_ref, @@ -118,23 +116,19 @@ def execute_bigquery_sql( @tool @handle_tool_errors def decode_table_values( - table_gcp_id: str, config: RunnableConfig, column_name: str | None = None + table_gcp_id: str, + runtime: ToolRuntime[AgentContext], + column_name: str | None = None, ) -> str: - """Fetch the dictionary mapping (code -> human-readable value) for a coded column. - - REQUIRED whenever a column has `needs_decoding: true` in `get_table_details`, - BEFORE writing any SQL that filters, joins, or displays that column. - - Returns pairs of `chave` (the literal value stored in the table) and `valor` (its meaning). + """Fetch the code->label dictionary for a coded (`needs_decoding`) column. Args: - table_gcp_id (str): Full BigQuery table reference (`project.dataset.table`). - column_name (str | None, optional): The specific column to decode. Always - provide this when you know which column you need; passing None returns - the entire dictionary for the table and wastes tokens. + table_gcp_id (str): Full table reference (`project.dataset.table`). + column_name (str | None, optional): The column to decode. Omit only to fetch the + whole table's dictionary, which costs more tokens. Returns: - str: JSON array of {nome_coluna, chave, valor} entries. + JSON array of {nome_coluna, chave (stored value), valor (meaning)}. """ if "`" in table_gcp_id: table_gcp_id = table_gcp_id.replace("`", "") @@ -167,8 +161,8 @@ def decode_table_values( search_query += "ORDER BY nome_coluna, chave" labels = { - "thread_id": config.get("configurable", {}).get("thread_id", "unknown"), - "user_id": config.get("configurable", {}).get("user_id", "unknown"), + "thread_id": runtime.context.thread_id, + "user_id": runtime.context.user_id, "tool_name": inspect.currentframe().f_code.co_name, } diff --git a/app/agent/tools/queries.py b/app/agent/tools/queries.py index 9158d7e..7c1868f 100644 --- a/app/agent/tools/queries.py +++ b/app/agent/tools/queries.py @@ -1,29 +1,43 @@ +# GraphQL queries fetch the explicit model translation columns (`namePt`/..., `descriptionPt`/...) +# rather than the unqualified `name`/`description` accessors, so the caller can select the thread's +# language with a pt fallback. Column names stay as the real BigQuery identifiers and are never localized. + DATASET_DETAILS_QUERY = """ query getDatasetDetails($id: ID!) { allDataset(id: $id, first: 1) { edges { node { id - name - description + namePt + nameEn + nameEs + descriptionPt + descriptionEn + descriptionEs organizations { edges { node { - name + namePt + nameEn + nameEs } } } themes { edges { node { - name + namePt + nameEn + nameEs } } } tags { edges { node { - name + namePt + nameEn + nameEs } } } @@ -31,8 +45,12 @@ edges { node { id - name - description + namePt + nameEn + nameEs + descriptionPt + descriptionEn + descriptionEs temporalCoverage cloudTables { edges { @@ -58,8 +76,12 @@ edges { node { id - name - description + namePt + nameEn + nameEs + descriptionPt + descriptionEn + descriptionEs temporalCoverage cloudTables { edges { @@ -75,7 +97,9 @@ node { id name - description + descriptionPt + descriptionEn + descriptionEs measurementUnit coveredByDictionary isPartition diff --git a/app/api/routers/chatbot.py b/app/api/routers/chatbot.py index dc8b409..6e71a88 100644 --- a/app/api/routers/chatbot.py +++ b/app/api/routers/chatbot.py @@ -6,6 +6,7 @@ from fastapi.responses import StreamingResponse from loguru import logger +from app.agent.context import AgentContext from app.api.dependencies import Agent, AsyncDB, FeedbackSender, RunningRuns, UserID from app.api.schemas import ConfigDict, UserMessage from app.api.streaming import run_agent, stream_events @@ -30,6 +31,7 @@ ResultTooLarge, materialize_export, ) +from app.i18n import MessageKey, translate from app.settings import settings from app.storage import generate_signed_url @@ -65,8 +67,8 @@ async def _authorize_thread( async def _authorize_message( database: AsyncDatabase, message_id: str, user_id: str -) -> Message: - """Fetch a message and verify the caller owns the thread it belongs to. +) -> tuple[Message, Thread]: + """Fetch a message and its thread, verifying the caller owns the thread. Args: database (AsyncDatabase): The database repository. @@ -74,7 +76,8 @@ async def _authorize_message( user_id (str): The authenticated caller. Returns: - Message: The message, whose thread is guaranteed to belong to `user_id`. + tuple[Message, Thread]: The message and its owning thread, + guaranteed to belong to `user_id`. Raises: HTTPException: 404 whether the message is missing or the caller doesn't own its thread. @@ -90,7 +93,7 @@ async def _authorize_message( detail=f"Message {message_id} not found", ) - return message + return message, thread @router.get("/threads") @@ -118,6 +121,7 @@ async def create_thread( thread_create = ThreadCreate( title=thread_payload.title, user_id=user_id, + language=thread_payload.language, ) return await database.create_thread(thread_create) @@ -169,13 +173,27 @@ async def send_message( running_runs: RunningRuns, user_id: UserID, ) -> StreamingResponse: - await _authorize_thread(database, thread_id, user_id) + thread = await _authorize_thread(database, thread_id, user_id) run_id = str(uuid.uuid4()) + # `configurable` carries only what the langgraph checkpointer needs (thread_id). + # `metadata` is what LangSmith surfaces as trace attributes, declared explicitly. config = ConfigDict( run_id=run_id, - configurable={"thread_id": thread_id, "user_id": user_id}, + configurable={"thread_id": thread_id}, + metadata={ + "thread_id": thread_id, + "user_id": user_id, + "language": thread.language, + }, + ) + + # Application data the tools and middleware read at run time rides on the context. + context = AgentContext( + thread_id=thread_id, + user_id=user_id, + language=thread.language, ) message_create = MessageCreate( @@ -193,6 +211,7 @@ async def send_message( run_agent( agent=agent, config=config, + context=context, thread_id=thread_id, user_message=message, model_uri=settings.MODEL_URI, @@ -220,28 +239,18 @@ def _cleanup(task: asyncio.Task): # pragma: no cover ) -# User-facing details the frontend surfaces to the end user when a download fails. -RESULTS_EXPIRED_DETAIL = "Estes resultados não estão mais disponíveis para download." - -RESULTS_TOO_LARGE_DETAIL = ( - "Estes resultados são grandes demais para baixar em um único arquivo." -) - -# Fallback base name when a query's slug yields nothing filesystem-safe. -DEFAULT_EXPORT_FILENAME = "resultados" - - -def _sanitize_filename(slug: str) -> str: +def _sanitize_filename(slug: str, fallback: str) -> str: """Sanitize a query's slug into a safe base filename. Args: slug (str): The query's slug. + fallback (str): Base name to use when the slug yields nothing filesystem-safe. Returns: str: A filesystem-safe base filename, without extension. """ filename = re.sub(r"[^\w-]+", "_", slug).strip("_") - return filename or DEFAULT_EXPORT_FILENAME + return filename or fallback @router.post("/messages/{message_id}/exports") @@ -261,7 +270,7 @@ async def export_message_results( ), ) - message = await _authorize_message(database, message_id, user_id) + message, thread = await _authorize_message(database, message_id, user_id) query_handle = await database.get_query_handle(message.id, query_ref) @@ -277,18 +286,21 @@ async def export_message_results( query_ref=query_handle.query_ref, destination_table=query_handle.destination_table, file_format=file_format, - filename=_sanitize_filename(query_handle.slug), + filename=_sanitize_filename( + query_handle.slug, + translate(MessageKey.DEFAULT_EXPORT_FILENAME, thread.language), + ), message_id=str(message.id), ) except ResultTableExpired as e: raise HTTPException( status_code=status.HTTP_410_GONE, - detail=RESULTS_EXPIRED_DETAIL, + detail=translate(MessageKey.RESULTS_EXPIRED, thread.language), ) from e except ResultTooLarge as e: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail=RESULTS_TOO_LARGE_DETAIL, + detail=translate(MessageKey.RESULTS_TOO_LARGE, thread.language), ) from e signed_url = generate_signed_url( diff --git a/app/api/schemas.py b/app/api/schemas.py index 32ae8b8..069c8c3 100644 --- a/app/api/schemas.py +++ b/app/api/schemas.py @@ -1,4 +1,4 @@ -from typing import Any, TypedDict +from typing import Any, NotRequired, TypedDict from pydantic import BaseModel @@ -6,6 +6,7 @@ class ConfigDict(TypedDict): run_id: str configurable: dict[str, Any] + metadata: NotRequired[dict[str, Any]] class UserMessage(BaseModel): diff --git a/app/api/streaming/agent_runner.py b/app/api/streaming/agent_runner.py index b900e4e..f3cf440 100644 --- a/app/api/streaming/agent_runner.py +++ b/app/api/streaming/agent_runner.py @@ -6,6 +6,7 @@ from langgraph.graph.state import CompiledStateGraph from loguru import logger +from app.agent.context import AgentContext from app.agent.schemas import StructuredResponse from app.api.schemas import ConfigDict from app.api.streaming.data_sources import resolve_data_source_names @@ -20,19 +21,7 @@ QueryHandle, ) from app.exports import CollectedQueryHandle, collect_query_handles - - -class ErrorMessage: - INTERRUPTED = ( - "A conexão com o servidor foi interrompida. Por favor, tente novamente." - ) - - MODEL_CALL_LIMIT_REACHED = ( - "Essa pergunta gerou um raciocínio muito longo e não consegui chegar a uma conclusão. " - "Por favor, tente ser mais específico ou divida sua pergunta em partes menores." - ) - - UNEXPECTED = "Ocorreu um erro inesperado. Por favor, tente novamente. Se o problema persistir, avise-nos." +from app.i18n import LanguageCode, MessageKey, translate def _truncate_json( @@ -93,11 +82,12 @@ def _truncate_json( return json.dumps(data, ensure_ascii=False, indent=2) -def _process_chunk(chunk: dict[str, Any]) -> StreamEvent | None: +def _process_chunk(chunk: dict[str, Any], language: LanguageCode) -> StreamEvent | None: """Process a streaming chunk from a react agent workflow into a StreamEvent. Args: chunk (dict[str, Any]): A raw update chunk from the agent workflow. + language (LanguageCode): A supported language code, for localizing server-emitted content. Returns: StreamEvent | None: Structured event or None if the chunk is ignored: @@ -192,7 +182,7 @@ def _process_chunk(chunk: dict[str, Any]) -> StreamEvent | None: update = chunk["ModelCallLimitMiddleware.before_model"] or {} if update.get("jump_to") == "end": event_data = EventData( - content=ErrorMessage.MODEL_CALL_LIMIT_REACHED, + content=translate(MessageKey.ERROR_MODEL_CALL_LIMIT, language), tool_calls=None, ) return StreamEvent(type="model_call_limit", data=event_data) @@ -227,6 +217,7 @@ async def _persist_query_handles( async def run_agent( agent: CompiledStateGraph, config: ConfigDict, + context: AgentContext, thread_id: str, user_message: Message, model_uri: str, @@ -242,6 +233,7 @@ async def run_agent( Args: agent (CompiledStateGraph): Agent compiled state graph. config (ConfigDict): Config for agent execution. + context (AgentContext): The run context. thread_id (str): Thread unique identifier. user_message (Message): User message. model_uri (str): Model URI. @@ -257,12 +249,13 @@ async def run_agent( async for mode, chunk in agent.astream( # pragma: no cover input={"messages": [{"role": "user", "content": user_message.content}]}, config=config, + context=context, stream_mode=["updates", "values"], ): if mode == "values": continue - event = _process_chunk(chunk) + event = _process_chunk(chunk, context.language) if event is None: continue @@ -275,9 +268,11 @@ async def run_agent( collected_handles, ) elif event.type == "final_answer": - # Resolve data source names to {dataset_name}—{table_name} + # Resolve data source names to a localized "{dataset_name} — {table_name}". if event.data.structured_response is not None: - await resolve_data_source_names(event.data.structured_response) + await resolve_data_source_names( + event.data.structured_response, context.language + ) structured_response = event.data.structured_response # Set the assistant message assistant_message = event.data.content @@ -290,12 +285,14 @@ async def run_agent( await queue.put(event) except asyncio.CancelledError: if status is None: - assistant_message = ErrorMessage.INTERRUPTED + assistant_message = translate( + MessageKey.ERROR_INTERRUPTED, context.language + ) status = MessageStatus.INTERRUPTED raise except Exception: logger.exception(f"Unexpected error in run {config['run_id']}:") - assistant_message = ErrorMessage.UNEXPECTED + assistant_message = translate(MessageKey.ERROR_UNEXPECTED, context.language) status = MessageStatus.ERROR event = StreamEvent( type="error", diff --git a/app/api/streaming/data_sources.py b/app/api/streaming/data_sources.py index 5d70538..0b7b342 100644 --- a/app/api/streaming/data_sources.py +++ b/app/api/streaming/data_sources.py @@ -5,17 +5,24 @@ import httpx from loguru import logger +from app.i18n import LanguageCode, localized_field from app.settings import settings -# Minimal GraphQL query to resolve a table's name and its dataset's name from the table UUID. +# Minimal GraphQL query to resolve a table's name and its dataset's name from the +# table UUID. All three explicit model translation columns are fetched so the +# display name can match the thread's language; `namePt` is the pt fallback. TABLE_NAME_QUERY = """ query getTableName($id: ID!) { allTable(id: $id, first: 1) { edges { node { - name + namePt + nameEn + nameEs dataset { - name + namePt + nameEn + nameEs } } } @@ -27,22 +34,27 @@ _GRAPHQL_URL = f"{settings.BASEDOSDADOS_BASE_URL}/graphql" # Dedicated HTTP client + cache for resolving data source display names from table UUIDs. +# The cache is keyed by (language, table_id) because the resolved name is localized. _http_client = httpx.AsyncClient(timeout=httpx.Timeout(5.0, read=60.0)) -_TABLE_NAME_CACHE: dict[str, str] = {} +_TABLE_NAME_CACHE: dict[tuple[str, str], str] = {} -async def _resolve_table_name(table_id: str) -> str | None: - """Resolve a table UUID to a human-readable "{dataset_name} — {table_name}" and caches results. +async def _resolve_table_name(table_id: str, language: LanguageCode) -> str | None: + """Resolve a table UUID to a human-readable "{dataset_name} — {table_name}", + localized to `language`, and cache results per (language, table_id). Args: table_id (str): A Table UUID. + language (LanguageCode): A supported language code. Returns: str | None: "{dataset_name} — {table_name}", or None if the table can't be resolved (unknown UUID, missing names, network error, etc.). """ - if table_id in _TABLE_NAME_CACHE: - return _TABLE_NAME_CACHE[table_id] + cache_key = (table_id, language) + + if cache_key in _TABLE_NAME_CACHE: + return _TABLE_NAME_CACHE[cache_key] try: response = await _http_client.post( @@ -62,20 +74,22 @@ async def _resolve_table_name(table_id: str) -> str | None: return None node = edges[0]["node"] - table_name = node.get("name") - dataset_name = (node.get("dataset") or {}).get("name") + table_name = localized_field(node, "name", language) + dataset_name = localized_field(node.get("dataset") or {}, "name", language) if not table_name or not dataset_name: return None name = f"{dataset_name} — {table_name}" - _TABLE_NAME_CACHE[table_id] = name + _TABLE_NAME_CACHE[cache_key] = name return name -async def resolve_data_source_names(structured_response: dict[str, Any]) -> None: +async def resolve_data_source_names( + structured_response: dict[str, Any], language: LanguageCode +) -> None: """Overwrite each data source's display name with a deterministic - "{dataset_name} — {table_name}" resolved from its table UUID. + "{dataset_name} — {table_name}" resolved from its table UUID, localized to `language`. The resolved name is authoritative; the model-provided `name` (see `app.agent.schemas.DataSource`) is kept as a fallback for any source whose @@ -85,6 +99,7 @@ async def resolve_data_source_names(structured_response: dict[str, Any]) -> None Args: structured_response (dict[str, Any]): The dumped StructuredResponse whose `data_sources` entries are enriched in place. + language (LanguageCode): A supported language code. """ data_sources = structured_response.get("data_sources") or [] resolvable = [source for source in data_sources if source["table_id"]] @@ -93,7 +108,7 @@ async def resolve_data_source_names(structured_response: dict[str, Any]) -> None return names = await asyncio.gather( - *(_resolve_table_name(source["table_id"]) for source in resolvable), + *(_resolve_table_name(source["table_id"], language) for source in resolvable), return_exceptions=True, ) diff --git a/app/db/models.py b/app/db/models.py index f5ac0e5..234a332 100644 --- a/app/db/models.py +++ b/app/db/models.py @@ -3,9 +3,20 @@ from enum import Enum from typing import Any -from pydantic import JsonValue, computed_field +from pydantic import JsonValue, computed_field, field_validator from sqlalchemy import Enum as SAEnum -from sqlmodel import JSON, TIMESTAMP, Column, Field, Integer, Relationship, SQLModel +from sqlmodel import ( + JSON, + TIMESTAMP, + Column, + Field, + Integer, + Relationship, + SQLModel, + String, +) + +from app.i18n import DEFAULT_LANGUAGE, LanguageCode, normalize_language # ============================================================================= @@ -13,6 +24,12 @@ # ============================================================================= class ThreadPayload(SQLModel): title: str + language: LanguageCode = Field(default=DEFAULT_LANGUAGE, sa_type=String) + + @field_validator("language", mode="before") + @classmethod + def _normalize_language(cls, value: str | None) -> LanguageCode: + return normalize_language(value) class ThreadCreate(ThreadPayload): diff --git a/app/i18n.py b/app/i18n.py new file mode 100644 index 0000000..ade5da1 --- /dev/null +++ b/app/i18n.py @@ -0,0 +1,123 @@ +from enum import StrEnum +from typing import Literal, get_args + +LanguageCode = Literal["pt", "en", "es"] + +_LANGUAGES: tuple[LanguageCode, ...] = get_args(LanguageCode) + +_LANGUAGE_NAMES: dict[LanguageCode, str] = { + "pt": "Portuguese", + "en": "English", + "es": "Spanish", +} + +_LANGUAGE_FIELD_SUFFIX: dict[LanguageCode, str] = { + "pt": "Pt", + "en": "En", + "es": "Es", +} + +DEFAULT_LANGUAGE: LanguageCode = "pt" + + +def normalize_language(value: str | None) -> LanguageCode: + """Coerce an arbitrary language value to a supported code, falling back to the default. + + Args: + value (str | None): A raw language value from a request. + + Returns: + LanguageCode: One of the supported language codes. + """ + normalized = (value or DEFAULT_LANGUAGE).lower() + return normalized if normalized in _LANGUAGES else DEFAULT_LANGUAGE + + +def localized_field(node: dict, field: str, language: LanguageCode) -> str | None: + """Pick a GraphQL node's localized `field` for `language`, pt as fallback. + + Args: + node (dict): A GraphQL node exposing the fields. + field (str): The base field name, e.g. "name" or "description". + language (LanguageCode): A supported language code. + + Returns: + str | None: The requested language's value, the pt value when it is empty + (coverage is partial), or None when neither is set. + """ + suffix = _LANGUAGE_FIELD_SUFFIX[language] + return node.get(f"{field}{suffix}") or node.get(f"{field}Pt") + + +# =========================================================================== +# == Server-emitted user-facing text == +# =========================================================================== +class MessageKey(StrEnum): + """Keys for server-emitted, user-facing strings (see `translate`).""" + + ERROR_INTERRUPTED = "error_interrupted" + ERROR_MODEL_CALL_LIMIT = "error_model_call_limit" + ERROR_UNEXPECTED = "error_unexpected" + RESULTS_EXPIRED = "results_expired" + RESULTS_TOO_LARGE = "results_too_large" + DEFAULT_EXPORT_FILENAME = "default_export_filename" + + +# Keep every key populated for all of _LANGUAGES (enforced by tests). +_MESSAGES = { + MessageKey.ERROR_INTERRUPTED: { + "pt": "A conexão com o servidor foi interrompida. Por favor, tente novamente.", + "en": "The connection to the server was interrupted. Please try again.", + "es": "Se interrumpió la conexión con el servidor. Por favor, inténtalo de nuevo.", + }, + MessageKey.ERROR_MODEL_CALL_LIMIT: { + "pt": ( + "Essa pergunta gerou um raciocínio muito longo e não consegui chegar a uma " + "conclusão. Por favor, tente ser mais específico ou divida sua pergunta em " + "partes menores." + ), + "en": ( + "This question led to a very long chain of reasoning and I couldn't reach a " + "conclusion. Please try to be more specific or break your question into smaller " + "parts." + ), + "es": ( + "Esta pregunta generó un razonamiento demasiado largo y no pude llegar a una " + "conclusión. Intenta ser más específico o divide tu pregunta en partes más " + "pequeñas." + ), + }, + MessageKey.ERROR_UNEXPECTED: { + "pt": "Ocorreu um erro inesperado. Por favor, tente novamente. Se o problema persistir, avise-nos.", + "en": "An unexpected error occurred. Please try again. If the problem persists, let us know.", + "es": "Ocurrió un error inesperado. Inténtalo de nuevo. Si el problema persiste, avísanos.", + }, + MessageKey.RESULTS_EXPIRED: { + "pt": "Estes resultados não estão mais disponíveis para download.", + "en": "These results are no longer available for download.", + "es": "Estos resultados ya no están disponibles para descargar.", + }, + MessageKey.RESULTS_TOO_LARGE: { + "pt": "Estes resultados são grandes demais para baixar em um único arquivo.", + "en": "These results are too large to download in a single file.", + "es": "Estos resultados son demasiado grandes para descargar en un solo archivo.", + }, + MessageKey.DEFAULT_EXPORT_FILENAME: { + "pt": "resultados", + "en": "results", + "es": "resultados", + }, +} + + +def translate(key: MessageKey, language: LanguageCode) -> str: + """Return the localized string for `key` in `language`. + + Args: + key (MessageKey): The message to localize. + language (LanguageCode): A supported language code. + + Returns: + str: The localized string. + """ + return _MESSAGES[key][language] diff --git a/app/main.py b/app/main.py index f3b3d12..e526021 100644 --- a/app/main.py +++ b/app/main.py @@ -1,6 +1,5 @@ import asyncio from contextlib import asynccontextmanager -from datetime import date from fastapi import FastAPI from fastapi.responses import RedirectResponse @@ -9,12 +8,14 @@ ModelCallLimitMiddleware, SummarizationMiddleware, ) -from langchain.chat_models import init_chat_model +from langchain_openai import ChatOpenAI from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver from loguru import logger from psycopg.rows import dict_row from psycopg_pool import AsyncConnectionPool +from app.agent.context import AgentContext +from app.agent.middleware import system_prompt_middleware from app.agent.prompts import SYSTEM_PROMPT from app.agent.schemas import StructuredResponse from app.agent.tools import BDToolkit @@ -52,18 +53,23 @@ async def lifespan(app: FastAPI): # pragma: no cover "row_factory": dict_row, } - model = init_chat_model( + model = ChatOpenAI( + api_key=settings.OPENAI_API_KEY, model=settings.MODEL_URI, - temperature=settings.MODEL_TEMPERATURE, - credentials=settings.GOOGLE_CREDENTIALS, - thinking_level=settings.THINKING_LEVEL, - include_thoughts=True, + reasoning={ + "effort": settings.REASONING_EFFORT, + "summary": "auto", + }, ) + # Once the running context passes the trigger, summarize: older turns + # collapse into one summary while the most recent tokens are kept verbatim, + # and the summary is built from the full discarded history (no trimming). summ_middleware = SummarizationMiddleware( model=model, - trigger=("fraction", 0.5), - keep=("fraction", 0.25), + trigger=("tokens", 500_000), + keep=("tokens", 100_000), + trim_tokens_to_summarize=None, ) limit_middleware = ModelCallLimitMiddleware( @@ -84,11 +90,14 @@ async def lifespan(app: FastAPI): # pragma: no cover agent = create_agent( model=model, tools=BDToolkit.get_tools(), - system_prompt=SYSTEM_PROMPT.format( - current_date=date.today().isoformat() - ), - middleware=[summ_middleware, limit_middleware], + system_prompt=SYSTEM_PROMPT, + middleware=[ + system_prompt_middleware, + summ_middleware, + limit_middleware, + ], response_format=StructuredResponse, + context_schema=AgentContext, checkpointer=checkpointer, ) diff --git a/app/settings.py b/app/settings.py index 040e17a..430d7fe 100644 --- a/app/settings.py +++ b/app/settings.py @@ -119,21 +119,15 @@ def GOOGLE_CREDENTIALS(self) -> Credentials: # pragma: no cover # ============================================================ # == LLM settings == # ============================================================ + OPENAI_API_KEY: NonEmptyStr = Field(description="OpenAI API Key.") MODEL_URI: NonEmptyStr = Field( description=( - "Defines the LLM to be used. Refer to the LangChain docs for valid values: " - "https://reference.langchain.com/python/langchain/models/#langchain.chat_models.init_chat_model." + "Defines the OpenAI model to be used. Refer to the OpenAI api docs " + "for valid values: https://developers.openai.com/api/docs/models." ) ) - MODEL_TEMPERATURE: float = Field( - description=( - "Controls the randomness of the model’s output. " - "A higher number makes responses more creative; " - "lower ones make them more deterministic." - ) - ) - THINKING_LEVEL: Literal["minimum", "low", "medium", "high"] = Field( - description="Controls the amount of thinking Gemini models performs before returning a response." + REASONING_EFFORT: Literal["none", "low", "medium", "high", "xhigh", "max"] = Field( + description="Controls how much GPT-5.6 models think when performing a task." ) # ============================================================ diff --git a/pyproject.toml b/pyproject.toml index 3f125d0..c9c621a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "google-cloud-storage>=3.10.1", "httpx>=0.28.1", "langchain>=1.2.0", - "langchain-google-genai>=4.1.3", + "langchain-openai>=1.4.1", "langgraph>=1.0.5", "langgraph-checkpoint-postgres>=3.0.2", "langsmith>=0.6.0", diff --git a/tests/app/agent/test_middleware.py b/tests/app/agent/test_middleware.py new file mode 100644 index 0000000..9bec1f4 --- /dev/null +++ b/tests/app/agent/test_middleware.py @@ -0,0 +1,43 @@ +from datetime import date + +from langchain.agents.middleware import ModelRequest +from langchain_core.messages import AIMessage, HumanMessage, SystemMessage + +from app.agent.middleware import system_prompt_middleware +from app.agent.prompts import SYSTEM_PROMPT + + +def _run_middleware(system_prompt: str, messages: list) -> ModelRequest: + """Run system_prompt_middleware over a request and return the request the model + would have been called with (the middleware overrides its `system_message`).""" + request = ModelRequest( + model=None, + messages=messages, + system_message=SystemMessage(content=system_prompt), + tool_choice=None, + tools=[], + response_format=None, + state={"messages": messages}, + runtime=None, + model_settings={}, + ) + + captured: dict = {} + + def handler(seen: ModelRequest) -> AIMessage: + captured["request"] = seen + return AIMessage(content="ok") + + system_prompt_middleware.wrap_model_call(request, handler) + + return captured["request"] + + +class TestSystemPromptMiddleware: + def test_real_system_prompt_renders_without_stray_placeholders(self): + """Render the actual SYSTEM_PROMPT: the date must land and no brace may survive.""" + request = _run_middleware(SYSTEM_PROMPT, [HumanMessage(content="oi")]) + content = request.system_message.content + + assert date.today().isoformat() in content + assert "{" not in content and "}" not in content diff --git a/tests/app/agent/tools/test_api.py b/tests/app/agent/tools/test_api.py index 1dff85f..08be1c4 100644 --- a/tests/app/agent/tools/test_api.py +++ b/tests/app/agent/tools/test_api.py @@ -3,9 +3,13 @@ import httpx import pytest import respx +from langchain.tools import ToolRuntime +from app.agent.context import AgentContext from app.agent.tools.api import ( + BASE_USAGE_GUIDE_URL, SKIP_DIRECTORY_DATASETS, + _fetch_usage_guide, get_dataset_details, get_table_details, search_datasets, @@ -13,6 +17,33 @@ from app.settings import settings +def _build_tool_runtime(context: AgentContext) -> ToolRuntime[AgentContext]: + """Build a ToolRuntime the way the agent's ToolNode injects it into a tool. + + Only `context` varies between tests; the other slots are inert stand-ins for + graph plumbing the tools under test never read. + """ + return ToolRuntime( + state={}, + context=context, + config={}, + stream_writer=None, + tool_call_id="test-tool-call", + store=None, + ) + + +def _runtime(language: str = "pt") -> ToolRuntime[AgentContext]: + """A ToolRuntime for the given language (thread/user ids are irrelevant here).""" + return _build_tool_runtime( + AgentContext( + thread_id="test-thread", + user_id="test-user", + language=language, + ) + ) + + class TestSearchDatasets: """Tests for search_datasets tool.""" @@ -21,6 +52,9 @@ class TestSearchDatasets: @respx.mock async def test_search_datasets_returns_overviews(self): """Test successful dataset search.""" + # The /search/ endpoint is locale-aware server-side, so the response + # already carries name/description/tags/themes/organizations in the + # requested locale — the tool reads them verbatim. mock_response = { "results": [ { @@ -38,7 +72,7 @@ async def test_search_datasets_returns_overviews(self): return_value=httpx.Response(200, json=mock_response) ) - result = await search_datasets.ainvoke({"query": "test"}) + result = await search_datasets.ainvoke({"query": "test", "runtime": _runtime()}) output = json.loads(result) assert len(output) == 1 @@ -59,11 +93,24 @@ async def test_search_datasets_returns_empty_results(self): return_value=httpx.Response(200, json={"results": []}) ) - result = await search_datasets.ainvoke({"query": "nonexistent"}) + result = await search_datasets.ainvoke( + {"query": "nonexistent", "runtime": _runtime()} + ) output = json.loads(result) assert output == [] + @respx.mock + async def test_forwards_context_language_as_locale(self): + """The thread's language is sent as the `locale` so the API returns localized text.""" + route = respx.get(self.SEARCH_ENDPOINT).mock( + return_value=httpx.Response(200, json={"results": []}) + ) + + await search_datasets.ainvoke({"query": "x", "runtime": _runtime("es")}) + + assert route.calls.last.request.url.params["locale"] == "es" + class TestGetDatasetDetails: """Tests for get_dataset_details tool.""" @@ -79,13 +126,18 @@ def mock_response(self): { "node": { "id": "DatasetNode:dataset-1", - "name": "Test Dataset", - "description": "Dataset description", - "tags": {"edges": [{"node": {"name": "tag1"}}]}, - "themes": {"edges": [{"node": {"name": "theme1"}}]}, + "namePt": "Test Dataset", + "descriptionPt": "Dataset description", + "tags": {"edges": [{"node": {"namePt": "tag1"}}]}, + "themes": {"edges": [{"node": {"namePt": "theme1"}}]}, "organizations": { "edges": [ - {"node": {"name": "org1", "slug": "org1_slug"}} + { + "node": { + "namePt": "org1", + "slug": "org1_slug", + } + } ] }, "tables": { @@ -93,8 +145,8 @@ def mock_response(self): { "node": { "id": "TableNode:table-1", - "name": "Test Table", - "description": "Table description", + "namePt": "Test Table", + "descriptionPt": "Table description", "temporalCoverage": { "start": "2020", "end": "2023", @@ -134,7 +186,9 @@ async def test_get_dataset_details_success(self, mock_response): return_value=httpx.Response(404) ) - result = await get_dataset_details.ainvoke({"dataset_id": "dataset-1"}) + result = await get_dataset_details.ainvoke( + {"dataset_id": "dataset-1", "runtime": _runtime()} + ) dataset = json.loads(result) assert dataset["id"] == "dataset-1" @@ -165,7 +219,9 @@ async def test_get_dataset_details_success_with_usage_guide(self, mock_response) return_value=httpx.Response(200, text="# This is a usage guide.") ) - result = await get_dataset_details.ainvoke({"dataset_id": "dataset-1"}) + result = await get_dataset_details.ainvoke( + {"dataset_id": "dataset-1", "runtime": _runtime()} + ) dataset = json.loads(result) assert dataset["usage_guide"] == "# This is a usage guide." @@ -180,8 +236,8 @@ async def test_get_dataset_details_without_tags_themes_orgs(self): { "node": { "id": "dataset-1", - "name": "Test Dataset", - "description": "Dataset description", + "namePt": "Test Dataset", + "descriptionPt": "Dataset description", "tags": {"edges": [{"node": {}}]}, "themes": {"edges": [{"node": {}}]}, "organizations": {"edges": [{"node": {}}]}, @@ -190,8 +246,8 @@ async def test_get_dataset_details_without_tags_themes_orgs(self): { "node": { "id": "table-1", - "name": "Test Table", - "description": "Table description", + "namePt": "Test Table", + "descriptionPt": "Table description", "temporalCoverage": { "start": "2020", "end": "2023", @@ -226,7 +282,9 @@ async def test_get_dataset_details_without_tags_themes_orgs(self): return_value=httpx.Response(200) ) - result = await get_dataset_details.ainvoke({"dataset_id": "dataset-1"}) + result = await get_dataset_details.ainvoke( + {"dataset_id": "dataset-1", "runtime": _runtime()} + ) dataset = json.loads(result) assert dataset["tags"] == [] @@ -243,14 +301,19 @@ async def test_table_without_cloud_tables(self): { "node": { "id": "dataset-1", - "name": "Test Dataset", + "namePt": "Test Dataset", "slug": "test_dataset", - "description": "Dataset description", - "tags": {"edges": [{"node": {"name": "tag1"}}]}, - "themes": {"edges": [{"node": {"name": "theme1"}}]}, + "descriptionPt": "Dataset description", + "tags": {"edges": [{"node": {"namePt": "tag1"}}]}, + "themes": {"edges": [{"node": {"namePt": "theme1"}}]}, "organizations": { "edges": [ - {"node": {"name": "org1", "slug": "org1_slug"}} + { + "node": { + "namePt": "org1", + "slug": "org1_slug", + } + } ] }, "tables": { @@ -258,9 +321,9 @@ async def test_table_without_cloud_tables(self): { "node": { "id": "table-1", - "name": "Test Table", + "namePt": "Test Table", "slug": "test_table", - "description": "Table description", + "descriptionPt": "Table description", "temporalCoverage": { "start": "2020", "end": "2023", @@ -281,7 +344,9 @@ async def test_table_without_cloud_tables(self): return_value=httpx.Response(200, json=mock_response) ) - result = await get_dataset_details.ainvoke({"dataset_id": "dataset-1"}) + result = await get_dataset_details.ainvoke( + {"dataset_id": "dataset-1", "runtime": _runtime()} + ) dataset = json.loads(result) assert dataset["tables"][0]["gcp_id"] is None @@ -296,7 +361,9 @@ async def test_get_dataset_details_dataset_not_found(self): ) ) - result = await get_dataset_details.ainvoke({"dataset_id": "nonexistent"}) + result = await get_dataset_details.ainvoke( + {"dataset_id": "nonexistent", "runtime": _runtime()} + ) output = json.loads(result) assert output["status"] == "error" @@ -305,6 +372,68 @@ async def test_get_dataset_details_dataset_not_found(self): == "Dataset 'nonexistent' not found. Verify the dataset ID from search_datasets results." ) + @respx.mock + async def test_localizes_metadata_to_context_language(self): + """Each field is picked in the thread's language, falling back to pt per field.""" + mock_response = { + "data": { + "allDataset": { + "edges": [ + { + "node": { + "id": "DatasetNode:dataset-1", + "namePt": "Conjunto", + "nameEn": "Dataset", + # description has no English value -> must fall back to pt + "descriptionPt": "Descrição", + "descriptionEn": "", + "tags": { + "edges": [ + { + "node": { + "namePt": "saúde", + "nameEn": "health", + } + } + ] + }, + "themes": {"edges": []}, + "organizations": {"edges": []}, + "tables": { + "edges": [ + { + "node": { + "id": "TableNode:table-1", + "namePt": "Tabela", + "nameEn": "Table", + "descriptionPt": "Desc", + "descriptionEn": "Desc EN", + "cloudTables": {"edges": []}, + } + } + ] + }, + } + } + ] + } + } + } + + respx.post(self.GRAPHQL_URL).mock( + return_value=httpx.Response(200, json=mock_response) + ) + + result = await get_dataset_details.ainvoke( + {"dataset_id": "dataset-1", "runtime": _runtime("en")} + ) + dataset = json.loads(result) + + assert dataset["name"] == "Dataset" # English picked + assert dataset["description"] == "Descrição" # English empty -> pt fallback + assert dataset["tags"] == ["health"] # English picked + assert dataset["tables"][0]["name"] == "Table" # English picked + class TestGetTableDetails: """Tests for get_table_details tool.""" @@ -320,8 +449,8 @@ def mock_response(self): { "node": { "id": "TableNode:table-1", - "name": "Test Table", - "description": "Table description", + "namePt": "Test Table", + "descriptionPt": "Table description", "temporalCoverage": { "start": "2020", "end": "2023", @@ -343,7 +472,7 @@ def mock_response(self): "node": { "id": "col-1", "name": "peso_liquido", - "description": "Peso líquido", + "descriptionPt": "Peso líquido", "measurementUnit": "kg", "coveredByDictionary": False, "isPartition": False, @@ -355,7 +484,7 @@ def mock_response(self): "node": { "id": "col-2", "name": "status", - "description": "Status", + "descriptionPt": "Status", "measurementUnit": None, "coveredByDictionary": True, "isPartition": False, @@ -367,7 +496,7 @@ def mock_response(self): "node": { "id": "col-3", "name": "id_municipio", - "description": "ID do município", + "descriptionPt": "ID do município", "measurementUnit": None, "coveredByDictionary": False, "isPartition": False, @@ -393,7 +522,7 @@ def mock_response(self): "node": { "id": "col-4", "name": "ano", - "description": "Ano", + "descriptionPt": "Ano", "measurementUnit": None, "coveredByDictionary": False, "isPartition": True, @@ -436,7 +565,9 @@ async def test_get_table_details_success(self, mock_response): return_value=httpx.Response(200, json=mock_response) ) - result = await get_table_details.ainvoke({"table_id": "table-1"}) + result = await get_table_details.ainvoke( + {"table_id": "table-1", "runtime": _runtime()} + ) table = json.loads(result) assert table["id"] == "table-1" @@ -487,7 +618,9 @@ async def test_get_table_details_null_temporal_coverage(self, mock_response): return_value=httpx.Response(200, json=mock_response) ) - result = await get_table_details.ainvoke({"table_id": "table-1"}) + result = await get_table_details.ainvoke( + {"table_id": "table-1", "runtime": _runtime()} + ) table = json.loads(result) assert table["period_start"] is None @@ -504,7 +637,9 @@ async def test_get_table_details_without_cloud_tables(self, mock_response): return_value=httpx.Response(200, json=mock_response) ) - result = await get_table_details.ainvoke({"table_id": "table-1"}) + result = await get_table_details.ainvoke( + {"table_id": "table-1", "runtime": _runtime()} + ) table = json.loads(result) assert table["gcp_id"] is None @@ -516,7 +651,9 @@ async def test_get_table_details_not_found(self): return_value=httpx.Response(200, json={"data": {"allTable": {"edges": []}}}) ) - result = await get_table_details.ainvoke({"table_id": "nonexistent"}) + result = await get_table_details.ainvoke( + {"table_id": "nonexistent", "runtime": _runtime()} + ) output = json.loads(result) assert output["status"] == "error" @@ -524,3 +661,109 @@ async def test_get_table_details_not_found(self): output["message"] == "Table 'nonexistent' not found. Verify the table ID from get_dataset_details results." ) + + @respx.mock + async def test_localizes_metadata_to_context_language(self): + """Table/column metadata is picked in the thread's language, pt as per-field fallback.""" + mock_response = { + "data": { + "allTable": { + "edges": [ + { + "node": { + "id": "TableNode:table-1", + "namePt": "Tabela", + "nameEn": "Table", + # no English description -> must fall back to pt + "descriptionPt": "Descrição", + "descriptionEn": "", + "temporalCoverage": None, + "cloudTables": {"edges": []}, + "columns": { + "edges": [ + { + "node": { + "id": "col-1", + "name": "status", + "descriptionPt": "Situação", + "descriptionEn": "Status", + "measurementUnit": None, + "coveredByDictionary": False, + "isPartition": False, + "bigqueryType": {"name": "STRING"}, + "directoryPrimaryKey": None, + } + } + ] + }, + "dataset": {"id": "DatasetNode:dataset-1"}, + } + } + ] + } + } + } + + respx.post(self.GRAPHQL_URL).mock( + return_value=httpx.Response(200, json=mock_response) + ) + + result = await get_table_details.ainvoke( + {"table_id": "table-1", "runtime": _runtime("en")} + ) + table = json.loads(result) + + assert table["name"] == "Table" # English picked + assert table["description"] == "Descrição" # English empty -> pt fallback + assert table["columns"][0]["description"] == "Status" # English picked + + +class TestFetchUsageGuide: + """The localized usage-guide fetch: requested language, then the default, deduped.""" + + GCP_DATASET_ID = "test_dataset" + FILENAME = "test-dataset" # underscores become dashes in the guide filename + + def _url(self, locale: str) -> str: + return f"{BASE_USAGE_GUIDE_URL}/{locale}/{self.FILENAME}.md" + + @respx.mock + async def test_returns_requested_language_guide(self): + respx.get(self._url("en")).mock( + return_value=httpx.Response(200, text="# EN guide") + ) + + assert await _fetch_usage_guide(self.GCP_DATASET_ID, "en") == "# EN guide" + + @respx.mock + async def test_falls_back_to_default_when_localized_missing(self): + respx.get(self._url("en")).mock(return_value=httpx.Response(404)) + pt = respx.get(self._url("pt")).mock( + return_value=httpx.Response(200, text="# PT guide") + ) + + assert await _fetch_usage_guide(self.GCP_DATASET_ID, "en") == "# PT guide" + assert pt.called + + @respx.mock + async def test_default_language_fetches_once(self): + pt = respx.get(self._url("pt")).mock( + return_value=httpx.Response(200, text="# PT guide") + ) + en = respx.get(self._url("en")).mock( + return_value=httpx.Response(200, text="# EN guide") + ) + + assert await _fetch_usage_guide(self.GCP_DATASET_ID, "pt") == "# PT guide" + assert pt.call_count == 1 + assert ( + not en.called + ) # dedupe: no second fetch when language is already the default + + @respx.mock + async def test_returns_none_when_no_guide_exists(self): + respx.get(url__startswith=BASE_USAGE_GUIDE_URL).mock( + return_value=httpx.Response(404) + ) + + assert await _fetch_usage_guide(self.GCP_DATASET_ID, "en") is None diff --git a/tests/app/agent/tools/test_bigquery.py b/tests/app/agent/tools/test_bigquery.py index f95a875..a71a082 100644 --- a/tests/app/agent/tools/test_bigquery.py +++ b/tests/app/agent/tools/test_bigquery.py @@ -5,44 +5,83 @@ import pytest from google.api_core.exceptions import BadRequest, NotFound from google.cloud import bigquery as bq +from langchain.tools import ToolRuntime from langchain_core.messages import ToolMessage from pytest_mock import MockerFixture +from app.agent.context import AgentContext from app.agent.tools.bigquery import ( MAX_BYTES_BILLED, + MAX_CONTEXT_ROWS, decode_table_values, execute_bigquery_sql, ) +def _mock_result(rows: list[dict], total_rows: int | None = None) -> MagicMock: + """Stand in for BigQuery's RowIterator: iterable over `rows`, and carrying a + `total_rows` count (the full result size, which may exceed the fetched rows).""" + result = MagicMock() + result.__iter__.return_value = iter(rows) + result.total_rows = len(rows) if total_rows is None else total_rows + return result + + @pytest.fixture -def mock_config() -> dict: - return {"configurable": {"thread_id": "test-thread", "user_id": "test-user"}} +def mock_context() -> AgentContext: + """The run context the agent injects into a tool + (its `thread_id`/`user_id` become the BigQuery job labels).""" + return AgentContext(thread_id="test-thread", user_id="test-user", language="pt") + + +def _build_tool_runtime(context: AgentContext) -> ToolRuntime[AgentContext]: + """Build a ToolRuntime the way the agent's ToolNode injects it into a tool. + + Only `context` varies between tests; the other slots are inert stand-ins for + graph plumbing the tools under test never read. + """ + return ToolRuntime( + state={}, + context=context, + config={}, + stream_writer=None, + tool_call_id="test-tool-call", + store=None, + ) -def _invoke_tool(tool, args: dict, config: dict | None = None) -> ToolMessage: +def _invoke_tool(tool, args: dict, context: AgentContext) -> ToolMessage: """Invoke a tool the way the agent's ToolNode does, returning the ToolMessage. The tool-call form (rather than a plain args dict) exercises the real content+artifact packaging, so the ToolMessage exposes both `.content` and, - for content_and_artifact tools, `.artifact`. + for content_and_artifact tools, `.artifact`. `context` is injected as the tool + runtime — the same AgentContext the agent provides at run time. """ + runtime = _build_tool_runtime(context) + return tool.invoke( - {"type": "tool_call", "id": "1", "name": tool.name, "args": args}, - config=config, + { + "type": "tool_call", + "id": "1", + "name": tool.name, + "args": {**args, "runtime": runtime}, + } ) class TestExecuteBigQuerySQL: """Tests for execute_bigquery_sql tool.""" - def test_successful_query(self, mocker: MockerFixture, mock_config: dict): + def test_successful_query(self, mocker: MockerFixture, mock_context: AgentContext): """Test successful SELECT query returns rows plus a query_ref handle.""" mock_dry_run_query_job = MagicMock() mock_dry_run_query_job.statement_type = "SELECT" mock_query_job = MagicMock() - mock_query_job.result.return_value = [{"col1": "value1"}, {"col1": "value2"}] + mock_query_job.result.return_value = _mock_result( + [{"col1": "value1"}, {"col1": "value2"}] + ) mock_query_job.destination.to_api_repr.return_value = { "projectId": "p", "datasetId": "d", @@ -62,7 +101,7 @@ def test_successful_query(self, mocker: MockerFixture, mock_config: dict): message = _invoke_tool( execute_bigquery_sql, {"sql_query": "SELECT * FROM project.dataset.table", "slug": "resultado"}, - config=mock_config, + context=mock_context, ) output = json.loads(message.content) @@ -72,14 +111,14 @@ def test_successful_query(self, mocker: MockerFixture, mock_config: dict): assert re.fullmatch(r"qr_[0-9a-f]{32}", message.artifact["query_ref"]) def test_successful_query_exposes_destination_table_on_artifact( - self, mocker: MockerFixture, mock_config: dict + self, mocker: MockerFixture, mock_context: AgentContext ): """The result table reference is carried on the artifact, not the content.""" mock_dry_run_query_job = MagicMock() mock_dry_run_query_job.statement_type = "SELECT" mock_query_job = MagicMock() - mock_query_job.result.return_value = [{"col1": "value1"}] + mock_query_job.result.return_value = _mock_result([{"col1": "value1"}]) mock_query_job.destination.to_api_repr.return_value = { "projectId": "p", "datasetId": "d", @@ -99,7 +138,7 @@ def test_successful_query_exposes_destination_table_on_artifact( message = _invoke_tool( execute_bigquery_sql, {"sql_query": "SELECT * FROM project.dataset.table", "slug": "resultado"}, - config=mock_config, + context=mock_context, ) assert message.artifact["type"] == "query_result" @@ -112,14 +151,14 @@ def test_successful_query_exposes_destination_table_on_artifact( } def test_successful_query_empty_result( - self, mocker: MockerFixture, mock_config: dict + self, mocker: MockerFixture, mock_context: AgentContext ): - """A query with no rows returns a message and no downloadable handle.""" + """A query with no rows returns an empty result and no downloadable handle.""" mock_dry_run_query_job = MagicMock() mock_dry_run_query_job.statement_type = "SELECT" mock_query_job = MagicMock() - mock_query_job.result.return_value = [] + mock_query_job.result.return_value = _mock_result([]) mock_bigquery_client = MagicMock(spec=bq.Client) mock_bigquery_client.query.side_effect = [ @@ -134,16 +173,100 @@ def test_successful_query_empty_result( message = _invoke_tool( execute_bigquery_sql, {"sql_query": "SELECT * FROM project.dataset.table", "slug": "resultado"}, - config=mock_config, + context=mock_context, ) - assert ( - json.loads(message.content) - == "Query returned 0 rows. Review the filters per the empty-result protocol." - ) + assert json.loads(message.content) == {"row_count": 0, "rows": []} assert message.artifact is None - def test_forbidden_statement_type(self, mocker: MockerFixture, mock_config: dict): + def test_large_result_is_truncated_for_context( + self, mocker: MockerFixture, mock_context: AgentContext + ): + """A result larger than the cap only serializes a prefix, flags the truncation, + and still mints a download handle over the full (materialized) result.""" + total = MAX_CONTEXT_ROWS + 500 + all_rows = [{"n": i} for i in range(total)] + + mock_dry_run_query_job = MagicMock() + mock_dry_run_query_job.statement_type = "SELECT" + + mock_query_job = MagicMock() + mock_query_job.result.return_value = _mock_result(all_rows, total_rows=total) + mock_query_job.destination.to_api_repr.return_value = { + "projectId": "p", + "datasetId": "d", + "tableId": "t", + } + + mock_bigquery_client = MagicMock(spec=bq.Client) + mock_bigquery_client.query.side_effect = [ + mock_dry_run_query_job, + mock_query_job, + ] + + mocker.patch( + "app.agent.tools.bigquery._bq_client", return_value=mock_bigquery_client + ) + + message = _invoke_tool( + execute_bigquery_sql, + {"sql_query": "SELECT * FROM project.dataset.table", "slug": "resultado"}, + context=mock_context, + ) + + output = json.loads(message.content) + + assert output["row_count"] == total + assert len(output["rows"]) == MAX_CONTEXT_ROWS + assert output["rows"][0] == {"n": 0} + assert output["truncated"] is True + assert str(total) in output["truncation_note"] + # Full result is still downloadable. + assert re.fullmatch(r"qr_[0-9a-f]{32}", message.artifact["query_ref"]) + + def test_result_at_cap_is_not_flagged_truncated( + self, mocker: MockerFixture, mock_context: AgentContext + ): + """A result exactly at the cap returns every row and no truncation flag.""" + all_rows = [{"n": i} for i in range(MAX_CONTEXT_ROWS)] + + mock_dry_run_query_job = MagicMock() + mock_dry_run_query_job.statement_type = "SELECT" + + mock_query_job = MagicMock() + mock_query_job.result.return_value = _mock_result(all_rows) + mock_query_job.destination.to_api_repr.return_value = { + "projectId": "p", + "datasetId": "d", + "tableId": "t", + } + + mock_bigquery_client = MagicMock(spec=bq.Client) + mock_bigquery_client.query.side_effect = [ + mock_dry_run_query_job, + mock_query_job, + ] + + mocker.patch( + "app.agent.tools.bigquery._bq_client", return_value=mock_bigquery_client + ) + + message = _invoke_tool( + execute_bigquery_sql, + {"sql_query": "SELECT * FROM project.dataset.table", "slug": "resultado"}, + context=mock_context, + ) + + output = json.loads(message.content) + + assert output["row_count"] == MAX_CONTEXT_ROWS + assert len(output["rows"]) == MAX_CONTEXT_ROWS + assert "truncated" not in output + assert "truncation_note" not in output + + def test_forbidden_statement_type( + self, mocker: MockerFixture, mock_context: AgentContext + ): """Test error when statement is not SELECT.""" mock_dry_run_query_job = MagicMock() mock_dry_run_query_job.statement_type = "DELETE" @@ -158,7 +281,7 @@ def test_forbidden_statement_type(self, mocker: MockerFixture, mock_config: dict message = _invoke_tool( execute_bigquery_sql, {"sql_query": "DELETE FROM project.dataset.table", "slug": "resultado"}, - config=mock_config, + context=mock_context, ) output = json.loads(message.content) @@ -167,7 +290,7 @@ def test_forbidden_statement_type(self, mocker: MockerFixture, mock_config: dict assert output["message"] == "Only SELECT statements are allowed, got DELETE." def test_bytes_billed_limit_exceeded( - self, mocker: MockerFixture, mock_config: dict + self, mocker: MockerFixture, mock_context: AgentContext ): """Test error when query exceeds bytes billed limit.""" mock_dry_run_query_job = MagicMock() @@ -190,7 +313,7 @@ def test_bytes_billed_limit_exceeded( message = _invoke_tool( execute_bigquery_sql, {"sql_query": "SELECT * FROM project.dataset.table", "slug": "resultado"}, - config=mock_config, + context=mock_context, ) output = json.loads(message.content) @@ -201,7 +324,9 @@ def test_bytes_billed_limit_exceeded( "Filter by partitioned columns." ) - def test_google_api_error_reraise(self, mocker: MockerFixture, mock_config: dict): + def test_google_api_error_reraise( + self, mocker: MockerFixture, mock_context: AgentContext + ): """Test that non-bytesBilledLimitExceeded GoogleAPICallError is re-raised.""" mock_dry_run_query_job = MagicMock() mock_dry_run_query_job.statement_type = "SELECT" @@ -221,7 +346,7 @@ def test_google_api_error_reraise(self, mocker: MockerFixture, mock_config: dict message = _invoke_tool( execute_bigquery_sql, {"sql_query": "SELECT * FROM project.dataset.table", "slug": "resultado"}, - config=mock_config, + context=mock_context, ) output = json.loads(message.content) @@ -233,7 +358,9 @@ def test_google_api_error_reraise(self, mocker: MockerFixture, mock_config: dict class TestDecodeTableValues: """Tests for decode_table_values tool.""" - def test_decode_all_columns(self, mocker: MockerFixture, mock_config: dict): + def test_decode_all_columns( + self, mocker: MockerFixture, mock_context: AgentContext + ): """Test decoding all columns from a table.""" mock_query_job = MagicMock() mock_query_job.result.return_value = [ @@ -251,7 +378,7 @@ def test_decode_all_columns(self, mocker: MockerFixture, mock_config: dict): message = _invoke_tool( decode_table_values, {"table_gcp_id": "project.dataset.table"}, - config=mock_config, + context=mock_context, ) output = json.loads(message.content) @@ -268,7 +395,7 @@ def test_decode_all_columns(self, mocker: MockerFixture, mock_config: dict): assert "nome_coluna = " not in param_names def test_decode_all_columns_with_backticks( - self, mocker: MockerFixture, mock_config: dict + self, mocker: MockerFixture, mock_context: AgentContext ): """Test decoding all columns from a table with backticks in its name.""" mock_query_job = MagicMock() @@ -287,7 +414,7 @@ def test_decode_all_columns_with_backticks( message = _invoke_tool( decode_table_values, {"table_gcp_id": "`project.dataset.table`"}, - config=mock_config, + context=mock_context, ) output = json.loads(message.content) @@ -303,7 +430,9 @@ def test_decode_all_columns_with_backticks( assert "table_name" in param_names assert "nome_coluna = " not in param_names - def test_decode_specific_column(self, mocker: MockerFixture, mock_config: dict): + def test_decode_specific_column( + self, mocker: MockerFixture, mock_context: AgentContext + ): """Test decoding a specific column.""" mock_query_job = MagicMock() mock_query_job.result.return_value = [ @@ -321,7 +450,7 @@ def test_decode_specific_column(self, mocker: MockerFixture, mock_config: dict): message = _invoke_tool( decode_table_values, {"table_gcp_id": "project.dataset.table", "column_name": "col1"}, - config=mock_config, + context=mock_context, ) output = json.loads(message.content) @@ -337,7 +466,9 @@ def test_decode_specific_column(self, mocker: MockerFixture, mock_config: dict): assert "table_name" in param_names assert "column_name" in param_names - def test_dictionary_not_found(self, mocker: MockerFixture, mock_config: dict): + def test_dictionary_not_found( + self, mocker: MockerFixture, mock_context: AgentContext + ): """Test error when dictionary table doesn't exist.""" error = NotFound( message="Table not found", @@ -354,7 +485,7 @@ def test_dictionary_not_found(self, mocker: MockerFixture, mock_config: dict): message = _invoke_tool( decode_table_values, {"table_gcp_id": "project.dataset.table"}, - config=mock_config, + context=mock_context, ) output = json.loads(message.content) @@ -362,10 +493,10 @@ def test_dictionary_not_found(self, mocker: MockerFixture, mock_config: dict): assert output["status"] == "error" assert output["message"] == "Dictionary table not found for this dataset." - def test_invalid_table_reference(self, mock_config: dict): + def test_invalid_table_reference(self, mock_context: AgentContext): """Test error when table reference format is invalid.""" message = _invoke_tool( - decode_table_values, {"table_gcp_id": "table"}, config=mock_config + decode_table_values, {"table_gcp_id": "table"}, context=mock_context ) output = json.loads(message.content) @@ -376,7 +507,9 @@ def test_invalid_table_reference(self, mock_config: dict): == "Invalid table reference: 'table'. Expected format: project.dataset.table" ) - def test_google_api_error_reraise(self, mocker: MockerFixture, mock_config: dict): + def test_google_api_error_reraise( + self, mocker: MockerFixture, mock_context: AgentContext + ): """Test that non-notFound GoogleAPICallError is re-raised.""" error = BadRequest( message="Syntax error", @@ -393,7 +526,7 @@ def test_google_api_error_reraise(self, mocker: MockerFixture, mock_config: dict message = _invoke_tool( decode_table_values, {"table_gcp_id": "project.dataset.table"}, - config=mock_config, + context=mock_context, ) output = json.loads(message.content) diff --git a/tests/app/api/routers/test_chatbot.py b/tests/app/api/routers/test_chatbot.py index ba3fb90..3cee458 100644 --- a/tests/app/api/routers/test_chatbot.py +++ b/tests/app/api/routers/test_chatbot.py @@ -12,7 +12,7 @@ from pytest_mock import MockerFixture from app.api.dependencies import get_database, get_feedback_sender -from app.api.routers.chatbot import DEFAULT_EXPORT_FILENAME, _sanitize_filename +from app.api.routers.chatbot import _sanitize_filename from app.api.streaming.schemas import StreamEvent from app.db.database import AsyncDatabase from app.db.models import ( @@ -26,8 +26,10 @@ MessageStatus, QueryHandle, Thread, + ThreadCreate, ) from app.exports import ExportedFile, ResultTableExpired, ResultTooLarge +from app.i18n import MessageKey, translate from app.main import app from app.settings import settings from tests.conftest import MessagesFactory, ThreadFactory @@ -41,6 +43,7 @@ def send_feedback(self, feedback: Feedback, created: bool): class MockAgent: def __init__(self, checkpointer=None): self.checkpointer = checkpointer + self.captured_config = None def invoke(self, input, config): return {"messages": [AIMessage("Mock response")]} @@ -48,12 +51,13 @@ def invoke(self, input, config): async def ainvoke(self, input, config): return {"messages": [AIMessage("Mock response")]} - def stream(self, input, config, stream_mode): + def stream(self, input, config, stream_mode, context=None): chunk = {"model": {"messages": [AIMessage("Mock response")]}} yield "updates", chunk yield "values", chunk - async def astream(self, input, config, stream_mode): + async def astream(self, input, config, stream_mode, context=None): + self.captured_config = config chunk = {"model": {"messages": [AIMessage("Mock response")]}} yield "updates", chunk yield "values", chunk @@ -240,6 +244,20 @@ def test_create_thread_success(self, client: TestClient, access_token: str): thread = Thread.model_validate(response.json()) assert thread.title == "Mock Title" + assert thread.language == "pt" # default when the client omits it + + def test_create_thread_normalizes_and_persists_language( + self, client: TestClient, access_token: str + ): + """A locale from the request body is normalized and stored on the thread.""" + response = client.post( + url="/api/v1/chatbot/threads", + json={"title": "Mock Title", "language": "EN"}, + headers={"Authorization": f"Bearer {access_token}"}, + ) + + assert response.status_code == status.HTTP_201_CREATED + assert Thread.model_validate(response.json()).language == "en" def test_create_thread_missing_title(self, client: TestClient, access_token: str): """Test thread creation without title field.""" @@ -505,6 +523,37 @@ def test_send_message_success( assert events[-1].type == "complete" assert events[-1].data.run_id is not None + async def test_send_message_sets_trace_metadata( + self, + client: TestClient, + access_token: str, + database: AsyncDatabase, + user_id: str, + ): + """The run config carries thread_id/user_id/language as `metadata` — what + LangSmith surfaces as trace attributes (see app.api.schemas.ConfigDict).""" + thread = await database.create_thread( + ThreadCreate(title="ES thread", user_id=user_id, language="es") + ) + + response = client.post( + url=f"/api/v1/chatbot/threads/{thread.id}/messages", + json={"content": "hola"}, + headers={"Authorization": f"Bearer {access_token}"}, + ) + + assert response.status_code == status.HTTP_201_CREATED + + # Drain the stream so the background run_agent task dispatches to the agent. + for _ in response.iter_lines(): + pass + + assert app.state.agent.captured_config["metadata"] == { + "thread_id": str(thread.id), + "user_id": user_id, + "language": "es", + } + def test_send_message_missing_content( self, client: TestClient, access_token: str, thread: Thread ): @@ -908,6 +957,52 @@ def test_result_too_large_is_bad_request( assert response.status_code == status.HTTP_400_BAD_REQUEST + async def test_failure_detail_uses_thread_language( + self, + client: TestClient, + access_token: str, + database: AsyncDatabase, + user_id: str, + mocker: MockerFixture, + ): + """Download-failure details are localized to the owning thread's language.""" + thread = await database.create_thread( + ThreadCreate(title="ES thread", user_id=user_id, language="es") + ) + message = await database.create_message( + MessageCreate( + thread_id=thread.id, + model_uri="mock-model", + role=MessageRole.ASSISTANT, + content="Respuesta", + status=MessageStatus.SUCCESS, + ) + ) + await database.create_query_handles( + [ + QueryHandle( + query_ref="qr_test", + message_id=message.id, + slug="resultado", + destination_table=self.DESTINATION, + ) + ] + ) + mocker.patch( + "app.api.routers.chatbot.materialize_export", + side_effect=ResultTooLarge("too large"), + ) + + response = client.post( + url=f"/api/v1/chatbot/messages/{message.id}/exports?query_ref=qr_test", + headers={"Authorization": f"Bearer {access_token}"}, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json()["detail"] == translate( + MessageKey.RESULTS_TOO_LARGE, "es" + ) + def test_missing_query_ref_param_is_unprocessable( self, client: TestClient, access_token: str, downloadable_message: Message ): @@ -957,9 +1052,9 @@ class TestSanitizeFilename: ], ) def test_sanitizes_slug(self, slug: str, expected: str): - assert _sanitize_filename(slug) == expected + assert _sanitize_filename(slug, "resultados") == expected @pytest.mark.parametrize("slug", ["", " ", "!!!", "/", "..."]) def test_falls_back_when_nothing_usable_remains(self, slug: str): - """A slug that sanitizes to empty falls back to the default, never ''.""" - assert _sanitize_filename(slug) == DEFAULT_EXPORT_FILENAME + """A slug that sanitizes to empty falls back to the provided fallback, never ''.""" + assert _sanitize_filename(slug, "resultados") == "resultados" diff --git a/tests/app/api/streaming/test_agent_runner.py b/tests/app/api/streaming/test_agent_runner.py index 6d0d376..dcf1e84 100644 --- a/tests/app/api/streaming/test_agent_runner.py +++ b/tests/app/api/streaming/test_agent_runner.py @@ -8,15 +8,13 @@ import pytest from langchain_core.messages import AIMessage, ToolMessage +from app.agent.context import AgentContext from app.agent.schemas import ( DataSource, StructuredResponse, - TemporalCoverage, - TemporalGranularity, ) from app.api.schemas import ConfigDict from app.api.streaming.agent_runner import ( - ErrorMessage, _process_chunk, _truncate_json, run_agent, @@ -24,6 +22,7 @@ from app.api.streaming.schemas import StreamEvent from app.db.models import Message, MessageCreate, MessageRole, MessageStatus from app.exports import OFFERED_EXPORT_FORMATS +from app.i18n import MessageKey, translate MODEL_URI = "mock-model" @@ -212,7 +211,7 @@ def test_agent_chunk_with_tool_calls(self): } } - event = _process_chunk(chunk) + event = _process_chunk(chunk, "pt") assert event is not None assert event.type == "tool_call" @@ -248,7 +247,7 @@ def test_agent_chunk_with_multiple_tool_calls(self): } } - event = _process_chunk(chunk) + event = _process_chunk(chunk, "pt") assert event is not None assert event.type == "tool_call" @@ -260,7 +259,7 @@ def test_agent_chunk_final_answer(self): """Test agent chunk without tool calls returns final_answer event.""" chunk = {"model": {"messages": [AIMessage(content="Here is your answer.")]}} - event = _process_chunk(chunk) + event = _process_chunk(chunk, "pt") assert event is not None assert event.type == "final_answer" @@ -274,7 +273,7 @@ def test_agent_chunk_empty_messages(self): """Test agent chunk with empty messages list returns empty final_answer.""" chunk = {"model": {"messages": []}} - event = _process_chunk(chunk) + event = _process_chunk(chunk, "pt") assert event is not None assert event.type == "final_answer" @@ -288,12 +287,7 @@ def test_agent_chunk_structured_response(self): data_sources=[ DataSource(dataset_id="ds1", table_id="tb1", name="Tabela 1") ], - temporal_coverage=TemporalCoverage( - period_start="2020", - period_end="2025", - granularity=TemporalGranularity.YEAR, - ), - follow_up_questions=["E em 2026?", "Por estado?", "Por região?"], + follow_up_prompts=["E em 2026?", "Por estado?", "Por região?"], ) # The model node sets `structured_response` alongside the internal @@ -316,7 +310,7 @@ def test_agent_chunk_structured_response(self): } } - event = _process_chunk(chunk) + event = _process_chunk(chunk, "pt") assert event is not None assert event.type == "final_answer" @@ -329,12 +323,7 @@ def test_agent_chunk_structured_response(self): assert event.data.structured_response["data_sources"] == [ {"dataset_id": "ds1", "table_id": "tb1", "name": "Tabela 1"} ] - assert event.data.structured_response["temporal_coverage"] == { - "period_start": "2020", - "period_end": "2025", - "granularity": "year", - } - assert event.data.structured_response["follow_up_questions"] == [ + assert event.data.structured_response["follow_up_prompts"] == [ "E em 2026?", "Por estado?", "Por região?", @@ -351,7 +340,7 @@ def test_agent_chunk_structured_response_sanitizes_links(self): } } - event = _process_chunk(chunk) + event = _process_chunk(chunk, "pt") assert event is not None assert event.type == "final_answer" @@ -373,7 +362,7 @@ def test_tools_chunk_single_tool(self): } } - event = _process_chunk(chunk) + event = _process_chunk(chunk, "pt") assert event is not None assert event.type == "tool_output" @@ -412,7 +401,7 @@ def test_tools_chunk_projects_query_result_artifact_on_serialization(self): } } - event = _process_chunk(chunk) + event = _process_chunk(chunk, "pt") output = event.data.tool_outputs[0] # In memory the handle is present (run_agent reads destination_table off it) ... @@ -443,7 +432,7 @@ def test_tools_chunk_surfaces_non_query_result_artifact_on_serialization(self): } } - event = _process_chunk(chunk) + event = _process_chunk(chunk, "pt") output = event.data.tool_outputs[0] # A client-facing artifact passes through redaction untouched, in memory ... @@ -479,7 +468,7 @@ def test_tools_chunk_multiple_parallel_tools(self): ] } - event = _process_chunk(chunk) + event = _process_chunk(chunk, "pt") assert event is not None assert event.type == "tool_output" @@ -502,7 +491,7 @@ def test_tools_chunk_with_error_status(self): } } - event = _process_chunk(chunk) + event = _process_chunk(chunk, "pt") assert event is not None assert event.type == "tool_output" @@ -512,7 +501,7 @@ def test_tools_chunk_unexpected_format(self): """Test tools chunk with unexpected format returns empty tool_outputs.""" chunk = {"tools": "unexpected string"} - event = _process_chunk(chunk) + event = _process_chunk(chunk, "pt") assert event is not None assert event.type == "tool_output" @@ -527,32 +516,32 @@ def test_model_call_limit_triggered_chunk(self): } } - event = _process_chunk(chunk) + event = _process_chunk(chunk, "pt") assert event is not None assert event.type == "model_call_limit" - assert event.data.content == ErrorMessage.MODEL_CALL_LIMIT_REACHED + assert event.data.content == translate(MessageKey.ERROR_MODEL_CALL_LIMIT, "pt") def test_model_call_limit_passthrough_chunk_returns_none(self): """Test before_model passthrough chunk (None payload) returns None.""" chunk = {"ModelCallLimitMiddleware.before_model": None} - assert _process_chunk(chunk) is None + assert _process_chunk(chunk, "pt") is None def test_model_call_limit_passthrough_chunk_no_jump_returns_none(self): """Test before_model chunk without jump_to returns None.""" chunk = {"ModelCallLimitMiddleware.before_model": {"messages": []}} - assert _process_chunk(chunk) is None + assert _process_chunk(chunk, "pt") is None def test_unrecognized_chunk_returns_none(self): """Test unrecognized chunk returns None.""" chunk = {"unknown_node": {"data": "something"}} - event = _process_chunk(chunk) + event = _process_chunk(chunk, "pt") assert event is None def test_empty_chunk_returns_none(self): """Test empty chunk returns None.""" chunk = {} - event = _process_chunk(chunk) + event = _process_chunk(chunk, "pt") assert event is None @@ -567,6 +556,69 @@ async def _drain(self, queue: asyncio.Queue[StreamEvent]) -> list[StreamEvent]: if event.type == "complete": return events + async def test_forwards_context_to_agent( + self, + mock_database: MagicMock, + mock_user_message: Message, + config: ConfigDict, + thread_id: str, + ): + """The run context must reach `agent.astream` — that's how tools and the + system-prompt middleware receive the thread's language/user/thread ids.""" + agent = MagicMock() + seen = {} + + async def astream(*args, **kwargs): + seen["context"] = kwargs.get("context") + yield ("updates", {"model": {"messages": [AIMessage(content="ok")]}}) + + agent.astream = astream + context = AgentContext(thread_id=thread_id, user_id="u", language="es") + queue: asyncio.Queue[StreamEvent] = asyncio.Queue() + + await run_agent( + agent=agent, + config=config, + thread_id=thread_id, + user_message=mock_user_message, + model_uri=MODEL_URI, + context=context, + queue=queue, + ) + + assert seen["context"] is context + + async def test_localizes_error_message_to_context_language( + self, + mock_database: MagicMock, + mock_user_message: Message, + config: ConfigDict, + thread_id: str, + ): + """A crash mid-run persists the error message in the thread's language.""" + agent = MagicMock() + + async def astream(*args, **kwargs): + raise RuntimeError("boom") + yield # pragma: no cover — make this an async generator + + agent.astream = astream + queue: asyncio.Queue[StreamEvent] = asyncio.Queue() + + await run_agent( + agent=agent, + config=config, + thread_id=thread_id, + user_message=mock_user_message, + model_uri=MODEL_URI, + context=AgentContext(thread_id=thread_id, user_id="u", language="es"), + queue=queue, + ) + + message = mock_database.create_message.call_args[0][0] + assert message.status == MessageStatus.ERROR + assert message.content == translate(MessageKey.ERROR_UNEXPECTED, "es") + async def test_emits_events_and_persists_success( self, mock_database: MagicMock, @@ -592,6 +644,9 @@ async def astream(*args, **kwargs): thread_id=thread_id, user_message=mock_user_message, model_uri=MODEL_URI, + context=AgentContext( + thread_id="test-thread", user_id="test-user", language="pt" + ), queue=queue, ) @@ -656,6 +711,9 @@ async def astream(*args, **kwargs): thread_id=thread_id, user_message=mock_user_message, model_uri=MODEL_URI, + context=AgentContext( + thread_id="test-thread", user_id="test-user", language="pt" + ), queue=queue, ) @@ -684,15 +742,10 @@ async def test_structured_response_is_emitted_and_persisted( data_sources=[ DataSource(dataset_id="ds1", table_id="tb1", name="model fallback") ], - temporal_coverage=TemporalCoverage( - period_start="2020", - period_end="2025", - granularity=TemporalGranularity.YEAR, - ), - follow_up_questions=["E em 2026?"], + follow_up_prompts=["E em 2026?"], ) - async def fake_resolve(structured_response: dict[str, Any]): + async def fake_resolve(structured_response: dict[str, Any], language: str): for source in structured_response.get("data_sources") or []: source["name"] = "Conjunto DS1 - Tabela TB1" @@ -724,6 +777,9 @@ async def astream(*args, **kwargs): thread_id=thread_id, user_message=mock_user_message, model_uri=MODEL_URI, + context=AgentContext( + thread_id="test-thread", user_id="test-user", language="pt" + ), queue=queue, ) @@ -742,14 +798,7 @@ async def astream(*args, **kwargs): "name": "Conjunto DS1 - Tabela TB1", } ] - assert events[0].data.structured_response["temporal_coverage"] == { - "period_start": "2020", - "period_end": "2025", - "granularity": "year", - } - assert events[0].data.structured_response["follow_up_questions"] == [ - "E em 2026?" - ] + assert events[0].data.structured_response["follow_up_prompts"] == ["E em 2026?"] mock_database.create_message.assert_called_once() message = mock_database.create_message.call_args[0][0] @@ -809,6 +858,9 @@ async def astream(*args, **kwargs): thread_id=thread_id, user_message=mock_user_message, model_uri=MODEL_URI, + context=AgentContext( + thread_id="test-thread", user_id="test-user", language="pt" + ), queue=queue, ) events = await self._drain(queue) @@ -879,6 +931,9 @@ async def astream(*args, **kwargs): thread_id=thread_id, user_message=mock_user_message, model_uri=MODEL_URI, + context=AgentContext( + thread_id="test-thread", user_id="test-user", language="pt" + ), queue=queue, ) @@ -914,19 +969,22 @@ async def astream(*args, **kwargs): thread_id=thread_id, user_message=mock_user_message, model_uri=MODEL_URI, + context=AgentContext( + thread_id="test-thread", user_id="test-user", language="pt" + ), queue=queue, ) events = await self._drain(queue) assert [e.type for e in events] == ["error", "complete"] - assert events[0].data.content == ErrorMessage.UNEXPECTED + assert events[0].data.content == translate(MessageKey.ERROR_UNEXPECTED, "pt") assert events[0].data.error_details == {"reason": "agent_failed"} mock_database.create_message.assert_called_once() message = mock_database.create_message.call_args[0][0] assert isinstance(message, MessageCreate) assert message.status == MessageStatus.ERROR - assert message.content == ErrorMessage.UNEXPECTED + assert message.content == translate(MessageKey.ERROR_UNEXPECTED, "pt") async def test_model_call_limit_persists_with_dedicated_status( self, @@ -953,19 +1011,24 @@ async def astream(*args, **kwargs): thread_id=thread_id, user_message=mock_user_message, model_uri=MODEL_URI, + context=AgentContext( + thread_id="test-thread", user_id="test-user", language="pt" + ), queue=queue, ) events = await self._drain(queue) assert [e.type for e in events] == ["model_call_limit", "complete"] - assert events[0].data.content == ErrorMessage.MODEL_CALL_LIMIT_REACHED + assert events[0].data.content == translate( + MessageKey.ERROR_MODEL_CALL_LIMIT, "pt" + ) assert events[-1].data.run_id == config["run_id"] mock_database.create_message.assert_called_once() message = mock_database.create_message.call_args[0][0] assert isinstance(message, MessageCreate) assert message.status == MessageStatus.MODEL_CALL_LIMIT - assert message.content == ErrorMessage.MODEL_CALL_LIMIT_REACHED + assert message.content == translate(MessageKey.ERROR_MODEL_CALL_LIMIT, "pt") async def test_complete_still_emitted_when_db_write_fails( self, @@ -1009,6 +1072,9 @@ async def astream(*args, **kwargs): thread_id=thread_id, user_message=mock_user_message, model_uri=MODEL_URI, + context=AgentContext( + thread_id="test-thread", user_id="test-user", language="pt" + ), queue=queue, ) @@ -1048,6 +1114,9 @@ async def astream(*args, **kwargs): thread_id=thread_id, user_message=mock_user_message, model_uri=MODEL_URI, + context=AgentContext( + thread_id="test-thread", user_id="test-user", language="pt" + ), queue=queue, ) ) @@ -1094,6 +1163,9 @@ async def astream(*args, **kwargs): thread_id=thread_id, user_message=mock_user_message, model_uri=MODEL_URI, + context=AgentContext( + thread_id="test-thread", user_id="test-user", language="pt" + ), queue=queue, ) ) @@ -1112,7 +1184,7 @@ async def astream(*args, **kwargs): message = mock_database.create_message.call_args[0][0] assert isinstance(message, MessageCreate) assert message.status == MessageStatus.INTERRUPTED - assert message.content == ErrorMessage.INTERRUPTED + assert message.content == translate(MessageKey.ERROR_INTERRUPTED, "pt") async def test_cancellation_after_final_answer_preserves_success( self, @@ -1147,6 +1219,9 @@ async def astream(*args, **kwargs): thread_id=thread_id, user_message=mock_user_message, model_uri=MODEL_URI, + context=AgentContext( + thread_id="test-thread", user_id="test-user", language="pt" + ), queue=queue, ) ) diff --git a/tests/app/api/streaming/test_data_sources.py b/tests/app/api/streaming/test_data_sources.py index a18b46f..32f4e0c 100644 --- a/tests/app/api/streaming/test_data_sources.py +++ b/tests/app/api/streaming/test_data_sources.py @@ -24,9 +24,52 @@ def _clear_cache(self): @staticmethod def _node_response(dataset_name: str | None, table_name: str | None): - node = {"name": table_name, "dataset": {"name": dataset_name}} + node = {"namePt": table_name, "dataset": {"namePt": dataset_name}} return {"data": {"allTable": {"edges": [{"node": node}]}}} + @staticmethod + def _bilingual_response(pt: tuple[str, str], en: tuple[str, str]): + node = { + "namePt": pt[1], + "nameEn": en[1], + "dataset": {"namePt": pt[0], "nameEn": en[0]}, + } + return {"data": {"allTable": {"edges": [{"node": node}]}}} + + @respx.mock + async def test_resolves_in_requested_language(self): + """The resolved name uses the requested language's fields.""" + respx.post(_GRAPHQL_URL).mock( + return_value=httpx.Response( + 200, + json=self._bilingual_response( + pt=("Diretórios", "Município"), en=("Directories", "Municipality") + ), + ) + ) + + assert await _resolve_table_name("tb1", "en") == "Directories — Municipality" + + @respx.mock + async def test_cache_is_keyed_per_language(self): + """The same table resolves independently per language — the cache must not collide.""" + route = respx.post(_GRAPHQL_URL).mock( + return_value=httpx.Response( + 200, + json=self._bilingual_response( + pt=("Diretórios", "Município"), en=("Directories", "Municipality") + ), + ) + ) + + pt_name = await _resolve_table_name("tb1", "pt") + en_name = await _resolve_table_name("tb1", "en") + + assert pt_name == "Diretórios — Município" + assert en_name == "Directories — Municipality" + # Two distinct cache keys => two lookups (not one shared, wrong-language entry). + assert route.call_count == 2 + @respx.mock async def test_builds_dataset_dash_table_name(self): """A resolved table yields '{dataset_name} — {table_name}'.""" @@ -36,7 +79,10 @@ async def test_builds_dataset_dash_table_name(self): ) ) - assert await _resolve_table_name("tb1") == "Diretórios Brasileiros — Município" + assert ( + await _resolve_table_name("tb1", "pt") + == "Diretórios Brasileiros — Município" + ) @respx.mock async def test_caches_successful_resolution(self): @@ -47,8 +93,8 @@ async def test_caches_successful_resolution(self): ) ) - first = await _resolve_table_name("tb1") - second = await _resolve_table_name("tb1") + first = await _resolve_table_name("tb1", "pt") + second = await _resolve_table_name("tb1", "pt") assert first == second == "Diretórios Brasileiros — Município" assert route.call_count == 1 @@ -60,8 +106,10 @@ async def test_returns_none_when_not_found(self): return_value=httpx.Response(200, json={"data": {"allTable": {"edges": []}}}) ) - assert await _resolve_table_name("missing") is None - assert "missing" not in _TABLE_NAME_CACHE + assert await _resolve_table_name("missing", "pt") is None + # Cache is keyed (table_id, language); assert the real key + # so this actually verifies that None results are not cached. + assert ("missing", "pt") not in _TABLE_NAME_CACHE @respx.mock async def test_returns_none_on_missing_dataset_name(self): @@ -72,7 +120,7 @@ async def test_returns_none_on_missing_dataset_name(self): ) ) - assert await _resolve_table_name("tb1") is None + assert await _resolve_table_name("tb1", "pt") is None @respx.mock async def test_returns_none_on_missing_table_name(self): @@ -83,14 +131,14 @@ async def test_returns_none_on_missing_table_name(self): ) ) - assert await _resolve_table_name("tb1") is None + assert await _resolve_table_name("tb1", "pt") is None @respx.mock async def test_returns_none_on_http_error(self): """A backend error is swallowed and resolves to None.""" respx.post(_GRAPHQL_URL).mock(return_value=httpx.Response(500)) - assert await _resolve_table_name("tb1") is None + assert await _resolve_table_name("tb1", "pt") is None @respx.mock async def test_returns_none_on_json_decode_error(self): @@ -101,7 +149,7 @@ async def test_returns_none_on_json_decode_error(self): return_value=httpx.Response(200, content=b"{'malformed': 'json'") ) - assert await _resolve_table_name("tb1") is None + assert await _resolve_table_name("tb1", "pt") is None class TestResolveDataSourceNames: @@ -111,7 +159,7 @@ async def test_resolved_name_overwrites_model_fallback( self, monkeypatch: pytest.MonkeyPatch ): """A successful resolution overwrites the model-provided fallback name.""" - resolve_name = AsyncMock(side_effect=lambda tid: f"Conjunto - {tid}") + resolve_name = AsyncMock(side_effect=lambda tid, language: f"Conjunto - {tid}") monkeypatch.setattr( "app.api.streaming.data_sources._resolve_table_name", resolve_name @@ -124,7 +172,7 @@ async def test_resolved_name_overwrites_model_fallback( ] } - await resolve_data_source_names(structured) + await resolve_data_source_names(structured, "pt") assert structured["data_sources"] == [ {"dataset_id": "ds1", "table_id": "tb1", "name": "Conjunto - tb1"}, @@ -146,7 +194,7 @@ async def test_keeps_model_fallback_when_unresolvable( ] } - await resolve_data_source_names(structured) + await resolve_data_source_names(structured, "pt") assert structured["data_sources"] == [ {"dataset_id": "ds1", "table_id": "tb1", "name": "model fallback"} @@ -162,8 +210,8 @@ async def test_no_data_sources_skips_resolution( "app.api.streaming.data_sources._resolve_table_name", resolve_name ) - await resolve_data_source_names({"data_sources": None}) - await resolve_data_source_names({"data_sources": []}) - await resolve_data_source_names({}) + await resolve_data_source_names({"data_sources": None}, "pt") + await resolve_data_source_names({"data_sources": []}, "pt") + await resolve_data_source_names({}, "pt") resolve_name.assert_not_awaited() diff --git a/tests/app/db/test_models.py b/tests/app/db/test_models.py new file mode 100644 index 0000000..218d513 --- /dev/null +++ b/tests/app/db/test_models.py @@ -0,0 +1,23 @@ +import pytest + +from app.db.models import ThreadPayload + + +class TestThreadLanguageNormalization: + """`ThreadPayload.language` is the single normalization boundary: a `before` validator + coerces raw input to a valid LanguageCode so everything downstream can trust it.""" + + def test_normalizes_case(self): + assert ThreadPayload(title="t", language="EN").language == "en" + + @pytest.mark.parametrize("value", ["fr", "de", "unknown", ""]) + def test_coerces_unsupported_to_default_without_raising(self, value: str): + # The Literal field would reject these, but the `before` validator coerces first — + # this guards against someone dropping mode="before" (which would 422 real requests). + assert ThreadPayload(title="t", language=value).language == "pt" + + def test_explicit_null_falls_back_to_default(self): + assert ThreadPayload(title="t", language=None).language == "pt" + + def test_omitted_uses_default(self): + assert ThreadPayload(title="t").language == "pt" diff --git a/tests/app/test_i18n.py b/tests/app/test_i18n.py new file mode 100644 index 0000000..abfb720 --- /dev/null +++ b/tests/app/test_i18n.py @@ -0,0 +1,70 @@ +from typing import get_args + +import pytest + +from app.i18n import ( + DEFAULT_LANGUAGE, + LanguageCode, + MessageKey, + localized_field, + normalize_language, + translate, +) + + +class TestNormalizeLanguage: + @pytest.mark.parametrize("value", ["pt", "en", "es"]) + def test_supported_codes_pass_through(self, value: str): + assert normalize_language(value) == value + + @pytest.mark.parametrize("value", ["PT", "En", "ES"]) + def test_case_is_normalized(self, value: str): + assert normalize_language(value) == value.lower() + + @pytest.mark.parametrize("value", [None, "", "fr", "de", "unknown"]) + def test_unsupported_falls_back_to_default(self, value): + assert normalize_language(value) == DEFAULT_LANGUAGE + + +class TestTranslate: + def test_every_key_resolves_in_every_language(self): + # Guards against a key that's missing a language (translate would KeyError) + # or has empty text — the common failure when adding a message or a locale. + for key in MessageKey: + for language in get_args(LanguageCode): + text = translate(key, language) + assert text and text.strip(), f"{key} / {language} is empty" + + def test_returns_language_specific_text(self): + expired = MessageKey.RESULTS_EXPIRED + assert translate(expired, "en") != translate(expired, "pt") + assert translate(expired, "en") != translate(expired, "es") + + def test_does_not_fall_back_on_unsupported_language(self): + # Normalization is the caller's responsibility (see normalize_language); + # translate trusts it receives a valid LanguageCode. + with pytest.raises(KeyError): + translate(MessageKey.ERROR_UNEXPECTED, "fr") + + +class TestLocalizedField: + """The metadata-localization primitive: pick `{field}{Suffix}`, fall back to pt.""" + + def test_returns_requested_language_value(self): + node = {"namePt": "Município", "nameEn": "Municipality", "nameEs": "Municipio"} + assert localized_field(node, "name", "en") == "Municipality" + assert localized_field(node, "name", "es") == "Municipio" + assert localized_field(node, "name", "pt") == "Município" + + def test_falls_back_to_pt_when_requested_language_missing(self): + # Partial translation coverage: only pt is populated for this node. + node = {"namePt": "Município"} + assert localized_field(node, "name", "en") == "Município" + + def test_falls_back_to_pt_when_requested_language_is_empty(self): + # An empty localized value is treated as "not translated" (the `or` branch). + node = {"namePt": "Município", "nameEn": ""} + assert localized_field(node, "name", "en") == "Município" + + def test_returns_none_when_neither_language_present(self): + assert localized_field({}, "name", "en") is None diff --git a/uv.lock b/uv.lock index 953869a..0e96e74 100644 --- a/uv.lock +++ b/uv.lock @@ -139,7 +139,7 @@ dependencies = [ { name = "google-cloud-storage" }, { name = "httpx" }, { name = "langchain" }, - { name = "langchain-google-genai" }, + { name = "langchain-openai" }, { name = "langgraph" }, { name = "langgraph-checkpoint-postgres" }, { name = "langsmith" }, @@ -173,7 +173,7 @@ requires-dist = [ { name = "google-cloud-storage", specifier = ">=3.10.1" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "langchain", specifier = ">=1.2.0" }, - { name = "langchain-google-genai", specifier = ">=4.1.3" }, + { name = "langchain-openai", specifier = ">=1.4.1" }, { name = "langgraph", specifier = ">=1.0.5" }, { name = "langgraph-checkpoint-postgres", specifier = ">=3.0.2" }, { name = "langsmith", specifier = ">=0.6.0" }, @@ -451,15 +451,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b5/36/7fb70f04bf00bc646cd5bb45aa9eddb15e19437a28b8fb2b4a5249fac770/filelock-3.20.3-py3-none-any.whl", hash = "sha256:4b0dda527ee31078689fc205ec4f1c1bf7d56cf88b6dc9426c4f230e46c2dce1", size = 16701, upload-time = "2026-01-09T17:55:04.334Z" }, ] -[[package]] -name = "filetype" -version = "1.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bb/29/745f7d30d47fe0f251d3ad3dc2978a23141917661998763bebb6da007eb1/filetype-1.2.0.tar.gz", hash = "sha256:66b56cd6474bf41d8c54660347d37afcc3f7d1970648de365c102ef77548aadb", size = 998020, upload-time = "2022-11-02T17:34:04.141Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/79/1b8fa1bb3568781e84c9200f951c735f3f157429f44be0495da55894d620/filetype-1.2.0-py2.py3-none-any.whl", hash = "sha256:7ce71b6880181241cf7ac8697a2f1eb6a8bd9b429f7ad6d27b8db9ba5f1c2d25", size = 19970, upload-time = "2022-11-02T17:34:01.425Z" }, -] - [[package]] name = "google-api-core" version = "2.28.1" @@ -496,11 +487,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/54/b03b568bff5748fd62327a1e36f40dcfa436eaf592fd7a481aa8bd4a3ee7/google_auth-2.46.0-py3-none-any.whl", hash = "sha256:fa51659c3745cb7024dd073f4ab766222767ea5f7dee2472110eaa03c9dbd2cb", size = 233748, upload-time = "2026-01-05T21:31:45.839Z" }, ] -[package.optional-dependencies] -requests = [ - { name = "requests" }, -] - [[package]] name = "google-cloud-bigquery" version = "3.39.0" @@ -567,27 +553,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/92/b1/d3cbd4d988afb3d8e4db94ca953df429ed6db7282ed0e700d25e6c7bfc8d/google_crc32c-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f", size = 34435, upload-time = "2025-12-16T00:35:22.107Z" }, ] -[[package]] -name = "google-genai" -version = "1.56.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "google-auth", extra = ["requests"] }, - { name = "httpx" }, - { name = "pydantic" }, - { name = "requests" }, - { name = "sniffio" }, - { name = "tenacity" }, - { name = "typing-extensions" }, - { name = "websockets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/70/ad/d3ac5a102135bd3f1e4b1475ca65d2bd4bcc22eb2e9348ac40fe3fadb1d6/google_genai-1.56.0.tar.gz", hash = "sha256:0491af33c375f099777ae207d9621f044e27091fafad4c50e617eba32165e82f", size = 340451, upload-time = "2025-12-17T12:35:05.412Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/84/93/94bc7a89ef4e7ed3666add55cd859d1483a22737251df659bf1aa46e9405/google_genai-1.56.0-py3-none-any.whl", hash = "sha256:9e6b11e0c105ead229368cb5849a480e4d0185519f8d9f538d61ecfcf193b052", size = 426563, upload-time = "2025-12-17T12:35:03.717Z" }, -] - [[package]] name = "google-resumable-media" version = "2.8.0" @@ -824,6 +789,46 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "jiter" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/1f/10936e16d8860c70698a1aa939a46aa0224813b782bce4e000e637da0b2d/jiter-0.16.0.tar.gz", hash = "sha256:7b24c3492c5f4f84a37946ad9cf504910cf6a782d6a4e0689b6673c5894b4a1c", size = 176431, upload-time = "2026-06-29T13:05:13.657Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/2b/52ace16ed031354f0539749a49e4bf33797d82bea5137910835fa4b09793/jiter-0.16.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:67c3bc1760f8c99d805dcab4e644027142a53b1d5d861f18780ebdbd5d40b72a", size = 306943, upload-time = "2026-06-29T13:03:14.035Z" }, + { url = "https://files.pythonhosted.org/packages/94/2e/34957c2c1b661c252ba9bcc60ae0bddc27e0f7202c6073326a13c5390eec/jiter-0.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5af7780e4a26bd7d0d989592bf9ef12ebf806b74ab709223ecca37c749872ea9", size = 307779, upload-time = "2026-06-29T13:03:15.418Z" }, + { url = "https://files.pythonhosted.org/packages/88/6c/59bd309cab4460c54cf1079f3eb7fe7af6a4c895c5c957a53378693bad2b/jiter-0.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5bf78d0e05e45cfdd66558893938d59afe3d1b1a824a202039b20e607d25a72", size = 335826, upload-time = "2026-06-29T13:03:17.11Z" }, + { url = "https://files.pythonhosted.org/packages/3b/8c/f5ef7b65f0df47afa16596969defb281ebb86e96df346d62be6fd853d620/jiter-0.16.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f4444a83f946605990c98f625cdd3d2725bfb818158760c5748c653170a20e0e", size = 362573, upload-time = "2026-06-29T13:03:18.781Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0b/ace4354da061ee38844a0c27dc2c21eecd27aea119e8da324bea987522d0/jiter-0.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3a23f0e4f957e1be65752d2dfac9a5a06b1917af8dc85deb639c3b9d02e31290", size = 457979, upload-time = "2026-06-29T13:03:20.293Z" }, + { url = "https://files.pythonhosted.org/packages/55/40/c0253d3772eb9dcd8e6606ee9b2d53ec8e5b814589c47f140aa585f21eaa/jiter-0.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c22a488f7b9218e245a0025a9ba6b100e2e54700831cf4cf16833a27fba3ad01", size = 372302, upload-time = "2026-06-29T13:03:21.739Z" }, + { url = "https://files.pythonhosted.org/packages/a8/d2/4839422241aa12860ce597b20068727094ba0bc480723c74924ca5bad483/jiter-0.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46add52f4ad47a08bfb1219f3e673da972191489a33016edefdb5ea55bfa8c48", size = 343805, upload-time = "2026-06-29T13:03:23.384Z" }, + { url = "https://files.pythonhosted.org/packages/e2/59/e196888a05befdda7dbe299b722d56f2f6eec65402bc34c0a3306d595feb/jiter-0.16.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:9c8a956fd72c2cf1e730d01ea080341f13aa0a97a4a33b51abebe725b7ae9ca9", size = 351107, upload-time = "2026-06-29T13:03:24.815Z" }, + { url = "https://files.pythonhosted.org/packages/ec/74/4cd9e0fca65232136400354b630fbfcd2de634e22ccbb96567725981b548/jiter-0.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:561926e0573ffe4a32498420a76d64b16c513e1ab413b9d28158a8764ac701e5", size = 388441, upload-time = "2026-06-29T13:03:26.266Z" }, + { url = "https://files.pythonhosted.org/packages/d9/8c/554691e48bc711299c0a293dd8a6179e24b2d66a54dc295421fcf64569c0/jiter-0.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:44d019fa8cdaf89bf29c71b39e3712143fdd0ac76725c6ef954f9957a5ea8730", size = 516354, upload-time = "2026-06-29T13:03:28.02Z" }, + { url = "https://files.pythonhosted.org/packages/a4/cb/01e9d69dc2cc6759d4f91e230b34489c4fdb2518992650633f9e20bece89/jiter-0.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0df91907609837f33341b8e6fe73b95991fdaa57caf1a0fbd343dffe826f386f", size = 547880, upload-time = "2026-06-29T13:03:29.534Z" }, + { url = "https://files.pythonhosted.org/packages/79/70/2953195f1c6ad00f49fa67e13df7e60acb3dd4f387101bc15abccddd905e/jiter-0.16.0-cp312-cp312-win32.whl", hash = "sha256:51d7b836acb0108d7c77df1742332cac2a1fa04a74d6dacec46e7091f0e91274", size = 203473, upload-time = "2026-06-29T13:03:31.025Z" }, + { url = "https://files.pythonhosted.org/packages/2d/05/2909a8b10699a4d560f8c502b6b2c5f3991b682b1922c1eedda242b225bd/jiter-0.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:1878349266f8ee36ecb1375cc5ba2f115f35fd9f0a1a4119e725e379126647f7", size = 196905, upload-time = "2026-06-29T13:03:32.472Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a9/6b82bb1c8d7790d602489b967b982a909e5d092875a6c2ade96444c8dfc5/jiter-0.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:2ed5738ae4af18271a51a528b8811b0cbfa4a1858de9d83359e4169855d6a331", size = 190618, upload-time = "2026-06-29T13:03:34.672Z" }, + { url = "https://files.pythonhosted.org/packages/91/c0/555fc60473d30d66894ba825e63615e3be7524fac23858356afa7a38906c/jiter-0.16.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:41977aa5654023948c2dae2a81cbf9c43343954bef1cd59a154dd15a4d84c195", size = 306203, upload-time = "2026-06-29T13:03:36.243Z" }, + { url = "https://files.pythonhosted.org/packages/d0/2b/c3eaf16f5d7c9bad66ea32f40a95bd169b29a91217fcc7f081375157e99c/jiter-0.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d28bb3c26762358dadf3e5bf0bccd29ae987d65e6988d2e6f49829c76b003c09", size = 306489, upload-time = "2026-06-29T13:03:37.846Z" }, + { url = "https://files.pythonhosted.org/packages/96/3f/02fdfc6705cad96127d883af5c34e4867f554f29ec7705ec1a46156400a9/jiter-0.16.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0542a7189c26920778658fc8fcf2af8bae05bae9924577f71804acef37996536", size = 335453, upload-time = "2026-06-29T13:03:39.221Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a6/e4bda5920d4b0d7c5dfb7174ce4a6b2e4d3e11c9162c452ef0eab4cdbdbd/jiter-0.16.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8fb8de1e23a0cb2a7f53c335049c7b72b6db41aa6227cdcc0972a1de5cb39450", size = 361625, upload-time = "2026-06-29T13:03:40.597Z" }, + { url = "https://files.pythonhosted.org/packages/b7/97/4e6b59b2c6e55cbb3e183595f81ad65dcfb21c915fee5e19e335df21bc55/jiter-0.16.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b72d0b2990ca754a9102779ac98d8597b7cb31678958562214a007f909eab78e", size = 456958, upload-time = "2026-06-29T13:03:42.074Z" }, + { url = "https://files.pythonhosted.org/packages/15/e0/97e9557686d2f94f4b93786eccb7eed28e9228ad132ea8237f44727314a7/jiter-0.16.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5f91b1c27fc22a57993d5a5cb8a627cb8ed4b10502716fac1ffbfe1d19d84e8", size = 372017, upload-time = "2026-06-29T13:03:43.658Z" }, + { url = "https://files.pythonhosted.org/packages/0f/94/db768b6938e0df35c86beeba3dfbbb025c9ee5c19e1aa271f2396e50864d/jiter-0.16.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c682bea068a90b764577bdb78a60a4c1d1606daf9cd4c893832a37c7cc9d9026", size = 343320, upload-time = "2026-06-29T13:03:45.226Z" }, + { url = "https://files.pythonhosted.org/packages/c1/d6/5a59d938244a30735fe62d9433fd325f9021ea29d89780ea4596ea93bc89/jiter-0.16.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:8d031aabecc4f1b6276adfb42e3aabb77c89d468bf616600e8d3a11328929053", size = 350520, upload-time = "2026-06-29T13:03:46.671Z" }, + { url = "https://files.pythonhosted.org/packages/67/f8/c4a857f49c9af125f6bbcac7e3eee7f7978ed89682833062e2dbf62576b1/jiter-0.16.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eab2cd170150e70153de16896a1774e3a1dca80154c56b54d7a812c479a7165e", size = 387550, upload-time = "2026-06-29T13:03:48.361Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d6/5fbc2f7d6b67b754caa61a993a2e626e815dec47ffc2f9e35f01adfebec7/jiter-0.16.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:6edb63a46e65a82c26800a868e49b2cac30dd5a4218b88d74bc2c848c8ad60bb", size = 515424, upload-time = "2026-06-29T13:03:49.881Z" }, + { url = "https://files.pythonhosted.org/packages/ed/54/284f0164b64a5fed915fea6ba7e9ba9b3d8d37c67d59cf2e3bb99d45cdfe/jiter-0.16.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:659039cc50b5addcc35fcc87ae2c1833b7c0a8e5326ef631a75e4478447bcf84", size = 546981, upload-time = "2026-06-29T13:03:51.363Z" }, + { url = "https://files.pythonhosted.org/packages/13/c5/2a467585a576594384e1d2c43e1224deaafc085f24e243529cf98beef8e1/jiter-0.16.0-cp313-cp313-win32.whl", hash = "sha256:c9c53be232c2e206ef9cdbad81a48bfa74c3d3f08bcf8124630a8a748aad993e", size = 202853, upload-time = "2026-06-29T13:03:53.015Z" }, + { url = "https://files.pythonhosted.org/packages/88/6a/de61d04b9eec69c71719968d2f716532a3bc121170c44a39e14979c6be81/jiter-0.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:baad945ed47f163ad833314f8e3288c396118934f94e7bbb9e243ce4b341a4fd", size = 196160, upload-time = "2026-06-29T13:03:54.447Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/b390ed59bafb3f31d008d1218578f10327714484b334439947f7e5b11e7f/jiter-0.16.0-cp313-cp313-win_arm64.whl", hash = "sha256:3c1fd2dbe1b0af19e987f03fe66c5f5bd105a2229c1aff4ab14890b24f41d21a", size = 189862, upload-time = "2026-06-29T13:03:55.754Z" }, + { url = "https://files.pythonhosted.org/packages/98/ab/664fd8c4be028b2bedd3d2ff08769c4ede23d0dbc87a77c62384a0515b5d/jiter-0.16.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:f17d61a28b4b3e0e3e2ba98490c70501403b4d196f78732439160e7fd3678127", size = 303106, upload-time = "2026-06-29T13:05:07.118Z" }, + { url = "https://files.pythonhosted.org/packages/1a/07/421f1d5b65493a76e16027b848aba6a7d28073ae75944fa4289cc914d39f/jiter-0.16.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:96e38eea538c8ddf853a35727c7be0741c76c13f04148ac5c116222f50ece3b3", size = 304658, upload-time = "2026-06-29T13:05:08.708Z" }, + { url = "https://files.pythonhosted.org/packages/0a/db/bba1155f01a01c3c37a89425d571da751bbedf5c54247b831a04cb971798/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d284fb8d94d5855d60c44fefcab4bf966f1da6fada73992b01f6f0c9bc0c6702", size = 339719, upload-time = "2026-06-29T13:05:10.41Z" }, + { url = "https://files.pythonhosted.org/packages/78/f7/18a1afcd64f35314b68c1f23afcd9994d0bc13e65cc77517afff4e83986d/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64d613743df53199b1aa256a7d328340da6d7078aac7705a7db9d7a791e9cfd2", size = 343885, upload-time = "2026-06-29T13:05:12.087Z" }, +] + [[package]] name = "jsonpatch" version = "1.33" @@ -861,10 +866,11 @@ wheels = [ [[package]] name = "langchain-core" -version = "1.2.6" +version = "1.5.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonpatch" }, + { name = "langchain-protocol" }, { name = "langsmith" }, { name = "packaging" }, { name = "pydantic" }, @@ -873,24 +879,35 @@ dependencies = [ { name = "typing-extensions" }, { name = "uuid-utils" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b9/ce/ba5ed5ea6df22965b2893c2ed28ebb456204962723d408904c4acfa5e942/langchain_core-1.2.6.tar.gz", hash = "sha256:b4e7841dd7f8690375aa07c54739178dc2c635147d475e0c2955bf82a1afa498", size = 833343, upload-time = "2026-01-02T21:35:44.749Z" } +sdist = { url = "https://files.pythonhosted.org/packages/65/3e/63af6b9d76d9be907c7c524d6ec18a2efed7e0e2d123fea0230d78dbd73f/langchain_core-1.5.3.tar.gz", hash = "sha256:a56457ac444fef41e9404443c187f0ecea708d36e816ea4ba9573c027f7d1a2d", size = 972461, upload-time = "2026-07-30T14:55:55.833Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6f/40/0655892c245d8fbe6bca6d673ab5927e5c3ab7be143de40b52289a0663bc/langchain_core-1.2.6-py3-none-any.whl", hash = "sha256:aa6ed954b4b1f4504937fe75fdf674317027e9a91ba7a97558b0de3dc8004e34", size = 489096, upload-time = "2026-01-02T21:35:43.391Z" }, + { url = "https://files.pythonhosted.org/packages/36/e6/c7c39efe0bc7e1b7c3d8f54f85846e04c901913c3d3e99068b218558c6f1/langchain_core-1.5.3-py3-none-any.whl", hash = "sha256:48b56fa580277209594dd7baf837f5b9a2a3651613f34ff9fb1728b429df015f", size = 561687, upload-time = "2026-07-30T14:55:54.419Z" }, ] [[package]] -name = "langchain-google-genai" -version = "4.1.3" +name = "langchain-openai" +version = "1.4.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "filetype" }, - { name = "google-genai" }, { name = "langchain-core" }, - { name = "pydantic" }, + { name = "openai" }, + { name = "tiktoken" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bf/1b/a83bf6cae4632363cef0b6f2ee1b4f62c8a5ebcf22cd8ef24430a736c2a8/langchain_openai-1.4.1.tar.gz", hash = "sha256:6d16be615d997db80294731b8e768783f1fb8e0313668e64acd50cd68acbad20", size = 3262416, upload-time = "2026-07-23T20:31:13.053Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/1c/8b604dc8be2735c8ae5c655e520066231057d1301958c20c776e62bd00fb/langchain_openai-1.4.1-py3-none-any.whl", hash = "sha256:8528bb34cc78fdfd2d895573c7917f9441cbb82db5f18ae0e6b3b75d95bdefb3", size = 122067, upload-time = "2026-07-23T20:31:11.809Z" }, +] + +[[package]] +name = "langchain-protocol" +version = "0.0.18" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ba/85/078d5aba488a82a53b8372ac1037dee4f64b020bac69e6a07e37a5059059/langchain_google_genai-4.1.3.tar.gz", hash = "sha256:28966c8fe58c9a401fdc37aeeeb0eb51744210803838ce050f022fc53d2f994e", size = 277024, upload-time = "2026-01-05T23:29:34.362Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d2/59/b5959aea96faa9146e2e49a7a22882b3528c62efafe9a6a95beab30c2305/langchain_protocol-0.0.18.tar.gz", hash = "sha256:ec3e11782f1ed0c9db38e5a9ed01b0e7a0d3fba406faa8aef6594b73c56a63e6", size = 6150, upload-time = "2026-06-18T17:08:26.959Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/aa/ca61dc2d202a23d7605a5c0ea24bd86a39a5c23c932a166b87c7797747c5/langchain_google_genai-4.1.3-py3-none-any.whl", hash = "sha256:5d710e2dcf449d49704bdbcd31729be90b386fa008395f9552a5c090241de1a5", size = 66262, upload-time = "2026-01-05T23:29:32.924Z" }, + { url = "https://files.pythonhosted.org/packages/99/2e/d82db9eec13ad0f72e7aaad5c4bc730ab111934fdc83c85523206eb9b0a0/langchain_protocol-0.0.18-py3-none-any.whl", hash = "sha256:70b53a86fbf9cedc863555effe44da192ab02d556ddbf2cf95b8873adcf41b5a", size = 7221, upload-time = "2026-06-18T17:08:25.996Z" }, ] [[package]] @@ -1100,6 +1117,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, ] +[[package]] +name = "openai" +version = "2.51.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/75/d8/06fda9685e47d9a8fc177ef57f8af75207938fad49a45ce23bfa7b6a2a5c/openai-2.51.0.tar.gz", hash = "sha256:4d61287c42eba54086d09346e709cbf7f8cec51822efce9cc399450b9385fba5", size = 1083410, upload-time = "2026-07-30T17:43:26.507Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/6f/c49e21245ad4865e886f6885e95e998bc024335b392c202bd571639e9d50/openai-2.51.0-py3-none-any.whl", hash = "sha256:91db13ce59a670fddc820a6983989650095e43e3acac09288bceb69356a8904e", size = 1652344, upload-time = "2026-07-30T17:43:24.225Z" }, +] + [[package]] name = "orjson" version = "3.11.5" @@ -1586,6 +1622,62 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, ] +[[package]] +name = "regex" +version = "2026.7.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/98/04b13f1ddfb63158025291c02e03eb42fbb7acb51d091d541050eb4e35e8/regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5", size = 416440, upload-time = "2026-07-19T00:19:48.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/b9/d11d7e501ac8fd7d617684423ebb9561e0b998481c1e4cbc0cb212c5d74a/regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d", size = 496778, upload-time = "2026-07-19T00:17:05.677Z" }, + { url = "https://files.pythonhosted.org/packages/3f/a9/a5ab6f312f24318019170dc485d5421fe4f89e43a98640da50d95a8a7041/regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd", size = 297122, upload-time = "2026-07-19T00:17:07.59Z" }, + { url = "https://files.pythonhosted.org/packages/b3/63/4cab4d7f2d384a144d420b763d97674cb70619c878ea6fcd7640d0e62143/regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6", size = 292009, upload-time = "2026-07-19T00:17:09.648Z" }, + { url = "https://files.pythonhosted.org/packages/22/85/102a81b218298957d4ea7d2f084fae537a71add9d6ff93c8e67284c5f45e/regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797", size = 796708, upload-time = "2026-07-19T00:17:11.542Z" }, + { url = "https://files.pythonhosted.org/packages/78/b5/dc136af5629938a037cd2b304c12240e132ec92f38be8ff9cc89af2a1f2d/regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18", size = 865651, upload-time = "2026-07-19T00:17:13.312Z" }, + { url = "https://files.pythonhosted.org/packages/e0/75/67402ae3cd9c8c988a4c805d15ee3eef015e7ca4cb112cf3e640fc1f4153/regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511", size = 911756, upload-time = "2026-07-19T00:17:15.015Z" }, + { url = "https://files.pythonhosted.org/packages/2a/8e/096d00c7c480ef2ff4265349b14e2261d4ab787ba1f74e2e80d1c58079c3/regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68", size = 801798, upload-time = "2026-07-19T00:17:17.208Z" }, + { url = "https://files.pythonhosted.org/packages/f0/41/e7ecac6edb5722417f85cc67eaf386322fbe8acf6918ec2fdc37c20dd9d0/regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11", size = 776933, upload-time = "2026-07-19T00:17:19.347Z" }, + { url = "https://files.pythonhosted.org/packages/6f/69/03c9b3f058d66403e0ca2c938696e81d51cd4c6d47ec5265f02f96948d9a/regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986", size = 784338, upload-time = "2026-07-19T00:17:21.057Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f7/b38ab3d43f284afbb618fcd15d0e77eb786ae461ce1f6bc7494619ddc0f2/regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b", size = 860452, upload-time = "2026-07-19T00:17:23.119Z" }, + { url = "https://files.pythonhosted.org/packages/15/5c/ff60ef0571121714f3cf9920bc183071e384a10b556d042e0fdb06cc07a5/regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb", size = 765958, upload-time = "2026-07-19T00:17:24.81Z" }, + { url = "https://files.pythonhosted.org/packages/aa/0f/bd34021162c0ab47f9a315bd56cd5642e920c8e5668a75ef6c6a6fca590d/regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035", size = 851765, upload-time = "2026-07-19T00:17:26.993Z" }, + { url = "https://files.pythonhosted.org/packages/2a/20/a2ca43edade0595cccfdc98636739f536d9e26898e7dbddc2b9e98898953/regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a", size = 789714, upload-time = "2026-07-19T00:17:28.699Z" }, + { url = "https://files.pythonhosted.org/packages/5d/47/e02db4015d424fc83c00ea0ac8c5e5ec14397943de9abf909d5ce3a25931/regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5", size = 267157, upload-time = "2026-07-19T00:17:31.051Z" }, + { url = "https://files.pythonhosted.org/packages/08/8e/c780c131f79b42ed22d1bd7da4096c2c35f813e835acd02ef0f018bd892c/regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312", size = 277777, upload-time = "2026-07-19T00:17:32.848Z" }, + { url = "https://files.pythonhosted.org/packages/3e/4c/e4d7e086449bdf379d89774bf1f89dc4a41943f3c5a6125a03905b34b5fb/regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d", size = 277136, upload-time = "2026-07-19T00:17:34.803Z" }, + { url = "https://files.pythonhosted.org/packages/5d/3d/84165e4299ff76f3a40fe1f2abf939e976f693383a08d2beea6af62bd2c1/regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40", size = 496552, upload-time = "2026-07-19T00:17:36.808Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/a65293e6e4cf28eb7ee1be5335a5386c40d6742e9f47fafc8fec785e16c7/regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38", size = 296983, upload-time = "2026-07-19T00:17:38.816Z" }, + { url = "https://files.pythonhosted.org/packages/95/47/2d0564e93d87bc48618360ddca232a2ca612bbdf53ce8465d45ca5ce14ee/regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11", size = 291832, upload-time = "2026-07-19T00:17:40.726Z" }, + { url = "https://files.pythonhosted.org/packages/07/cd/42dfbabff3dfc9603c501c0e2e2c5adbb09d127b267bf5348de0af338c15/regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13", size = 796775, upload-time = "2026-07-19T00:17:42.382Z" }, + { url = "https://files.pythonhosted.org/packages/df/5d/f6a4839f2b934e3eed5973fd07f5929ee97d4c98939fb275ea23c274ee16/regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae", size = 865687, upload-time = "2026-07-19T00:17:44.185Z" }, + { url = "https://files.pythonhosted.org/packages/14/b0/b47d6c36049bc59806a50bd4c86ced70bbe058d787f80281b1d7a9b0e024/regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da", size = 911962, upload-time = "2026-07-19T00:17:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/2a/be/ff61f28f9273658cfe23acbbac5217221f6519960ed401e61dfdab12bc35/regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15", size = 801817, upload-time = "2026-07-19T00:17:48.25Z" }, + { url = "https://files.pythonhosted.org/packages/c3/bb/8b4f7f26b333f9f79e1b453613c39bb4776f51d38ae66dd0ba31d6b354ca/regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f", size = 776908, upload-time = "2026-07-19T00:17:50.183Z" }, + { url = "https://files.pythonhosted.org/packages/09/13/610110fc5921d380516d03c26b652555f08aa0d23ea78a771231873c3638/regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939", size = 784426, upload-time = "2026-07-19T00:17:52.454Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f5/1ef9e2a83a5947c57ebff0b377cb5727c3d5ec1992317a320d035cd0dbb6/regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96", size = 860600, upload-time = "2026-07-19T00:17:54.229Z" }, + { url = "https://files.pythonhosted.org/packages/a0/02/073af33a3ec149241d11c80acea91e722aa0adbf05addd50f251c4fe89c3/regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220", size = 765950, upload-time = "2026-07-19T00:17:56.041Z" }, + { url = "https://files.pythonhosted.org/packages/81/a9/d1e9f819dc394a568ef370cd56cf25394e957a2235f8370f23b576e5a475/regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc", size = 851794, upload-time = "2026-07-19T00:17:57.897Z" }, + { url = "https://files.pythonhosted.org/packages/03/3a/8ae83eda7579feacdf984e71fb9e70635fb6f832eeddca58427ec4fca926/regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2", size = 789845, upload-time = "2026-07-19T00:17:59.97Z" }, + { url = "https://files.pythonhosted.org/packages/4b/23/c195cbfe5a75fdec64d8f6554fd15237b837919d2c61bdc141d7c807b08b/regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404", size = 267135, upload-time = "2026-07-19T00:18:01.958Z" }, + { url = "https://files.pythonhosted.org/packages/b2/80/a11de8404b7272b70acb45c1c05987cce60b45d5693da2e176f0e390d564/regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e", size = 277747, upload-time = "2026-07-19T00:18:04.121Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/0f5c8eff1b4f1f3d83276d365fccecf666afcc7d947420943bf394d07adb/regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8", size = 277129, upload-time = "2026-07-19T00:18:05.821Z" }, + { url = "https://files.pythonhosted.org/packages/dc/4c/44b74742052cedda40f9ae469532a037112f7311a36669a891fba8984bb0/regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2", size = 501134, upload-time = "2026-07-19T00:18:07.567Z" }, + { url = "https://files.pythonhosted.org/packages/f0/45/bbd038b5e39ee5613a5a689290145b40058cc152c41de9cc23639d2b9734/regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda", size = 299418, upload-time = "2026-07-19T00:18:09.38Z" }, + { url = "https://files.pythonhosted.org/packages/65/38/c5bde94b4cedfd5850d64c3f08222d8e1600e84f6ee71d9b44b4b8163f74/regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff", size = 294486, upload-time = "2026-07-19T00:18:11.188Z" }, + { url = "https://files.pythonhosted.org/packages/d7/6a/2f5e107cb26c960b781967178899daf2787a7ab151844ed3c01d6fc95474/regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1", size = 811643, upload-time = "2026-07-19T00:18:12.975Z" }, + { url = "https://files.pythonhosted.org/packages/37/d4/a2f963406d7d73a62eed84ba05a258afb6cad1b21aa4517443ce40506b78/regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf", size = 871081, upload-time = "2026-07-19T00:18:14.733Z" }, + { url = "https://files.pythonhosted.org/packages/45/a3/44be546340bedb15f13063f5e7fe16793ea4d9ea2e805d09bd174ac27724/regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732", size = 917372, upload-time = "2026-07-19T00:18:16.724Z" }, + { url = "https://files.pythonhosted.org/packages/f8/f6/e0870b0fd2a40dba0074e4b76e514b21313d37946c9248453e34ec43923e/regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a", size = 816089, upload-time = "2026-07-19T00:18:18.617Z" }, + { url = "https://files.pythonhosted.org/packages/ae/27/957e8e22690ad6634572b39b71f130a6105f4d0718bb16849eac00fff147/regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba", size = 785206, upload-time = "2026-07-19T00:18:20.464Z" }, + { url = "https://files.pythonhosted.org/packages/76/a4/186e410941e731037c01166069ab86da9f65e8f8110c18009ccf4bd623ee/regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc", size = 800431, upload-time = "2026-07-19T00:18:22.716Z" }, + { url = "https://files.pythonhosted.org/packages/73/9f/e4e10e023d291d64a33e246610b724493bf1ce98e0e59c9b7c837e5acfb7/regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62", size = 864906, upload-time = "2026-07-19T00:18:24.772Z" }, + { url = "https://files.pythonhosted.org/packages/24/57/ccb20b6be5f1f52a053d1ba2a8f7a077edb9d918248b8490d7506c6832b3/regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1", size = 773559, upload-time = "2026-07-19T00:18:27.008Z" }, + { url = "https://files.pythonhosted.org/packages/a3/82/f3b263cf8fad927dc102891da8502e718b7ff9d19af7a2a07c03865d7188/regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e", size = 857739, upload-time = "2026-07-19T00:18:29.107Z" }, + { url = "https://files.pythonhosted.org/packages/47/2e/1687bd1b6c2aed5e672ccf845fc11557821fe7366d921b50889ea5ce57bf/regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0", size = 804522, upload-time = "2026-07-19T00:18:31.362Z" }, + { url = "https://files.pythonhosted.org/packages/76/7c/cc4e7655181b2d9235b704f2c5e19d8eff002bbc437bae59baee0e381aca/regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4", size = 269141, upload-time = "2026-07-19T00:18:33.479Z" }, + { url = "https://files.pythonhosted.org/packages/bb/14/961b4c7b05a2391c32dbc85e27773076671ef8f97f36cec70fe414734c02/regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974", size = 280036, upload-time = "2026-07-19T00:18:35.419Z" }, + { url = "https://files.pythonhosted.org/packages/ce/67/795644550d788ddbb6dc458c95895f8009978ea6d6ea76b005eb3f45e8c9/regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d", size = 279394, upload-time = "2026-07-19T00:18:37.454Z" }, +] + [[package]] name = "requests" version = "2.32.5" @@ -1860,6 +1952,51 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ea/c4/53efc88d890d7dd38337424a83bbff32007d9d3390a79a4b53bfddaa64e8/testcontainers-4.14.0-py3-none-any.whl", hash = "sha256:64e79b6b1e6d2b9b9e125539d35056caab4be739f7b7158c816d717f3596fa59", size = 125385, upload-time = "2026-01-07T23:35:21.343Z" }, ] +[[package]] +name = "tiktoken" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "regex" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/e5/5f3cb2159769d0f4324c0e9e87f9de3c4b1cd45848a96b2eb3566ad5ca77/tiktoken-0.13.0.tar.gz", hash = "sha256:c9435714c3a84c2319499de9a300c0e604449dd0799ff246458b3bb6a7f433c1", size = 38986, upload-time = "2026-05-15T04:51:27.153Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/8e/144bde4e01df66b34bb865557c7cd754ed08b036217ebd79c9db5e9048a9/tiktoken-0.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:32ac870a806cfb260a02d0cb70426aef02e038297f8ad50df5040bb5af360791", size = 1034888, upload-time = "2026-05-15T04:50:31.579Z" }, + { url = "https://files.pythonhosted.org/packages/36/18/d4ac9d20956cdebca04841316660ed584c2fecdc2b81722a28bc7ad3b1e4/tiktoken-0.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4d9980f11429ed2d737c463bb1fb78cf330caa026adf002f714aced7849a687b", size = 982970, upload-time = "2026-05-15T04:50:32.961Z" }, + { url = "https://files.pythonhosted.org/packages/74/ed/6bb8d05b9f731f749fee5c6f5ca63e981143c826a5985877330507bd13b7/tiktoken-0.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:3f277ebea5edd7b8bf03c6f9431e1d67d517530115572b2dc1d465326e8f88c7", size = 1115741, upload-time = "2026-05-15T04:50:34.475Z" }, + { url = "https://files.pythonhosted.org/packages/34/de/2ca96b07a82d972b74fe4b46de055b79c904e45c7eab699354a0bfa697dc/tiktoken-0.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a116178fa7e1b4065bff05214360373a65cac22f965be7b3f73d00a0dbfe7649", size = 1136523, upload-time = "2026-05-15T04:50:35.782Z" }, + { url = "https://files.pythonhosted.org/packages/ee/dc/9dafec002c2d4424378563cf4cf5c7fb93631d2a55013c8b87554ee4012c/tiktoken-0.13.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c397ddda233208345b01bd30f2fca79ff730e55731d0108a603f9bc57f6af3b", size = 1181954, upload-time = "2026-05-15T04:50:36.99Z" }, + { url = "https://files.pythonhosted.org/packages/a1/d0/1f8578c45b2f24759b46f0b50d31878c63c73e6bf0f2227e10ec5c5408dc/tiktoken-0.13.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:95097e4f89b06403976e498abf61a0ee73a7497e73fb599cb211d8197a054d91", size = 1240069, upload-time = "2026-05-15T04:50:38.221Z" }, + { url = "https://files.pythonhosted.org/packages/aa/90/28d7f154888610aa9237e541986beb62b479df29d193a5a0617dbb1514d0/tiktoken-0.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:8f2d16e7a7c783ad81f36e457d046d1f1c8af70b22aec8a13238efe531977c41", size = 874748, upload-time = "2026-05-15T04:50:39.587Z" }, + { url = "https://files.pythonhosted.org/packages/9c/83/b096c859c2a47c11731bf2f5885f4028b809dfe2396582883eed9cae372f/tiktoken-0.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5df5d1507bd245f1ccad4a074698240021239e455eb0bb4ced4e3d7181872154", size = 1034228, upload-time = "2026-05-15T04:50:40.988Z" }, + { url = "https://files.pythonhosted.org/packages/53/61/c68e123b6d753e3fc2751e9b18e732c9d8bf1e1926762e736eee935d931c/tiktoken-0.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8fe806a50664e83a6ffd56cbd1e4f5dcc6cd32a3e7538f70dc38b1a271384545", size = 982978, upload-time = "2026-05-15T04:50:42.195Z" }, + { url = "https://files.pythonhosted.org/packages/ef/8b/96cc178cc584e65d363134500f297790b06cd48cdeb1e8fcf7bbe60f4715/tiktoken-0.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:125bc05005e747f993a83dc67934249932d6e4209854452cd4c0b1d53fba3ba2", size = 1116355, upload-time = "2026-05-15T04:50:43.564Z" }, + { url = "https://files.pythonhosted.org/packages/86/f5/bab735d2c72ea55404b295d02d092644eb5f7cc6205e34d35eb9abfb9ab2/tiktoken-0.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5e6358911cab4adee6712da27d65573496a4f68cf8a2b5fca6a4ad10fc5748cf", size = 1135772, upload-time = "2026-05-15T04:50:44.782Z" }, + { url = "https://files.pythonhosted.org/packages/4e/b9/6de04ebdf904edfaad87788011b3735087a0c9ea671b9027e1e4e965e8c8/tiktoken-0.13.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:975cbd78d085d75d26b59660e262736dcaed1e35f8f142cd6291025c01d25486", size = 1182415, upload-time = "2026-05-15T04:50:46.422Z" }, + { url = "https://files.pythonhosted.org/packages/0d/9c/470a05f3b1caf038f44880e334d47ab674e0c80d514c66b375d14d5afa10/tiktoken-0.13.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:75ab9bc99fa020a4c283424590ecd7f3afd70c1c281cb3fa3192a6c3af9f9615", size = 1239879, upload-time = "2026-05-15T04:50:48.052Z" }, + { url = "https://files.pythonhosted.org/packages/42/a6/c1936d16055436cb32e6c6128d68629622e00f4768562f55653752d34768/tiktoken-0.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:6b1615f0ff71953d19729ceb18865429c185b0a23c5353f1bbca34a394bf60f7", size = 874829, upload-time = "2026-05-15T04:50:49.202Z" }, + { url = "https://files.pythonhosted.org/packages/d6/07/acb5992c3772b5a36284f742cfb7a5895aa4471d1848ac31464ad50d7fdf/tiktoken-0.13.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6eb4a5bfbc6426938026b1a334e898ac53541360d62d8c689870160cc80abd67", size = 1033600, upload-time = "2026-05-15T04:50:50.4Z" }, + { url = "https://files.pythonhosted.org/packages/14/e9/742e9aec30f59b9f161f7ff7cd072e02ea836c9e1c0854a8076dfcd40d5c/tiktoken-0.13.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:43cee3e5400573b2046fbf092cc7a5bc30164f9e4c95ce20714da929df48737a", size = 982516, upload-time = "2026-05-15T04:50:52.03Z" }, + { url = "https://files.pythonhosted.org/packages/72/74/ca1541b053e7648254d2e4b42a253e1bb4359f2c91a0a8d49228c794e1a0/tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:7de52e3f566d19b3b11bd37eea552c6c305ad74081f736882bd44d148ed4c48d", size = 1115518, upload-time = "2026-05-15T04:50:53.543Z" }, + { url = "https://files.pythonhosted.org/packages/46/e3/93825eaf5a4a504795b787e5d5dea07fbeb3dabf97aa7b450be8bde59c89/tiktoken-0.13.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:51384448aa508e4df84c0f7c1dc3211c7f7b8096325660ee5fc82f3e11b381ce", size = 1136867, upload-time = "2026-05-15T04:50:55.191Z" }, + { url = "https://files.pythonhosted.org/packages/8c/46/002b68de6827091d5ae90b048f326e8aad8d953520950e5ce1508879414f/tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e28157350f7ebf35008dd8e9e0fdb621f976e4230c881099c85e8cf07eaa50e2", size = 1181826, upload-time = "2026-05-15T04:50:56.296Z" }, + { url = "https://files.pythonhosted.org/packages/db/c6/d393e3185a276505182f7abd93fe714f3c444a2be9180798fa052347504e/tiktoken-0.13.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:165cf1820ea4a354985c2490a5205d4cc74661c934aca79dd0368232fff94e0f", size = 1239489, upload-time = "2026-05-15T04:50:57.918Z" }, + { url = "https://files.pythonhosted.org/packages/b7/4d/bc07d1f1635d4897a202acc0ae11c2886eaa7325c359ba4741b47bf8e225/tiktoken-0.13.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6c43a675ca14f6f2749ba7f12075d37456015a24b859f2517b9beb4ef30807ec", size = 873820, upload-time = "2026-05-15T04:50:59.528Z" }, +] + +[[package]] +name = "tqdm" +version = "4.70.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/21/3b/6c24bec5be5e743ffd99576daa5cc077722fc7d5bbc00bd133fa0c698dc6/tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220", size = 795438, upload-time = "2026-07-27T11:33:15.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/1c/01bfd571a64e7f270e6bab5e33777debe0edc56759233ce84f27dec92d14/tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953", size = 80184, upload-time = "2026-07-27T11:33:13.167Z" }, +] + [[package]] name = "traitlets" version = "5.14.3"