diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ef419db --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,20 @@ +name: ci + +on: + pull_request: + push: + branches: [main] + +jobs: + # Offline by design: no network, no credentials, sub-second. Live round trips + # are a manual pre-release step, not a per-PR gate. + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + - run: uv venv --python 3.12 + - run: uv pip install -e '.[dev]' + - run: uv run ruff check . + - run: uv run ruff format --check . + - run: uv run pytest -q diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..130baad --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.venv/ +__pycache__/ +*.egg-info/ +.pytest_cache/ +.ruff_cache/ +dist/ +build/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..9915d1d --- /dev/null +++ b/README.md @@ -0,0 +1,135 @@ +# unstract-cli + +`unstract` — one CLI for the Unstract suite: extract a document with +LLMWhisperer, run it through a Document Studio API deployment, get structured +JSON back. It also clones one organization's resources into another. + +```bash +uv tool install git+https://github.com/Zipstack/unstract-cli +unstract config init +unstract config doctor +``` + +Or run it without installing: `uvx --from git+https://github.com/Zipstack/unstract-cli unstract --discover groups`. + +## Output + +`unstract` prints a table by default — in a terminal and in a pipe alike, so +what you see while trying something is what a script sees running it. + +**Parsing anything? Pass `-o json`.** stdout then carries exactly one envelope, +on success and on failure alike: + +```json +{"ok": true, "data": {...}, "error": null, "meta": {"contract_version": 1}} +``` + +`-o json` output depends on nothing but the command and its arguments — not the +terminal, not the config, not the environment. `-o raw` prints one field +unwrapped, for piping a document's text somewhere else. Diagnostics, warnings +and progress always go to stderr. + +Consuming the JSON: ignore fields you do not recognise, and refuse a +`meta.contract_version` above the one you were written against. `unstract +--discover full` publishes the whole contract alongside every command and flag. + +If a coding agent is driving (detected from the environment it sets), the +*default* becomes json. `--agent yes|no` forces that either way, and an explicit +`-o` always wins over both. + +Failures exit non-zero with a stable code. The codes are this CLI's own +convention, not a service's — they are the `ExitCode` enum in +`core/errors.py`, and `--discover full` publishes the table so a caller does not +have to copy it: + +| Code | Meaning | +|------|---------| +| 0 | success | +| 1 | generic failure | +| 2 | usage error | +| 3 | authentication failed | +| 4 | not found | +| 5 | validation failed | +| 6 | rate limited | +| 7 | timed out (the job handle is in the error payload — resume, do not resubmit) | +| 8 | server error | +| 9 | result already consumed (one-shot read; use `--save` next time) | +| 10 | the result was read but could not be saved — it is in `error.details` | +| 130 | interrupted (128 + SIGINT) — the user stopped it, not a failure | + +## Configuration + +`~/.unstract/config.toml`, or a project-local `.unstract.toml` found by upward +search, or `$UNSTRACT_CONFIG`, or `--config`. Every setting resolves +**flag > env > profile > built-in default**, and the CLI is fully usable with no +config file at all. The flag tier is the connection options on each product +group — `unstract docstudio --base-url … --org-id … deployment run …`, and +`--base-url`/`--api-key` on `whisper` — which override the profile for that one +invocation without writing anything. + +```toml +default_profile = "cloud-us" + +[profiles.cloud-us.llmwhisperer] +base_url = "https://llmwhisperer-api.us-central.unstract.com/api/v2" +api_key = "env:LLMWHISPERER_API_KEY" + +[profiles.cloud-us.docstudio] +base_url = "https://us-central.unstract.com" +org_id = "org_ABC123" +api_key = "env:UNSTRACT_DEPLOYMENT_KEY" + +[profiles.cloud-us.deployments.invoices] +api_name = "invoice-parser" +``` + +One `api_key` on the `docstudio` block covers every alias under it: a key minted +under **Settings → API Key Manager** authenticates every API deployment in the +organisation, so an alias normally carries only its `api_name`. Give an alias its +own `api_key` when its deployment has a separate key of its own. + +Get an LLMWhisperer key from the LLMWhisperer console; a deployment key is shown +on the API deployment's own page in the Unstract UI, and an organisation-wide one +under Settings → API Key Manager. `config init` also writes an +`onprem-example` profile as a shape to copy for a self-hosted install — its host +is a placeholder, and only the *active* profile is ever resolved. + +A credential can be written into the file literally, but `env:VAR_NAME` +indirection is what `config init` writes and what the examples use: the file +then records where a secret lives rather than the secret itself, and stays safe +to copy or commit. Either way the file is created `0600`, and `config doctor` +warns when its mode is wider than that. + +`unstract config doctor` reports where each setting resolved from — including +whether an `env:` reference is actually set in the current process — without +echoing any value. It exits non-zero when one of its own checks failed, so a +setup script can branch on it. + +A project-local `.unstract.toml` **found by upward search** may not supply +`api_key` or `base_url`. Those are ignored, with a warning; everything else in it +— profile selection, `org_id`, deployment aliases — applies as usual. A checkout +you did not write is not trusted to name the host your key is sent to. Name the +file explicitly (`--config` or `$UNSTRACT_CONFIG`) and it is honoured in full. + +What that protects is the key and the host, not the routing: `org_id`, +`api_name` and profile selection stay repo-controllable by design, so a +project file can still decide *which* deployment a command runs against on a +host you trust. Read one before you run inside a checkout you did not write. + +`clone` is the exception, and it is an operator command: a human moving one +organisation's resources into another, holding two admin Platform keys. It is +not part of the document-processing path the rest of this CLI wraps, so an agent +serving a user request should not reach for it unasked. It talks to two +deployments at once, which no single profile describes, so it takes both +endpoints as flags and both keys from `UNSTRACT_SRC_PLATFORM_KEY` / +`UNSTRACT_TGT_PLATFORM_KEY`. It exits 0 when nothing failed, which is not the +same as everything having moved: oversize and unsupported documents are skipped +by design, and `data.skipped` counts them. + +## Development + +```bash +uv venv && uv pip install -e '.[dev]' +uv run pytest # offline; no network, no credentials +uv run ruff check . +``` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ef13f4d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,54 @@ +[project] +name = "unstract-cli" +version = "0.1.0" +description = "Unified, LLM-friendly CLI for the Unstract suite of products" +readme = "README.md" +requires-python = ">=3.12" + +dependencies = [ + # Pinned to a major: `--discover` reads the shape of + # `click.Parameter.to_info_dict()`. + "click>=8.1,<9", + # Writing the config file only; reading it uses the stdlib `tomllib`. + "tomli-w>=1.0", + # Pinned to a commit: the CLI derives its flags and help text from these + # clients, so one that moves changes the CLI's surface. Both pins move to + # released versions before this ships. + "unstract-client @ git+https://github.com/Zipstack/unstract-python-client@a77ef6a", + "llmwhisperer-client @ git+https://github.com/Zipstack/llm-whisperer-python-client@7f64caf", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "ruff>=0.6", +] + +[project.scripts] +unstract = "unstract_cli.__main__:main" +# `unstract-client` installs an `unstract` script too, so whichever package is +# installed last wins that name; this one always reaches this CLI. +unstract-cli = "unstract_cli.__main__:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +# Needed for the git-pinned clients above. +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.build.targets.wheel] +packages = ["src/unstract_cli"] + +[tool.ruff] +line-length = 90 +target-version = "py312" +src = ["src", "tests"] + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "N", "UP", "B", "C4", "SIM"] +ignore = ["E501"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/src/unstract_cli/__init__.py b/src/unstract_cli/__init__.py new file mode 100644 index 0000000..c85c094 --- /dev/null +++ b/src/unstract_cli/__init__.py @@ -0,0 +1,3 @@ +"""Unstract CLI.""" + +__version__ = "0.1.0" diff --git a/src/unstract_cli/__main__.py b/src/unstract_cli/__main__.py new file mode 100644 index 0000000..6150bf9 --- /dev/null +++ b/src/unstract_cli/__main__.py @@ -0,0 +1,87 @@ +"""Entry point: turns every failure into an envelope plus a stable exit code. + +Click's own error handling is bypassed on purpose. By default it prints prose to +stderr and exits 1 or 2 with nothing on stdout, which leaves a caller parsing +stdout with an empty stream and no way to tell a usage error from a server +failure. +""" + +from __future__ import annotations + +import sys + +import click + +from unstract_cli.app import cli +from unstract_cli.config import ConfigError +from unstract_cli.core.errors import CLIError, ExitCode +from unstract_cli.core.output import AgentMode, OutputFormat, emit_error, resolve_format + + +def _option_from_argv(argv: list[str], *spellings: str) -> str | None: + """Best-effort read of one option before Click has parsed anything. + + A failure during parsing still has to be rendered, and the parsed context + does not exist yet at that point. + """ + for i, arg in enumerate(argv): + for spelling in spellings: + if arg.startswith(f"{spelling}="): + return arg.split("=", 1)[1] + if arg == spelling and i + 1 < len(argv): + return argv[i + 1] + return None + + +def _format_from_argv(argv: list[str]) -> OutputFormat: + """Resolve the format the same way the parsed run would.""" + try: + return resolve_format( + _option_from_argv(argv, "--output", "-o"), + _option_from_argv(argv, "--agent") or AgentMode.AUTO, + ) + except ValueError: + # An unusable value here is Click's error to report, not ours to guess + # around; render the failure in the default and let it through. + return resolve_format(None) + + +def main(argv: list[str] | None = None) -> int: + args = list(sys.argv[1:] if argv is None else argv) + fmt = _format_from_argv(args) + try: + cli.main(args=args, standalone_mode=False) + except CLIError as exc: + return int(emit_error(exc, fmt)) + except ConfigError as exc: + return int(emit_error(CLIError(str(exc), ExitCode.USAGE), fmt)) + except click.UsageError as exc: + return int( + emit_error( + CLIError(exc.format_message(), ExitCode.USAGE, hint="Run with --help."), + fmt, + ) + ) + except OSError as exc: + # Not a crash worth a traceback: a full disk or an unwritable path is + # the caller's to fix, and they still need a parseable envelope. + return int( + emit_error( + CLIError(str(exc), ExitCode.GENERIC, hint="Check the path and disk."), + fmt, + ) + ) + except (click.Abort, KeyboardInterrupt): + # Nothing here prompts, so Click's Abort can only mean an interrupt. + return int( + emit_error( + CLIError("Interrupted.", ExitCode.INTERRUPTED, retryable=True), fmt + ) + ) + except click.exceptions.Exit as exc: # --help and --version exit through here + return int(exc.exit_code) + return int(ExitCode.SUCCESS) + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/src/unstract_cli/app.py b/src/unstract_cli/app.py new file mode 100644 index 0000000..477e510 --- /dev/null +++ b/src/unstract_cli/app.py @@ -0,0 +1,261 @@ +"""The root Click application: global options and the command groups. + +Global options are declared once here and reach every command through the Click +context, so no command re-implements profile selection or output formatting. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + +import click + +from unstract_cli.commands.config_cmd import config_group +from unstract_cli.config import ( + DOCSTUDIO, + LLMWHISPERER, + ConfigError, + ResolvedConfig, + load_config, + set_config_path, +) +from unstract_cli.core.discover import TIERS, discover +from unstract_cli.core.errors import CLIError, ExitCode +from unstract_cli.core.output import ( + AgentMode, + OutputFormat, + diagnostic, + emit_result, + resolve_format, +) + + +@dataclass +class Context: + """Everything a command needs from the global options.""" + + output: OutputFormat = OutputFormat.TABLE + quiet: bool = False + verbosity: int = 0 + profile: str | None = None + #: Socket timeout for the deployment client, which has none of its own. + transport_timeout: float | None = None + #: Command-line overrides, keyed `product.setting` -- the top tier of + #: flag > env > profile > default. + overrides: dict[str, Any] = field(default_factory=dict) + _config: ResolvedConfig | None = field(default=None, repr=False) + + @property + def config(self) -> ResolvedConfig: + """Load the config lazily, so commands that need none never read a file.""" + if self._config is None: + try: + cfg = load_config() + except ConfigError as exc: + raise CLIError(str(exc), ExitCode.USAGE) from exc + for warning in cfg.warnings: + diagnostic(warning, quiet=self.quiet, verbosity=self.verbosity) + self._config = ResolvedConfig( + file=cfg, profile_name=self.profile, overrides=self.overrides + ) + return self._config + + def override(self, product: str, values: dict[str, Any]) -> None: + """Record the connection flags given for one product. + + Called from the product group, before any command runs, so the flag tier + is populated by the time a command resolves anything. + """ + for key, value in values.items(): + if value is None: + continue + if key == "api_key": + diagnostic( + "warning: a key passed on the command line lands in shell " + "history and in the process list. Prefer the environment " + "variable or `env:` indirection in a profile.", + quiet=self.quiet, + verbosity=self.verbosity, + ) + self.overrides[f"{product}.{key}"] = value + + def secrets(self) -> list[str]: + """Resolved credentials, for scrubbing anything on its way to a stream.""" + out: list[str] = [] + for product in (LLMWHISPERER, DOCSTUDIO): + try: + if value := self.config.get(product, "api_key"): + out.append(str(value)) + except ConfigError: + continue + return out + + +pass_context = click.make_pass_decorator(Context, ensure=True) + + +# `invoke_without_command` so `--discover` is answerable on its own: it is +# how a caller learns which commands exist, so it cannot require one. +@click.group( + invoke_without_command=True, + context_settings={"help_option_names": ["-h", "--help"]}, +) +@click.option( + "--config", + "config_file", + default=None, + type=click.Path(dir_okay=False), + help="Config file to use, overriding discovery.", +) +@click.option("--profile", "-p", default=None, help="Configuration profile to use.") +@click.option( + "--output", + "-o", + default=None, + type=click.Choice([f.value for f in OutputFormat]), + help="Output format. Defaults to table; pass json to parse the output.", +) +@click.option( + "--agent", + type=click.Choice([m.value for m in AgentMode]), + default=AgentMode.AUTO.value, + help="Whether a coding agent is driving this: sets the default format to " + "json. Only the default -- an explicit --output always wins.", +) +@click.option( + "--quiet", + "-q", + is_flag=True, + default=False, + help="Suppress diagnostics on stderr. stdout is unaffected.", +) +@click.option("--verbose", "-v", count=True, help="Increase diagnostic detail.") +@click.option( + "--discover", + "discover_tier", + type=click.Choice(TIERS), + default=None, + help="Describe this CLI as JSON instead of running a command, useful for agents.", +) +@click.version_option(package_name="unstract-cli") +@click.pass_context +def cli( + ctx: click.Context, + config_file: str | None, + profile: str | None, + output: str | None, + agent: str, + quiet: bool, + verbose: int, + discover_tier: str | None, +) -> None: + """The official CLI for Unstract. + + LLMWhisperer extracts text and layout from documents; Document Studio runs + them through API deployments that return structured JSON. + + Scripting or driving this from an agent: `-o json` prints one + `{ok, data, error, meta}` envelope on stdout and nothing else, failures + exit non-zero with a stable code, and `--discover groups|summary|full` + describes the commands, their flags and the output contract as JSON without + running anything. + """ + set_config_path(config_file) + ctx.obj = Context( + output=resolve_format(output, agent), + quiet=quiet, + verbosity=verbose, + profile=profile, + ) + if discover_tier: + # Discovery is how a caller learns what to run, so it has to answer + # before any configuration exists. + emit_result(discover(cli, discover_tier), ctx.obj.output) + ctx.exit(int(ExitCode.SUCCESS)) + if ctx.invoked_subcommand is None: + click.echo(ctx.get_help()) + ctx.exit(int(ExitCode.SUCCESS)) + + +def _connection_options(*, org_id: bool = False) -> Callable[[Any], Any]: + """The per-product connection settings, as flags. + + They sit on the product group rather than on each command: they say where to + connect, which is the same question for every command underneath. + """ + options = [ + click.option("--base-url", default=None, help="Service URL to use."), + click.option("--api-key", default=None, help="API key to use."), + ] + if org_id: + options.append( + click.option("--org-id", default=None, help="Organisation to run against.") + ) + + def decorate(func: Any) -> Any: + for option in reversed(options): + func = option(func) + return func + + return decorate + + +@cli.group("whisper") +@_connection_options() +@pass_context +def whisper_group(ctx: Context, **overrides: str | None) -> None: + """Extract text and layout from documents with LLMWhisperer.""" + ctx.override(LLMWHISPERER, overrides) + + +@cli.group("docstudio") +@_connection_options(org_id=True) +@click.option( + "--transport-timeout", + type=float, + default=None, + help="Seconds before a stalled connection is given up on. Unset means it " + "is not, which is what the client has always done.", +) +@pass_context +def docstudio_group( + ctx: Context, transport_timeout: float | None, **overrides: str | None +) -> None: + """Run Document Studio API deployments.""" + ctx.transport_timeout = transport_timeout + ctx.override(DOCSTUDIO, overrides) + + +@docstudio_group.group("deployment") +def deployment_group() -> None: + """Work with a deployed API.""" + + +cli.add_command(config_group) + +# Imported for their side effect of registering commands, and imported last +# because those modules hang their commands off the groups declared just above. +from unstract_cli.commands import clone_cmd, docstudio_cmd, whisper_cmd # noqa: E402,F401 + + +def command_tree() -> dict[str, Any]: + """The registered command tree, read back from Click itself. + + Describing commands anywhere but from the parser lets the description drift + from what the parser accepts, so discovery and help always read this. + """ + + def walk(command: click.Command) -> dict[str, Any]: + entry: dict[str, Any] = {"help": (command.help or "").strip().split("\n")[0]} + if isinstance(command, click.Group): + entry["commands"] = { + name: walk(sub) for name, sub in sorted(command.commands.items()) + } + return entry + + return walk(cli)["commands"] + + +__all__ = ["Context", "cli", "command_tree", "pass_context"] diff --git a/src/unstract_cli/commands/__init__.py b/src/unstract_cli/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/unstract_cli/commands/clone_cmd.py b/src/unstract_cli/commands/clone_cmd.py new file mode 100644 index 0000000..a2cfcbc --- /dev/null +++ b/src/unstract_cli/commands/clone_cmd.py @@ -0,0 +1,210 @@ +"""`unstract clone` -- copying one organization's resources into another. + +Two endpoints, each with its own key, so this command takes them as flags rather +than from a profile: a profile describes one connection. +""" + +from __future__ import annotations + +import logging +from typing import Any + +import click + +# The size grammar and the list syntax come from the client rather than a copy +# here, so both spellings of this command accept the same strings. +from unstract.clone.cli import _parse_size, _split_csv +from unstract.clone.context import ( + DEFAULT_CONCURRENCY, + CloneOptions, + OrgEndpoint, +) +from unstract.clone.exceptions import CloneError +from unstract.clone.orchestrator import clone as run_clone +from unstract.clone.report import CloneReport + +from unstract_cli.app import Context, cli, pass_context +from unstract_cli.commands.common import finish +from unstract_cli.core.errors import CLIError, ExitCode, remember_secret +from unstract_cli.core.output import OutputFormat, emit_text + + +@cli.command("clone") +@click.option("--source-url", required=True, help="Base URL of the source deployment.") +@click.option( + "--source-org", required=True, help="Source organization_id (slug in the URL path)." +) +@click.option( + "--source-key", + envvar="UNSTRACT_SRC_PLATFORM_KEY", + required=True, + help="Source admin's Platform API key (or env UNSTRACT_SRC_PLATFORM_KEY).", +) +@click.option("--target-url", required=True, help="Base URL of the target deployment.") +@click.option( + "--target-org", required=True, help="Target organization_id (slug in the URL path)." +) +@click.option( + "--target-key", + envvar="UNSTRACT_TGT_PLATFORM_KEY", + required=True, + help="Target admin's Platform API key (or env UNSTRACT_TGT_PLATFORM_KEY).", +) +@click.option( + "--dry-run", is_flag=True, help="Plan only -- do not write anything to the target." +) +@click.option( + "--include", default=None, help="Comma-separated phases to run (default: all)." +) +@click.option("--exclude", default=None, help="Comma-separated phases to skip.") +@click.option( + "--on-name-conflict", + type=click.Choice(["adopt", "abort"]), + default="adopt", + show_default=True, + help="What to do when a like-named entity exists on the target.", +) +@click.option( + "--api-prefix", + default="api/v1", + show_default=True, + help="Backend URL prefix, matching the deployment's own.", +) +@click.option( + "--file-strategy", + type=click.Choice(["platform_api", "skip"]), + default="platform_api", + show_default=True, + help="How to move Prompt Studio documents. 'skip' copies metadata only.", +) +@click.option("--skip-files", is_flag=True, help="Alias for --file-strategy=skip.") +@click.option( + "--max-file-size", + default="25MB", + show_default=True, + help="Per-file cap for the files phase. Oversize files are reported, not fatal.", +) +@click.option( + "--concurrency", + type=click.IntRange(min=1, max=32), + default=DEFAULT_CONCURRENCY, + show_default=True, + help="Per-phase worker count. 1 is strictly sequential.", +) +@click.option( + "--clone-group-members", + is_flag=True, + help="Also add group members on the target, matched by email.", +) +@pass_context +def clone( + ctx: Context, + source_url: str, + source_org: str, + source_key: str, + target_url: str, + target_org: str, + target_key: str, + **params: Any, +) -> None: + """Copy an organization's resources into another organization. + + Adapters, connectors, workflows, pipelines, API deployments, Prompt Studio + projects and their files, user groups and sharing state. Run --dry-run first: + it reports what would be written without writing it. + """ + for key in (source_key, target_key): + remember_secret(key) + _configure_logging(ctx) + + options = CloneOptions( + dry_run=params["dry_run"], + include=_split_csv(params["include"]), + exclude=_split_csv(params["exclude"]) or (), + on_name_conflict=params["on_name_conflict"], + verbose=ctx.verbosity > 0, + file_strategy="skip" if params["skip_files"] else params["file_strategy"], + max_file_size=_parse_size(params["max_file_size"]), + concurrency=params["concurrency"], + clone_group_members=params["clone_group_members"], + ) + + def endpoint(url: str, org: str, key: str) -> OrgEndpoint: + return OrgEndpoint( + base_url=url, + organization_id=org, + platform_key=key, + api_path_prefix=params["api_prefix"], + ) + + try: + report = run_clone( + endpoint(source_url, source_org, source_key), + endpoint(target_url, target_org, target_key), + options, + ) + except CloneError as exc: + raise CLIError( + str(exc), + ExitCode.USAGE, + hint="The clone could not start. Check the URLs, orgs and keys.", + ) from exc + + _finish(ctx, report) + + +def _configure_logging(ctx: Context) -> None: + """Send the orchestrator's progress to stderr, at the run's own verbosity.""" + logging.basicConfig( + level=logging.WARNING + if ctx.quiet + else (logging.DEBUG if ctx.verbosity else logging.INFO), + format="%(asctime)s %(levelname)-7s %(name)s: %(message)s", + datefmt="%H:%M:%S", + ) + + +def _skipped(report: CloneReport) -> dict[str, Any]: + """What the run did not copy, summarised at the top of the payload. + + Skipping an oversize or unsupported file is reported rather than fatal, so + the run still exits 0; a consumer reading only the exit code would otherwise + have to walk the whole report to discover documents that never arrived. + """ + by_phase = {phase.name: phase.skipped for phase in report.phases if phase.skipped} + return { + "total": sum(by_phase.values()), + "by_phase": by_phase, + "oversize_files": len(report.oversize_files), + "unsupported_files": len(report.unsupported_files), + } + + +def _finish(ctx: Context, report: CloneReport) -> None: + """Emit the report, then fail if the clone did not fully succeed.""" + failure = None + if report.aborted: + failure = f"Clone aborted: {report.abort_reason}" + elif failed := [phase.name for phase in report.phases if phase.failed]: + failure = f"Clone completed with failures in: {', '.join(sorted(failed))}" + + payload = {**report.as_dict(), "skipped": _skipped(report)} + # A person running this reads the report itself; every other format gets the + # single envelope, which carries the same content as data. + rendered = ctx.output is OutputFormat.TABLE + if rendered: + emit_text(report.render(), secrets=ctx.secrets()) + elif not failure: + finish(ctx, payload) + + if failure: + raise CLIError( + failure, + ExitCode.GENERIC, + details=None if rendered else payload, + hint="The report lists what was copied and what was not. Re-running " + "adopts what already exists on the target rather than duplicating it.", + ) + + +__all__ = ["clone"] diff --git a/src/unstract_cli/commands/common.py b/src/unstract_cli/commands/common.py new file mode 100644 index 0000000..600bc7c --- /dev/null +++ b/src/unstract_cli/commands/common.py @@ -0,0 +1,101 @@ +"""Pieces every product command shares: the wait flags and result emission.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import click + +from unstract_cli.app import Context +from unstract_cli.core.output import emit_result + +#: Poll interval and the ceiling on the whole wait. Both are flags. +DEFAULT_INTERVAL = 3.0 +DEFAULT_TIMEOUT = 300.0 + +F = Callable[..., Any] + + +def wait_options(*, default: bool = True) -> Callable[[F], F]: + """`--wait` and its two knobs. + + ``--wait`` is a gate, not a duration: how long to wait is ``--timeout`` and + how often to check is ``--interval``, so neither has two spellings. + """ + + def decorate(func: F) -> F: + for option in reversed( + [ + click.option( + "--wait/--no-wait", + default=default, + help="Poll until the job reaches a terminal state.", + ), + click.option( + "--interval", + type=float, + default=DEFAULT_INTERVAL, + show_default=True, + help="Seconds between polls.", + ), + click.option( + "--timeout", + "wait_timeout", + type=float, + default=DEFAULT_TIMEOUT, + show_default=True, + help="Seconds to wait before giving up. The job keeps running.", + ), + click.option( + "--save", + type=click.Path(dir_okay=False), + default=None, + help="Write the result here before printing it.", + ), + ] + ): + func = option(func) + return func + + return decorate + + +def raw_field(field: str) -> Callable[[click.Command], click.Command]: + """Declare which field `--output raw` prints for this command. + + Recorded on the command so `--discover full` can report it: a caller asking + for raw output has to know what it is going to get. + """ + + def decorate(command: click.Command) -> click.Command: + command.raw_field = field + return command + + return decorate + + +def finish( + ctx: Context, + data: Any, + *, + raw_field: str | None = None, + meta: dict[str, Any] | None = None, +) -> None: + """Emit one result envelope, scrubbing any resolved credential from it.""" + emit_result( + data, + ctx.output, + meta=meta, + raw_field=raw_field, + secrets=ctx.secrets(), + ) + + +__all__ = [ + "DEFAULT_INTERVAL", + "DEFAULT_TIMEOUT", + "finish", + "raw_field", + "wait_options", +] diff --git a/src/unstract_cli/commands/config_cmd.py b/src/unstract_cli/commands/config_cmd.py new file mode 100644 index 0000000..aa02390 --- /dev/null +++ b/src/unstract_cli/commands/config_cmd.py @@ -0,0 +1,374 @@ +"""The `config` command group -- local only, no network calls. + +These commands map to no API operation: they operate purely on the local config +layer, and they are how a user or an agent bootstraps every other command. + +Nothing here prompts: `init` refuses to clobber an existing file unless +`--force` is passed, rather than asking, so the CLI behaves the same whether or +not a human is watching. +""" + +from __future__ import annotations + +from typing import Any + +import click + +from unstract_cli.config import ( + DOCSTUDIO, + KEY_SOURCES, + LLMWHISPERER, + PRODUCTS, + ConfigError, + ConfigFile, + ResolvedConfig, + config_path, + load_config, + save_config, + settings_for, + starter_profiles, +) +from unstract_cli.core.clients import llmwhisperer, translated +from unstract_cli.core.errors import CLIError, ExitCode +from unstract_cli.core.output import ( + OutputFormat, + diagnostic, + emit_result, + resolve_format, +) + +#: Keys whose value is never echoed back, even on explicit request: this output +#: is as likely to land in a log or a transcript as on a screen. +_SECRET_KEY_HINTS = ("key", "token", "secret") + + +def _is_secret(key: str) -> bool: + return any(hint in key.lower() for hint in _SECRET_KEY_HINTS) + + +def _fmt(obj: Any) -> OutputFormat: + """Output format from the root context, defaulting when invoked standalone.""" + return getattr(obj, "output", None) or resolve_format(None) + + +def _check_product(product: str) -> str: + if product not in PRODUCTS: + raise CLIError( + f"Unknown config target {product!r}.", + ExitCode.USAGE, + hint="Valid targets: " + ", ".join(PRODUCTS) + ".", + ) + return product + + +@click.group(name="config", help="Manage CLI configuration profiles (local only).") +def config_group() -> None: + """Local configuration management. These commands make no network calls.""" + + +@config_group.command("init", help="Create a starter config file with profile stubs.") +@click.option( + "--force", is_flag=True, default=False, help="Overwrite an existing config file." +) +@click.pass_obj +def config_init(obj: Any, force: bool) -> None: + path = config_path() + if path.exists() and not force: + # Never prompt: state the situation and the exact flag that resolves it. + raise CLIError( + f"Config already exists at {path}.", + ExitCode.USAGE, + hint="Pass --force to overwrite it, or edit the file directly.", + ) + + replaced = path.exists() + new = ConfigFile( + default_profile="cloud-us", profiles=starter_profiles(), path=path, exists=True + ) + written = save_config(new, path) + emit_result( + { + "created": str(written), + "default_profile": "cloud-us", + "profiles": sorted(new.profiles), + "replaced_existing": replaced, + "note": ( + "Credentials use env: indirection, so this file holds no secrets. " + "Set the referenced environment variables to authenticate. " + KEY_SOURCES + ), + }, + _fmt(obj), + ) + + +@config_group.command("list", help="List profiles defined in the config file.") +@click.pass_obj +def config_list(obj: Any) -> None: + cfg = _loaded(obj) + emit_result( + { + "path": str(cfg.path), + "exists": cfg.exists, + "default_profile": cfg.default_profile, + "profiles": { + name: { + block: sorted(settings) if isinstance(settings, dict) else settings + for block, settings in blocks.items() + } + for name, blocks in cfg.profiles.items() + }, + }, + _fmt(obj), + ) + + +@config_group.command("get") +@click.argument("product") +@click.argument("key") +@click.pass_obj +def config_get(obj: Any, product: str, key: str) -> None: + """Show a resolved setting, following flag > env > profile > default. + + PRODUCT and KEY are positional -- not flags. Credentials are reported as + configured or not, never echoed. + + \b + Examples: + unstract config get docstudio org_id + unstract --profile cloud-eu config get llmwhisperer base_url + """ + _check_product(product) + try: + value = _resolved(obj).get(product, key) + except ConfigError as exc: + raise CLIError(str(exc), ExitCode.USAGE) from exc + + emit_result( + { + "product": product, + "key": key, + "value": ("***SET***" if value else None) if _is_secret(key) else value, + "configured": value is not None, + }, + _fmt(obj), + ) + + +@config_group.command("set") +@click.argument("product") +@click.argument("key") +@click.argument("value") +@click.option("--profile", "-p", "profile", default=None, help="Profile to write to.") +@click.pass_obj +def config_set(obj: Any, product: str, key: str, value: str, profile: str | None) -> None: + """Set a value in the config file. + + PRODUCT, KEY and VALUE are positional -- not flags. Writes to the active + profile unless --profile names another. + + \b + Examples: + unstract config set docstudio org_id org_ABC123 + unstract config set llmwhisperer api_key 'env:LLMWHISPERER_API_KEY' + + \b + Prefer `env:VAR_NAME` for credentials: the file then records where the secret + lives rather than the secret itself, and a literal value also lands in your + shell history. + """ + _check_product(product) + cfg = _loaded(obj) + name = profile or getattr(obj, "profile", None) or cfg.default_profile or "cloud-us" + + cfg.profiles.setdefault(name, {}).setdefault(product, {})[key] = value + if not cfg.default_profile: + cfg.default_profile = name + written = save_config(cfg) + + warning = None + if _is_secret(key) and not value.startswith("env:"): + warning = ( + "Value stored literally. Prefer `env:VAR_NAME` so the config file holds " + "a reference rather than the secret itself." + ) + + emit_result( + { + "profile": name, + "product": product, + "key": key, + "path": str(written), + "warning": warning, + }, + _fmt(obj), + ) + + +def _probe(resolved: ResolvedConfig) -> dict[str, Any]: + """Check each product's credentials against the service, where that is possible. + + LLMWhisperer has a read-only usage endpoint, so its key can be verified for + real. A deployment has no side-effect-free endpoint -- the only thing to call + is an execution -- so its entry reports that the settings resolve and says + plainly that nothing was verified. Claiming otherwise would be worse than + not checking. + """ + out: dict[str, Any] = {} + try: + with translated(endpoint="get-usage-info"): + llmwhisperer(resolved).get_usage_info() + except CLIError as exc: + out[LLMWHISPERER] = { + "checked": True, + "ok": False, + "detail": exc.message, + "exit_code": int(exc.exit_code), + } + except ConfigError as exc: + out[LLMWHISPERER] = {"checked": False, "ok": False, "detail": str(exc)} + else: + out[LLMWHISPERER] = { + "checked": True, + "ok": True, + "detail": "The key was accepted by the usage endpoint.", + } + + resolves = all( + resolved.get(DOCSTUDIO, key) for key in ("org_id", "api_key", "base_url") + ) + out[DOCSTUDIO] = { + "checked": False, + # Null, not True: nothing was called, so there is no verdict to report. + # A `true` beside `checked: false` reads as a live check that passed. + "ok": None, + "resolved": resolves, + "detail": ( + "Credentials resolve (org and key present) but were NOT verified -- " + "the deployment API has no side-effect-free endpoint to call, so a " + "wrong key is only discovered by running a deployment." + if resolves + else "Organisation or key is missing; nothing was called." + ), + } + return out + + +@config_group.command("doctor", help="Diagnose how each setting resolves.") +@click.option( + "--probe/--no-probe", + default=False, + help="Also check the resolved credentials against the service.", +) +@click.pass_obj +def config_doctor(obj: Any, probe: bool) -> None: + """Report where each setting resolves from, without echoing any secret. + + Answers the question that costs the most time: the CLI reports a key as "not + configured", but you set it -- where is it looking? For `env:` references it + says whether the variable is present in THIS process, a shell `export` in a + login profile the CLI never inherited being the classic trap. + + Resolution is answered offline. --probe adds the second question -- does the + resolved key work -- which needs the network, so it is opt-in. + + Exits 0 only when nothing it checked failed. A setting that is simply not + configured is a report, not a failure; a setting that points somewhere and + does not arrive -- an unset `env:` variable, an unknown profile, a probe the + service rejected -- exits non-zero, because a setup script branches on that. + """ + resolved = _resolved(obj) + problems: list[str] = [] + products: dict[str, Any] = {} + for product in PRODUCTS: + entry: dict[str, Any] = {} + for key in settings_for(product): + try: + entry[key] = resolved.resolution_source(product, key) + except ConfigError as exc: + entry[key] = {"resolved": False, "source": "unset", "detail": str(exc)} + if detail := entry[key].get("detail"): + problems.append(f"{product}.{key}: {detail}") + products[product] = entry + + try: + aliases = list(resolved.deployment_aliases()) + except ConfigError as exc: + aliases = [] + problems.append(str(exc)) + for alias in aliases: + # An alias carries a key of its own, so it is a second place a project + # file can name one -- and it falls back to the profile's key silently. + if detail := resolved.withheld_detail("deployments", alias, "api_key"): + problems.append(f"deployment alias {alias}: {detail}") + try: + # Resolved the way a run resolves it: that an alias is *listed* says + # nothing about whether the settings behind it arrive. + resolved.deployment(alias) + except ConfigError as exc: + problems.append(f"deployment alias {alias}: {exc}") + + report: dict[str, Any] = { + "active_profile": resolved.active_profile, + "config_path": str(resolved.file.path), + "config_exists": resolved.file.exists, + "products": products, + "deployment_aliases": aliases, + } + if any( + not entry["api_key"]["resolved"] + for entry in products.values() + if "api_key" in entry + ): + # The next question after "no key" is always where one comes from. The + # field name avoids the word the payload scrubber redacts on. + report["getting_started"] = KEY_SOURCES + if probe: + report["probe"] = _probe(resolved) + problems += [ + f"probe {name}: {result.get('detail')}" + for name, result in report["probe"].items() + if result["ok"] is False + ] + + if problems: + report["problems"] = problems + more = "" if len(problems) == 1 else f" (+{len(problems) - 1} more)" + raise CLIError( + f"{len(problems)} configuration check(s) failed: {problems[0]}{more}", + ExitCode.GENERIC, + details=report, + hint=( + "`details` carries the whole report, including where each setting " + "resolved from." + ), + ) + emit_result(report, _fmt(obj)) + + +def _loaded(obj: Any) -> ConfigFile: + """The config file, with its warnings reported. + + These commands load the file themselves rather than through the root + context, and they are the two a user runs *to understand* their config -- + reading it here without repeating what it warned about would make them the + quietest commands in the CLI about their own subject. + """ + cfg = load_config() + for warning in cfg.warnings: + diagnostic( + warning, + quiet=getattr(obj, "quiet", False), + verbosity=getattr(obj, "verbosity", 0), + ) + return cfg + + +def _resolved(obj: Any) -> ResolvedConfig: + """The root context's config, or a freshly loaded one when invoked standalone.""" + # Already loaded means the context already reported its warnings. + if (existing := getattr(obj, "_config", None)) is not None: + return existing + return ResolvedConfig(file=_loaded(obj), profile_name=getattr(obj, "profile", None)) + + +__all__ = ["config_group"] diff --git a/src/unstract_cli/commands/docstudio_cmd.py b/src/unstract_cli/commands/docstudio_cmd.py new file mode 100644 index 0000000..099fc89 --- /dev/null +++ b/src/unstract_cli/commands/docstudio_cmd.py @@ -0,0 +1,167 @@ +"""`unstract docstudio deployment ...` -- running a deployed API. + +The deployment client reports failure by returning a status code rather than +raising, and it has no polling loop of its own, so both are handled here. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any +from urllib.parse import parse_qs, urlparse + +import click +from unstract.api_deployments.client import APIDeploymentsClient + +from unstract_cli.app import Context, deployment_group, pass_context +from unstract_cli.commands.common import finish, raw_field, wait_options +from unstract_cli.core.clients import ( + deployment, + raise_for_result, + translated, + translating, +) +from unstract_cli.core.errors import CLIError, ExitCode +from unstract_cli.core.params import requested, spec_options +from unstract_cli.core.poll import PollSpec, classify, preflight, wait_for_completion + +PRODUCT = "docstudio" + +#: The run POST and the status GET spell the state under different names, and +#: the API answers HTTP 422 while still executing -- only the body decides. +RUN_POLL = PollSpec( + handle_field="status_check_api_endpoint", + terminal_success=("COMPLETED", "SUCCESS"), + terminal_failure=("ERROR", "ERROR_EXCEPTION", "FAILED", "STOPPED"), + status_field=("execution_status", "status"), +) + +#: `--output raw` prints one field rather than the whole payload. +RAW_FIELD = "extraction_result" + +#: Parameters the run POST and the status GET share: what was asked for in the +#: run has to be asked for again when the result is read. +_SHARED_WITH_STATUS = ("include_metadata", "include_metrics", "include_extracted_text") + + +@raw_field(RAW_FIELD) +@deployment_group.command("run") +@click.argument("target") +@click.argument("files", nargs=-1, required=True, type=click.Path(exists=True)) +@wait_options() +@spec_options( + PRODUCT, + "execute", + client_method=APIDeploymentsClient.structure_file, + # `files` is the FILES argument; `timeout` selects the server's own + # execution mode and would fight the CLI's polling for the same job. + exclude=("files", "timeout"), +) +@pass_context +def run( + ctx: Context, + target: str, + files: tuple[str, ...], + wait: bool, + interval: float, + wait_timeout: float, + save: str | None, + **params: Any, +) -> None: + """Run a deployment against one or more documents. + + TARGET is a deployment alias or an API name. With --wait (the default) this + polls until the execution finishes and returns its result. + """ + client = deployment(ctx.config, target, ctx.transport_timeout) + sent = requested(params) + if save: + preflight(save) + with translated(endpoint=client.api_url): + # Queued execution, so the request returns a handle instead of holding + # the connection open for the length of the job. + started = client.structure_file(list(files), timeout=0, **sent) + raise_for_result(started, endpoint=client.api_url) + + if not wait: + finish(ctx, started, raw_field=RAW_FIELD) + return + + result = wait_for_completion( + initial=started, + spec=RUN_POLL, + poll=_status_poller( + client, {k: v for k, v in sent.items() if k in _SHARED_WITH_STATUS} + ), + save=save, + interval=interval, + timeout=wait_timeout, + on_status=lambda status: ( + click.echo(f"status: {status}", err=True) if not ctx.quiet else None + ), + ) + # A waited result names no execution, so the handle is returned as meta for + # correlation. + finish(ctx, result, raw_field=RAW_FIELD, meta=_handle_meta(started)) + + +def _handle_meta(started: dict[str, Any]) -> dict[str, Any]: + """The execution's identity, from wherever the run response carries it.""" + if execution_id := started.get("execution_id"): + return {"execution_id": execution_id} + endpoint = str(started.get("status_check_api_endpoint") or "") + found = parse_qs(urlparse(endpoint).query).get("execution_id") + return {"execution_id": found[0]} if found else {} + + +def _status_poller( + client: APIDeploymentsClient, params: dict[str, Any] +) -> Callable[[str], dict[str, Any]]: + """Poll one execution, failing on a status code the poll loop cannot use.""" + + def poll(endpoint: str) -> dict[str, Any]: + result = client.check_execution_status(endpoint, **params) + # A retryable status is left to the client's own retry policy, which has + # already run; the client reports those as still pending. + if not result.get("pending"): + raise_for_result(result, endpoint=client.api_url) + return result + + return translating(poll, client.api_url) + + +@raw_field(RAW_FIELD) +@deployment_group.command("status") +@click.argument("target") +@click.argument("execution_id") +@spec_options( + PRODUCT, + "status", + client_method=APIDeploymentsClient.check_execution_status, + exclude=("execution_id",), +) +@pass_context +def status(ctx: Context, target: str, execution_id: str, **params: Any) -> None: + """Report the state of a running or finished execution.""" + client = deployment(ctx.config, target, ctx.transport_timeout) + endpoint = f"{client.api_url}?execution_id={execution_id}" + with translated(endpoint=client.api_url): + result = client.check_execution_status(endpoint, **requested(params)) + if not result.get("pending"): + raise_for_result(result, endpoint=client.api_url) + # A finished-and-failed execution is reported inside an HTTP 200, so the + # status code alone would call this a success. + if classify(result, RUN_POLL) == "failure": + raise CLIError( + f"Execution {execution_id} finished with status " + f"{result.get('execution_status')!r}.", + ExitCode.VALIDATION, + details=result, + endpoint=client.api_url, + hint="Inspect `details` for the per-file error, or check the execution logs.", + extra={"execution_id": execution_id}, + ) + finish(ctx, result, raw_field=RAW_FIELD) + + +__all__ = ["run", "status"] diff --git a/src/unstract_cli/commands/whisper_cmd.py b/src/unstract_cli/commands/whisper_cmd.py new file mode 100644 index 0000000..9f834a5 --- /dev/null +++ b/src/unstract_cli/commands/whisper_cmd.py @@ -0,0 +1,365 @@ +"""`unstract whisper ...` -- text and layout extraction. + +Every flag below the command name is derived from the committed spec, so this +module holds only what the spec cannot say: which parameter is an argument, +which the CLI owns, and how a result is polled for and retrieved. +""" + +from __future__ import annotations + +from typing import Any + +import click +from unstract.llmwhisperer.client_v2 import LLMWhispererClientV2 + +from unstract_cli.app import Context, pass_context, whisper_group +from unstract_cli.commands.common import finish, raw_field, wait_options +from unstract_cli.core.clients import llmwhisperer, translated, translating +from unstract_cli.core.errors import CLIError, ExitCode, remember_secret +from unstract_cli.core.params import requested, spec_options +from unstract_cli.core.poll import ( + PollSpec, + classify, + extract_status, + persist, + preflight, + wait_for_completion, +) + +PRODUCT = "llmwhisperer" + +#: Terminal states as the body reports them. `unknown` is one: the service +#: returns it for a hash it no longer knows, which no amount of polling changes. +EXTRACT_POLL = PollSpec( + handle_field="whisper_hash", + terminal_success=("processed",), + terminal_failure=("error", "unknown"), + status_field="status", +) + +#: `--output raw` prints one field rather than the whole payload. Extraction +#: results carry the text under this name. +RAW_FIELD = "result_text" + + +def _is_url(source: str) -> bool: + return source.startswith(("http://", "https://")) + + +@raw_field(RAW_FIELD) +@whisper_group.command("extract") +@click.argument("source") +@wait_options() +@spec_options( + PRODUCT, + "extract", + client_method=LLMWhispererClientV2.whisper, + # `url` is the SOURCE argument when it looks like one. + exclude=("url",), +) +@pass_context +def extract( + ctx: Context, + source: str, + wait: bool, + interval: float, + wait_timeout: float, + save: str | None, + **params: Any, +) -> None: + """Extract text from a document, given a file path or a URL. + + With --wait (the default) this returns the extracted text. With --no-wait it + returns the whisper_hash, and `whisper status` and `whisper retrieve` take + it from there. + """ + client = llmwhisperer(ctx.config) + sent = requested(params) + if save: + preflight(save) + + if sent.get("use_webhook") and wait: + raise CLIError( + "--wait and --use-webhook are mutually exclusive.", + ExitCode.USAGE, + hint=( + "A webhook delivers the result itself; pass --no-wait to submit " + "and return immediately." + ), + ) + + with translated(endpoint="whisper"): + # The CLI's own poll loop is used over the client's so that waiting + # behaves the same for every product. + accepted = client.whisper( + **({"url": source} if _is_url(source) else {"file_path": source}), + **sent, + wait_for_completion=False, + ) + + if not wait: + finish(ctx, accepted) + return + + result = wait_for_completion( + initial=accepted, + spec=EXTRACT_POLL, + poll=translating(client.whisper_status, "whisper-status"), + retrieve=translating( + lambda handle: _extraction(client.whisper_retrieve(handle)), + "whisper-retrieve", + ), + save=save, + interval=interval, + timeout=wait_timeout, + on_status=lambda status: ( + click.echo(f"status: {status}", err=True) if not ctx.quiet else None + ), + ) + # Waiting returns the text, which identifies the job nowhere; the hash is + # what a later status, retrieve or highlights call needs. + finish( + ctx, + result, + raw_field=RAW_FIELD, + meta={"whisper_hash": accepted.get("whisper_hash")} + if accepted.get("whisper_hash") + else None, + ) + + +def _extraction(payload: Any) -> Any: + """The extracted result out of a retrieve response. + + A retrieve is the acknowledging read, so an empty result here is a document + that was processed, billed and consumed for nothing -- reporting it as a + success would hide that. + """ + result = payload.get("extraction", payload) if isinstance(payload, dict) else payload + if not result: + raise CLIError( + "The service returned no extraction for a completed job.", + ExitCode.SERVER_ERROR, + details=payload, + hint=( + "The read has been acknowledged, so it cannot be repeated. " + "`details` carries the response exactly as it arrived." + ), + ) + return result + + +@whisper_group.command("status") +@click.argument("whisper_hash") +@pass_context +def status(ctx: Context, whisper_hash: str) -> None: + """Report the state of a submitted extraction.""" + client = llmwhisperer(ctx.config) + with translated(endpoint="whisper-status"): + result = client.whisper_status(whisper_hash) + # A failed extraction is reported inside an HTTP 200, so the status code + # alone would call this a success. + if classify(result, EXTRACT_POLL) == "failure": + raise CLIError( + f"Extraction finished with status {extract_status(result)!r}.", + ExitCode.VALIDATION, + details=result, + endpoint="whisper-status", + hint=( + "`details` carries the service's own message. An `unknown` status " + "means the service no longer holds this hash." + ), + extra={"whisper_hash": whisper_hash}, + ) + finish(ctx, result) + + +@raw_field(RAW_FIELD) +@whisper_group.command("retrieve") +@click.argument("whisper_hash") +@click.option( + "--save", + type=click.Path(dir_okay=False), + default=None, + help="Write the result here before printing it.", +) +@pass_context +def retrieve(ctx: Context, whisper_hash: str, save: str | None) -> None: + """Fetch a finished extraction. + + A result can be read exactly once, so --save writes it to disk before it is + printed: a broken pipe or a full terminal buffer after the read cannot be + recovered by asking again. + """ + client = llmwhisperer(ctx.config) + if save: + preflight(save) + with translated(endpoint="whisper-retrieve"): + payload = client.whisper_retrieve(whisper_hash) + result = _extraction(payload) + if save: + persist(save, result) + finish(ctx, result, raw_field=RAW_FIELD) + + +@whisper_group.command("detail") +@click.argument("whisper_hash") +@pass_context +def detail(ctx: Context, whisper_hash: str) -> None: + """Report processing detail for one extraction.""" + client = llmwhisperer(ctx.config) + with translated(endpoint="whisper-detail"): + finish(ctx, client.whisper_detail(whisper_hash)) + + +@whisper_group.command("highlights") +@click.argument("whisper_hash") +@spec_options( + PRODUCT, + "highlights", + client_method=LLMWhispererClientV2.get_highlight_data, + exclude=("whisper_hash",), +) +@click.option( + "--target-width", + type=int, + default=None, + help="Width of the page as displayed. With --target-height, adds a bounding box per line.", +) +@click.option( + "--target-height", + type=int, + default=None, + help="Height of the page as displayed.", +) +@pass_context +def highlights( + ctx: Context, + whisper_hash: str, + target_width: int | None, + target_height: int | None, + **params: Any, +) -> None: + """Fetch line metadata, optionally scaled to a page you are rendering. + + The scaling is arithmetic on the metadata, not a second request, so it is + folded in here rather than being a command of its own. + """ + client = llmwhisperer(ctx.config) + with translated(endpoint="highlights"): + data = client.get_highlight_data(whisper_hash, **requested(params)) + + if target_width and target_height: + data = { + "lines": data, + "rects": _bounding_boxes(client, data, target_width, target_height), + } + finish(ctx, data) + + +def _line_metadata(value: Any) -> list[int] | None: + """The `[page, base_y, height, page_height]` list the geometry needs. + + The service returns it as a named object carrying the list under `raw`, and + the client's geometry takes the bare list, so both shapes are read. + """ + if isinstance(value, dict): + value = value.get("raw") + if ( + isinstance(value, list) + and len(value) >= 4 + and all(isinstance(item, (int, float)) for item in value) + # The page height is a divisor in the scaling, and the service reports a + # line it has no geometry for as all zeros. + and value[3] + ): + return value + return None + + +def _bounding_boxes( + client: LLMWhispererClientV2, + data: Any, + target_width: int, + target_height: int, +) -> dict[str, list[int]]: + """(page, x1, y1, x2, y2) per line, for the lines that carry metadata.""" + if not isinstance(data, dict): + return {} + lines = {line: _line_metadata(value) for line, value in data.items()} + return { + str(line): list(client.get_highlight_rect(metadata, target_width, target_height)) + for line, metadata in lines.items() + if metadata is not None + } + + +@whisper_group.command("usage") +@pass_context +def usage(ctx: Context) -> None: + """Report this key's usage and remaining quota.""" + client = llmwhisperer(ctx.config) + with translated(endpoint="get-usage-info"): + finish(ctx, client.get_usage_info()) + + +@whisper_group.group("webhook") +def webhook_group() -> None: + """Manage the webhooks an extraction can deliver its result to.""" + + +@webhook_group.command("create") +@click.argument("name") +@click.option("--url", required=True, help="Where the result is delivered.") +@click.option("--auth-token", required=True, help="Token sent with the delivery.") +@pass_context +def webhook_create(ctx: Context, name: str, url: str, auth_token: str) -> None: + """Register a webhook.""" + remember_secret(auth_token) + client = llmwhisperer(ctx.config) + with translated(endpoint="whisper-manage-callback"): + finish(ctx, client.register_webhook(url, auth_token, name)) + + +@webhook_group.command("update") +@click.argument("name") +@click.option("--url", required=True, help="Where the result is delivered.") +@click.option("--auth-token", required=True, help="Token sent with the delivery.") +@pass_context +def webhook_update(ctx: Context, name: str, url: str, auth_token: str) -> None: + """Replace a webhook's URL and token.""" + remember_secret(auth_token) + client = llmwhisperer(ctx.config) + with translated(endpoint="whisper-manage-callback"): + finish(ctx, client.update_webhook_details(name, url, auth_token)) + + +@webhook_group.command("get") +@click.argument("name") +@pass_context +def webhook_get(ctx: Context, name: str) -> None: + """Show one webhook's configuration. + + The token is reported as redacted, including for a webhook registered + elsewhere: it authenticates deliveries wherever it was set, and this output + is as likely to land in a log as on a screen. + """ + client = llmwhisperer(ctx.config) + with translated(endpoint="whisper-manage-callback"): + details = client.get_webhook_details(name) + if isinstance(details, dict): + remember_secret(details.get("auth_token")) + finish(ctx, details) + + +@webhook_group.command("delete") +@click.argument("name") +@pass_context +def webhook_delete(ctx: Context, name: str) -> None: + """Remove a webhook.""" + client = llmwhisperer(ctx.config) + with translated(endpoint="whisper-manage-callback"): + finish(ctx, client.delete_webhook(name)) + + +__all__ = ["extract", "highlights", "retrieve", "status", "usage", "webhook_group"] diff --git a/src/unstract_cli/config.py b/src/unstract_cli/config.py new file mode 100644 index 0000000..042705d --- /dev/null +++ b/src/unstract_cli/config.py @@ -0,0 +1,579 @@ +"""Profile-based configuration. + +Two products with different hosts, different keys, and `org_id` as a URL *path +segment* rather than a flag. Named profiles (kubectl/aws style) hold per-product +host, key and org, plus deployment aliases so a deployment can be named instead +of spelled out. + +The resolution chain -- **flag > env > profile > built-in default** -- is +implemented once here and used by every parameter. It is never re-implemented +per command. + +The CLI is fully usable with **no config file at all**, driven entirely by +environment variables; that is the expected mode in CI and agent sandboxes. +""" + +from __future__ import annotations + +import errno +import os +import stat +import tomllib +from copy import deepcopy +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import tomli_w + +from unstract_cli.core.errors import remember_secret + +LLMWHISPERER = "llmwhisperer" +DOCSTUDIO = "docstudio" +PRODUCTS: tuple[str, ...] = (LLMWHISPERER, DOCSTUDIO) + +#: Built-in defaults, lowest precedence. +DEFAULT_BASE_URLS: dict[str, str] = { + LLMWHISPERER: "https://llmwhisperer-api.us-central.unstract.com/api/v2", + DOCSTUDIO: "https://us-central.unstract.com", +} + +#: Environment variables per (product, setting), checked before the config file. +ENV_VARS: dict[tuple[str, str], tuple[str, ...]] = { + (LLMWHISPERER, "api_key"): ("LLMWHISPERER_API_KEY",), + (LLMWHISPERER, "base_url"): ("LLMWHISPERER_BASE_URL",), + (DOCSTUDIO, "api_key"): ("UNSTRACT_DEPLOYMENT_KEY",), + (DOCSTUDIO, "base_url"): ("UNSTRACT_BASE_URL",), + (DOCSTUDIO, "org_id"): ("UNSTRACT_ORG_ID",), +} + + +#: Where the two credentials are minted. Quoted wherever the CLI reports one as +#: missing: knowing a key is unset is no help without knowing where one is made. +KEY_SOURCES = ( + "Get an LLMWhisperer key from the LLMWhisperer console; a deployment key is " + "shown on the API deployment's own page in the Unstract UI, and a key " + "covering every deployment in the organisation is minted under " + "Settings -> API Key Manager." +) + + +def settings_for(product: str) -> tuple[str, ...]: + """The settings a product actually has. + + Products differ: `org_id` is a URL path segment for one and meaningless for + the other, and reporting a setting a user has no way to supply reads as a + misconfiguration they cannot fix. + """ + return tuple(sorted(key for prod, key in ENV_VARS if prod == product)) + + +#: Filename a project can commit to point the CLI at its own settings. +PROJECT_CONFIG_NAME = ".unstract.toml" + +#: Where the config lives when nothing else selects one. +HOME_CONFIG = Path("~/.unstract/config.toml") + + +class ConfigError(Exception): + """Configuration could not be loaded or resolved.""" + + +#: Set by the root `--config` flag. Highest precedence, matching the +#: flag > env > file ordering used for every other setting. +_config_override: Path | None = None + + +def set_config_path(path: str | Path | None) -> None: + """Point this process at a specific config file (the `--config` flag).""" + global _config_override + _config_override = Path(path).expanduser() if path else None + + +def find_project_config(start: Path | None = None) -> Path | None: + """Search upward from the working directory for ``.unstract.toml``. + + Mirrors how git and ruff resolve project settings: running the CLI inside a + project picks up that project's config with no flag. The search stops at the + filesystem root, and at ``$HOME`` so a stray file in a parent directory + cannot silently capture every invocation. + + A symlinked candidate is skipped rather than followed: the file it points at + is chosen by whoever wrote the link, and this path is written to as well as + read from -- `config set` and `config init --force` would rewrite the target. + """ + current = (start or Path.cwd()).resolve() + home = Path.home().resolve() + for directory in (current, *current.parents): + candidate = directory / PROJECT_CONFIG_NAME + if candidate.is_file() and not candidate.is_symlink(): + return candidate + if directory == home: + break + return None + + +def config_path() -> Path: + """Location of the config file. + + Resolution: ``--config``, then ``$UNSTRACT_CONFIG``, then a project-local + ``.unstract.toml`` found by upward search, then ``~/.unstract/config.toml``. + + Several config files coexisting is expected, not exceptional: a per-project + file checked into a repo, a throwaway one in CI, and a personal default, each + selected per invocation. + """ + return _resolve_config_path()[0] + + +def _resolve_config_path() -> tuple[Path, bool]: + """The config path, and whether it was *discovered* rather than named. + + The boolean is the trust signal: a path the user named (``--config`` or + ``$UNSTRACT_CONFIG``) is trusted, one found by walking up from the working + directory is not. See ``UNTRUSTED_PROJECT_KEYS``. + """ + if _config_override is not None: + return _config_override, False + if override := os.environ.get("UNSTRACT_CONFIG"): + return Path(override).expanduser(), False + if local := find_project_config(): + return local, True + return HOME_CONFIG.expanduser(), False + + +def _deref(value: Any) -> Any: + """Resolve ``env:VAR_NAME`` indirection so config files hold no secrets. + + An unset variable resolves to ``None`` rather than the literal string, so a + missing credential surfaces as "not configured" instead of being sent as the + nonsense value ``"env:FOO"``. + + An empty string resolves the same way: the placeholders a generated config + carries must not satisfy `require`. + """ + if isinstance(value, str): + if value.startswith("env:"): + return os.environ.get(value[4:].strip()) or None + return value or None + return value + + +#: Settings a *discovered* project-local file may not supply: a checkout the +#: user did not write must not choose the host their key is sent to. Everything +#: else -- org_id, profile selection, deployment aliases -- is still honoured. +UNTRUSTED_PROJECT_KEYS = frozenset({"api_key", "base_url"}) + + +@dataclass +class ConfigFile: + """Parsed contents of the config file.""" + + default_profile: str | None = None + profiles: dict[str, dict[str, Any]] = field(default_factory=dict) + path: Path | None = None + exists: bool = False + #: Non-fatal diagnostics (e.g. loose file permissions), surfaced on stderr. + warnings: tuple[str, ...] = () + #: True when `path` was found by walking up from the working directory rather + #: than named. Such a file is not trusted with credentials or hosts. + is_project_local: bool = False + #: Keys withheld from an untrusted file, as ``{(profile, *blocks, key): value}``. + #: Excluded from resolution, but kept so a write-back does not drop them. + withheld: dict[tuple[str, ...], Any] = field(default_factory=dict) + + +def _strip_untrusted(profiles: dict[str, Any]) -> dict[tuple[str, ...], Any]: + """Remove the untrusted keys from a profile tree, in place, reporting what went.""" + withheld: dict[tuple[str, ...], Any] = {} + + def walk(node: Any, trail: tuple[str, ...]) -> None: + if not isinstance(node, dict): + return + for key in list(node): + if key in UNTRUSTED_PROJECT_KEYS: + withheld[(*trail, key)] = node.pop(key) + else: + walk(node[key], (*trail, key)) + + walk(profiles, ()) + return withheld + + +def _is_discovered(path: Path) -> bool: + """Whether this path is the file an upward search would have found. + + Trust follows the file, not the call: naming the project-local file that + discovery would have picked anyway does not make its contents any more the + user's own. ``--config`` and ``$UNSTRACT_CONFIG`` are a deliberate choice and + are resolved before this, so they stay trusted. + """ + candidate = find_project_config() + return candidate is not None and candidate.resolve() == path.resolve() + + +def load_config(path: Path | None = None) -> ConfigFile: + """Load the config file. A missing file is normal, not an error.""" + if path is not None: + target, project_local = path, _is_discovered(path) + else: + target, project_local = _resolve_config_path() + if not target.exists(): + return ConfigFile(path=target, exists=False, is_project_local=project_local) + + try: + with target.open("rb") as fh: + raw = tomllib.load(fh) + except (OSError, tomllib.TOMLDecodeError) as exc: + raise ConfigError(f"Could not read config at {target}: {exc}") from exc + + warnings: list[str] = [] + try: + mode = target.stat().st_mode + if mode & (stat.S_IRWXG | stat.S_IRWXO): + warnings.append( + f"Config file {target} is readable by other users " + f"(mode {stat.filemode(mode)}); consider `chmod 600`." + ) + except OSError: # pragma: no cover - stat failure is not worth failing on + pass + + profiles = raw.get("profiles", {}) + if not isinstance(profiles, dict): + raise ConfigError(f"`profiles` in {target} must be a table.") + + # Said out loud rather than dropped in silence; the rest of the file still + # applies. + withheld: dict[tuple[str, ...], Any] = {} + if project_local: + withheld = _strip_untrusted(profiles) + if withheld: + names = ", ".join(sorted(".".join(trail) for trail in withheld)) + warnings.append( + f"Ignoring {names} from project config {target}: a discovered " + f"{PROJECT_CONFIG_NAME} may not supply credentials or base URLs. " + "Pass --config explicitly, or set the environment variable instead." + ) + + return ConfigFile( + default_profile=raw.get("default_profile"), + profiles=profiles, + path=target, + exists=True, + warnings=tuple(warnings), + is_project_local=project_local, + withheld=withheld, + ) + + +def _restored_profiles(cfg: ConfigFile, target: Path) -> dict[str, Any]: + """The profiles to write, with anything withheld put back. + + Withholding a key from resolution is the security property; deleting it from + the user's file is not, and `config set` loads, mutates and saves the whole + document. Restored **only** when writing back to the file they came from -- + into any other path this would copy untrusted values somewhere they are + trusted. + """ + if not cfg.withheld or cfg.path is None or target.resolve() != cfg.path.resolve(): + return cfg.profiles + + profiles = deepcopy(cfg.profiles) + for (*parents, leaf), value in cfg.withheld.items(): + node: dict[str, Any] = profiles + for segment in parents: + child = node.get(segment) + if not isinstance(child, dict): + child = node[segment] = {} + node = child + node.setdefault(leaf, value) + return profiles + + +def save_config(cfg: ConfigFile, path: Path | None = None) -> Path: + """Write the config file with owner-only permissions.""" + target = path or cfg.path or config_path() + target.parent.mkdir(parents=True, exist_ok=True) + + doc: dict[str, Any] = {} + if cfg.default_profile: + doc["default_profile"] = cfg.default_profile + doc["profiles"] = _restored_profiles(cfg, target) + + # 0600 from the outset, never widened even briefly. O_NOFOLLOW because this + # write truncates, and the path is not always one the user chose. + flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0) + try: + fd = os.open(target, flags, 0o600) + except OSError as exc: + if exc.errno not in (errno.ELOOP, errno.EMLINK): + raise + raise ConfigError( + f"Refusing to write config through the symlink at {target}: it would " + f"overwrite {os.readlink(target)} instead. Pass --config with the path " + "of the real file." + ) from exc + with os.fdopen(fd, "wb") as fh: + tomli_w.dump(doc, fh) + os.chmod(target, 0o600) + return target + + +@dataclass +class ResolvedConfig: + """Effective settings for one invocation. + + ``overrides`` holds command-line flags, which outrank everything else. + """ + + file: ConfigFile + profile_name: str | None = None + overrides: dict[str, Any] = field(default_factory=dict) + + @property + def active_profile(self) -> str | None: + """Profile selected by flag, ``UNSTRACT_PROFILE``, or the file default.""" + return ( + self.profile_name + or os.environ.get("UNSTRACT_PROFILE") + or self.file.default_profile + ) + + def _profile(self) -> dict[str, Any]: + name = self.active_profile + if not name: + return {} + profile = self.file.profiles.get(name) + if profile is None: + if self.file.exists and self.file.profiles: + known = ", ".join(sorted(self.file.profiles)) or "none" + raise ConfigError( + f"Profile {name!r} not found in {self.file.path}. " + f"Known profiles: {known}" + ) + return {} + return profile if isinstance(profile, dict) else {} + + def _product_block(self, product: str) -> dict[str, Any]: + # One accepted shape only, settings nested under the product name: a + # config that looks applied but is not fails later with no obvious cause. + block = self._profile().get(product) + return block if isinstance(block, dict) else {} + + def get(self, product: str, key: str, default: Any = None) -> Any: + """Resolve one setting: **flag > env > profile > built-in default**.""" + value = self._resolve(product, key, default) + if key == "api_key": + remember_secret(value) + return value + + def _resolve(self, product: str, key: str, default: Any = None) -> Any: + if (value := self.overrides.get(f"{product}.{key}")) is not None: + return value + if (value := self.overrides.get(key)) is not None: + return value + + for env_var in ENV_VARS.get((product, key), ()): + if value := os.environ.get(env_var): + return value + + if (value := _deref(self._product_block(product).get(key))) is not None: + return value + + if default is not None: + return default + if key == "base_url": + return DEFAULT_BASE_URLS.get(product) + return None + + def require(self, product: str, key: str) -> Any: + """Resolve a setting, or raise a message naming exactly how to supply it.""" + if (value := self.get(product, key)) is not None: + return value + + hints: list[str] = [] + if env_vars := ENV_VARS.get((product, key)): + hints.append(f"set ${env_vars[0]}") + hints.append(f"or add `{key}` to the [profiles..{product}] block") + # Credentials have no flag by design: a secret on the command line + # lands in shell history and in the process list. + if key != "api_key": + hints.append(f"or pass --{key.replace('_', '-')}") + raise ConfigError( + f"Missing required setting {product}.{key}. To fix: {'; '.join(hints)}." + ) + + def deployment(self, alias: str) -> dict[str, Any]: + """Resolve a deployment alias to its api_name, org and key. + + ``org_id`` and ``api_key`` are optional per alias and fall back to the + profile's Document Studio block, so the common case is one line per + deployment. + """ + aliases = self._profile().get("deployments") + entry = aliases.get(alias) if isinstance(aliases, dict) else None + if not isinstance(entry, dict): + known = ( + ", ".join(sorted(aliases)) + if isinstance(aliases, dict) and aliases + else "none" + ) + raise ConfigError( + f"Deployment alias {alias!r} not found in profile " + f"{self.active_profile!r}. Known aliases: {known}." + ) + if not entry.get("api_name"): + raise ConfigError(f"Deployment alias {alias!r} has no `api_name`.") + api_key = self._alias_setting(alias, entry, "api_key") + remember_secret(api_key) + return { + "api_name": entry["api_name"], + "org_id": self._alias_setting(alias, entry, "org_id"), + "api_key": api_key, + } + + def _alias_setting(self, alias: str, entry: dict[str, Any], key: str) -> Any: + """One alias setting, falling back to the profile only where the alias is silent. + + An ``env:`` reference that does not resolve is not silence. Falling back + there runs the deployment against the profile's organisation, with the + profile's key, and reports success. + """ + raw = entry.get(key) + if isinstance(raw, str) and raw.startswith("env:"): + if value := _deref(raw): + return value + raise ConfigError( + f"Deployment alias {alias!r} sets {key} to {raw!r}, and " + f"${raw[4:].strip()} is not set in this process's environment." + ) + return raw or self.get(DOCSTUDIO, key) + + def deployment_aliases(self) -> tuple[str, ...]: + """Names of the deployment aliases defined in the active profile.""" + aliases = self._profile().get("deployments") + return tuple(sorted(aliases)) if isinstance(aliases, dict) else () + + def resolution_source(self, product: str, key: str) -> dict[str, Any]: + """Report where a setting resolves from, without echoing a secret. + + `config doctor` uses this to answer the question that costs the most + time: "the CLI says the key is not configured, but I set it -- where is + it looking?" + """ + if ( + self.overrides.get(f"{product}.{key}") is not None + or self.overrides.get(key) is not None + ): + return {"resolved": True, "source": "flag/override"} + + for env_var in ENV_VARS.get((product, key), ()): + if os.environ.get(env_var): + return {"resolved": True, "source": f"env:{env_var}"} + + raw = self._product_block(product).get(key) + if isinstance(raw, str) and raw.startswith("env:"): + var = raw[4:].strip() + present = bool(os.environ.get(var)) + return { + "resolved": present, + "source": f"profile -> env:{var}", + "detail": None + if present + else f"${var} is not set in this process's environment", + } + if raw not in (None, ""): + return {"resolved": True, "source": "profile (literal)"} + + report: dict[str, Any] = ( + {"resolved": True, "source": "built-in default"} + if key == "base_url" and DEFAULT_BASE_URLS.get(product) + else {"resolved": False, "source": "unset"} + ) + if detail := self.withheld_detail(product, key): + report["detail"] = detail + return report + + def withheld_detail(self, *trail: str) -> str | None: + """Why a setting the config file plainly holds did not arrive, if that is why. + + Reporting only where a value came *from* would leave the user staring at + a setting they can see in the file. Takes a trail rather than a + product/key pair so a deployment alias's own key -- nested a level deeper + -- is answerable too. + """ + if (self.active_profile, *trail) not in self.file.withheld: + return None + return ( + f"{self.file.path} sets {trail[-1]}, and a discovered " + f"{PROJECT_CONFIG_NAME} is not trusted with it." + ) + + +def starter_profiles() -> dict[str, dict[str, Any]]: + """Profile stubs written by `config init`. + + Every credential uses ``env:`` indirection: the generated file is a map of + where secrets live, never a copy of them. + + One key on the product block, and aliases that carry only ``api_name``: a + key can cover every deployment in the organisation, so a key per alias is + the exception -- for an organisation whose deployments hold separate keys -- + rather than the shape to start from. + """ + return { + "cloud-us": { + LLMWHISPERER: { + "base_url": DEFAULT_BASE_URLS[LLMWHISPERER], + "api_key": "env:LLMWHISPERER_API_KEY", + }, + DOCSTUDIO: { + "base_url": DEFAULT_BASE_URLS[DOCSTUDIO], + "org_id": "", + "api_key": "env:UNSTRACT_DEPLOYMENT_KEY", + }, + "deployments": {"example": {"api_name": "your-api-deployment-name"}}, + }, + "cloud-eu": { + LLMWHISPERER: { + "base_url": "https://llmwhisperer-api.eu-west.unstract.com/api/v2", + "api_key": "env:LLMWHISPERER_API_KEY", + }, + }, + # A shape to copy for a self-hosted install, not a profile to select: + # its host is a placeholder and only the active profile is resolved. + "onprem-example": { + LLMWHISPERER: { + "base_url": "https://llmwhisperer.unstract.internal.example/api/v2", + "api_key": "env:LLMWHISPERER_API_KEY", + }, + DOCSTUDIO: { + "base_url": "https://unstract.internal.example", + "org_id": "", + "api_key": "env:UNSTRACT_DEPLOYMENT_KEY", + }, + }, + } + + +__all__ = [ + "DEFAULT_BASE_URLS", + "DOCSTUDIO", + "ENV_VARS", + "HOME_CONFIG", + "KEY_SOURCES", + "LLMWHISPERER", + "PRODUCTS", + "PROJECT_CONFIG_NAME", + "UNTRUSTED_PROJECT_KEYS", + "ConfigError", + "ConfigFile", + "ResolvedConfig", + "config_path", + "find_project_config", + "load_config", + "save_config", + "set_config_path", + "settings_for", + "starter_profiles", +] diff --git a/src/unstract_cli/core/__init__.py b/src/unstract_cli/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/unstract_cli/core/clients.py b/src/unstract_cli/core/clients.py new file mode 100644 index 0000000..d0f3fde --- /dev/null +++ b/src/unstract_cli/core/clients.py @@ -0,0 +1,199 @@ +"""Building the product clients, and turning their failures into CLI errors. + +The entry point deliberately does not catch bare ``Exception``: an unexpected +crash should look like a crash. Everything a client raises on purpose is +expected, so it is translated here into a ``CLIError`` carrying an exit code, a +hint and the response detail. + +The two clients report failure differently -- LLMWhisperer raises with a status +code attached, the deployment client returns a dict containing one -- so both +shapes converge here rather than in each command. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from typing import Any + +from requests.exceptions import ConnectionError, Timeout +from unstract.api_deployments.client import ( + APIDeploymentsClient, + APIDeploymentsClientException, +) +from unstract.llmwhisperer.client_v2 import ( + LLMWhispererClientException, + LLMWhispererClientV2, +) + +from unstract_cli.config import DOCSTUDIO, LLMWHISPERER, ResolvedConfig +from unstract_cli.core.errors import CLIError, ExitCode, error_from_status +from unstract_cli.core.params import find_operation + + +def llmwhisperer(config: ResolvedConfig) -> LLMWhispererClientV2: + """Build the LLMWhisperer client from the resolved configuration.""" + return LLMWhispererClientV2( + base_url=config.require(LLMWHISPERER, "base_url"), + api_key=config.require(LLMWHISPERER, "api_key"), + logging_level="ERROR", + ) + + +def deployment_url(base_url: str, org_id: str, api_name: str) -> str: + """The deployment's full URL, laid out as the spec declares the route. + + The client takes the whole URL and reads the organisation and API name back + out of its last two segments, so the route is built from the spec rather + than from a format string that can disagree with it. + """ + path = find_operation(DOCSTUDIO, "execute")["path"] + path = path.format(org_name=org_id, api_name=api_name) + return base_url.rstrip("/") + path + + +def deployment( + config: ResolvedConfig, target: str, transport_timeout: float | None = None +) -> APIDeploymentsClient: + """Build a deployment client for an alias, or for a bare API name. + + An alias carries its own organisation and key; a bare name falls back to the + profile's, so an unconfigured caller can still name a deployment directly. + """ + if target in config.deployment_aliases(): + entry = config.deployment(target) + api_name, org_id, api_key = ( + entry["api_name"], + entry["org_id"], + entry["api_key"], + ) + else: + api_name = target + org_id = config.get(DOCSTUDIO, "org_id") + api_key = config.get(DOCSTUDIO, "api_key") + + missing = [ + name for name, value in (("org_id", org_id), ("api_key", api_key)) if not value + ] + if missing: + raise CLIError( + f"Deployment {target!r} is missing {' and '.join(missing)}.", + ExitCode.USAGE, + hint=( + "Define the deployment as an alias in the active profile, or set " + "$UNSTRACT_ORG_ID and $UNSTRACT_DEPLOYMENT_KEY." + ), + ) + + return APIDeploymentsClient( + api_url=deployment_url(config.require(DOCSTUDIO, "base_url"), org_id, api_name), + api_key=api_key, + logging_level="ERROR", + transport_timeout=transport_timeout, + ) + + +def _message_and_details(value: Any) -> tuple[str, Any]: + """Split a client's error value into a one-line message and the raw detail. + + LLMWhisperer raises with either a string or the decoded error body, and the + body's own wording is better than anything invented here. + """ + if isinstance(value, dict): + for key in ("message", "error", "detail", "reason"): + if text := value.get(key): + return str(text), value + return str(value), value + return str(value), None + + +@contextmanager +def translated(endpoint: str | None = None) -> Iterator[None]: + """Turn a client failure into a CLIError with an exit code and a hint.""" + try: + yield + except LLMWhispererClientException as exc: + message, details = _message_and_details(exc.value) + status = exc.status_code or ( + details.get("status_code") if isinstance(details, dict) else None + ) + if status: + raise error_from_status( + int(status), message, details=details, endpoint=endpoint + ) from exc + raise CLIError(message, details=details, endpoint=endpoint) from exc + except APIDeploymentsClientException as exc: + raise CLIError(str(exc), ExitCode.USAGE, endpoint=endpoint) from exc + except Timeout as exc: + raise CLIError( + str(exc), + ExitCode.TIMEOUT, + endpoint=endpoint, + retryable=True, + hint="The request timed out in transit; the job may still be running.", + ) from exc + except ConnectionError as exc: + raise CLIError( + str(exc), + ExitCode.SERVER_ERROR, + endpoint=endpoint, + retryable=True, + hint="Could not reach the service. Check the base URL and connectivity.", + ) from exc + + +def translating( + call: Callable[..., Any], endpoint: str | None = None +) -> Callable[..., Any]: + """Wrap one call so its failures are CLIErrors where they happen. + + A ``with translated(...)`` around a loop converts nothing until the loop is + left, by which point what the loop knew -- the job handle above all -- is out + of scope. + """ + + def wrapped(*args: Any, **kwargs: Any) -> Any: + with translated(endpoint=endpoint): + return call(*args, **kwargs) + + return wrapped + + +def raise_for_result(result: dict[str, Any], endpoint: str | None = None) -> None: + """Fail on a deployment response that reports an error status. + + The deployment client returns its status code instead of raising, so a + failure would otherwise be reported as a successful run whose payload + happens to contain an error. + """ + status = int(result.get("status_code") or 0) + reported = result.get("error") + if status and not 200 <= status < 300: + raise error_from_status( + status, + str(reported or f"Request failed with status {status}"), + details=result, + endpoint=endpoint, + ) + if reported: + # HTTP success carrying a failure in the body. Not retryable: a re-run + # starts a second billed execution rather than retrying the first. + raise CLIError( + str(reported), + ExitCode.VALIDATION, + http_status=status or None, + details=result, + endpoint=endpoint, + hint="The request was accepted and the work was not done; `details` " + "carries the service's own report.", + ) + + +__all__ = [ + "deployment", + "deployment_url", + "llmwhisperer", + "raise_for_result", + "translated", + "translating", +] diff --git a/src/unstract_cli/core/discover.py b/src/unstract_cli/core/discover.py new file mode 100644 index 0000000..ed175d8 --- /dev/null +++ b/src/unstract_cli/core/discover.py @@ -0,0 +1,142 @@ +"""`--discover`: the CLI describing itself, in three tiers. + +An agent driving this CLI needs to know what exists before it can run anything, +and `--help` is prose scraped from a terminal. Discovery answers the same +question as JSON, at whichever depth the question needs: + +* ``groups`` -- what products are here at all +* ``summary`` -- what commands each group has +* ``full`` -- every flag with its type, default and allowed values, plus the + exit codes and the output contract, which is enough to construct a call and + read its answer without a second round trip + +Every tier is read back from Click itself. Describing commands from anywhere +else lets the description drift from what the parser accepts. +""" + +from __future__ import annotations + +from typing import Any + +import click + +from unstract_cli.core.errors import _ERROR_CODES, ExitCode +from unstract_cli.core.output import CONTRACT_VERSION + +TIERS = ("groups", "summary", "full") + + +def contract() -> dict[str, Any]: + """How to consume this CLI's output, published rather than assumed. + + Both halves of the compatibility bargain are written down here: what we + promise not to break, and what a consumer has to do for that promise to be + worth anything. + """ + return { + "version": CONTRACT_VERSION, + "envelope": ["ok", "data", "error", "meta"], + "rules": [ + "Pass `-o json`. The default format is for people and is free to " + "change; json is the parseable one and never varies with the " + "terminal, the config or the environment.", + "Ignore fields you do not recognise. New ones are added within a " + "major version.", + "Refuse a `meta.contract_version` whose value is greater than the " + "one you were written against: the shape has changed under you.", + "Branch on the exit code, not on the message text.", + "Read stdout for the envelope only. Diagnostics are on stderr.", + ], + } + + +def exit_codes() -> list[dict[str, Any]]: + """The exit-code table, which is part of the contract callers branch on.""" + return [ + { + "code": int(code), + "name": code.name.lower(), + "error_code": _ERROR_CODES.get(code, ""), + } + for code in ExitCode + ] + + +def _param(param: click.Parameter) -> dict[str, Any]: + """One flag or argument, in the terms a caller needs to supply it.""" + entry: dict[str, Any] = { + "name": param.name, + "kind": "argument" if isinstance(param, click.Argument) else "option", + "type": getattr(param.type, "name", "text"), + "required": bool(param.required), + } + if isinstance(param, click.Option): + entry["flags"] = list(param.opts) + list(param.secondary_opts) + entry["help"] = param.help or "" + entry["repeatable"] = bool(param.multiple) + if isinstance(param.type, click.Choice): + entry["choices"] = list(param.type.choices) + if param.default is not None and not isinstance(param, click.Argument): + entry["default"] = param.default + return entry + + +def _describe(command: click.Command, tier: str) -> dict[str, Any]: + entry: dict[str, Any] = {"help": (command.help or "").strip().split("\n")[0]} + if tier == "full" and not isinstance(command, click.Group): + entry["params"] = [ + _param(p) for p in command.params if p.name not in ("help", "discover") + ] + # Which field `--output raw` prints for this command, where it has one. + if raw := getattr(command, "raw_field", None): + entry["raw_field"] = raw + if isinstance(command, click.Group): + entry["commands"] = { + name: _describe(sub, tier) for name, sub in sorted(command.commands.items()) + } + return entry + + +def discover(root: click.Group, tier: str) -> dict[str, Any]: + """Describe the CLI at one tier. + + ``groups`` stops at the top level rather than walking further, so the cheap + question stays cheap: an agent starts here and drills down only where it + needs to. + """ + if tier not in TIERS: + raise ValueError(f"Unknown discovery tier {tier!r}. One of: {', '.join(TIERS)}") + + if tier == "groups": + top = sorted(root.commands.items()) + + def summary(name: str, command: click.Command) -> dict[str, str]: + return {"name": name, "help": (command.help or "").strip().split("\n")[0]} + + return { + "tier": tier, + "groups": [ + summary(name, sub) for name, sub in top if isinstance(sub, click.Group) + ], + # Leaf commands are listed apart from the groups, so a consumer + # walking groups for their commands does not drop them. + "commands": [ + summary(name, sub) + for name, sub in top + if not isinstance(sub, click.Group) + ], + } + + payload: dict[str, Any] = { + "tier": tier, + "commands": { + name: _describe(sub, tier) for name, sub in sorted(root.commands.items()) + }, + } + if tier == "full": + payload["exit_codes"] = exit_codes() + payload["contract"] = contract() + return payload + + +__all__ = ["TIERS", "contract", "discover", "exit_codes"] diff --git a/src/unstract_cli/core/errors.py b/src/unstract_cli/core/errors.py new file mode 100644 index 0000000..a36402a --- /dev/null +++ b/src/unstract_cli/core/errors.py @@ -0,0 +1,286 @@ +"""Exit codes, structured errors, and secret redaction. + +Exit codes are a stable API: a caller branches on them without parsing prose. +Every failure also carries `hint` and `retryable` so the caller can self-correct +rather than retry blindly. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from enum import IntEnum +from typing import Any + + +class ExitCode(IntEnum): + SUCCESS = 0 + GENERIC = 1 + USAGE = 2 + AUTH = 3 + NOT_FOUND = 4 + VALIDATION = 5 + RATE_LIMITED = 6 + TIMEOUT = 7 + SERVER_ERROR = 8 + ALREADY_CONSUMED = 9 + SAVE_FAILED = 10 + #: 128 + SIGINT, the value a shell and every job runner already read as + #: "the user stopped it" rather than as a failure of the command. + INTERRUPTED = 130 + + +#: HTTP status -> exit code. An in-progress 422 never reaches here: the poll +#: engine branches on the response body first. +_STATUS_MAP: dict[int, ExitCode] = { + 400: ExitCode.VALIDATION, + 401: ExitCode.AUTH, + 403: ExitCode.AUTH, + 404: ExitCode.NOT_FOUND, + # Only the deployment status endpoint answers 406; the whisper equivalent + # is a 400 whose body says so, which is prose we do not translate on. + 406: ExitCode.ALREADY_CONSUMED, + 408: ExitCode.TIMEOUT, + 409: ExitCode.VALIDATION, + 422: ExitCode.VALIDATION, + 429: ExitCode.RATE_LIMITED, +} + +_ERROR_CODES: dict[ExitCode, str] = { + ExitCode.GENERIC: "error", + ExitCode.USAGE: "usage_error", + ExitCode.AUTH: "auth_error", + ExitCode.NOT_FOUND: "not_found", + ExitCode.VALIDATION: "validation_error", + ExitCode.RATE_LIMITED: "rate_limited", + ExitCode.TIMEOUT: "timeout", + ExitCode.SERVER_ERROR: "server_error", + ExitCode.ALREADY_CONSUMED: "already_consumed", + ExitCode.SAVE_FAILED: "save_failed", + ExitCode.INTERRUPTED: "interrupted", +} + + +def exit_code_for_status(status: int) -> ExitCode: + """Map an HTTP status onto its exit code.""" + if code := _STATUS_MAP.get(status): + return code + if 500 <= status < 600: + return ExitCode.SERVER_ERROR + # A 3xx that was not followed, or a status no spec declares, is still a + # failure: never fall through to SUCCESS. + return ExitCode.GENERIC + + +def is_retryable(status: int) -> bool: + """Retry only on rate limiting and server faults -- never on 4xx. + + Retrying a 4xx re-sends a request the server already rejected on its merits, + and for one-shot reads a blind retry can consume a result the first attempt + already delivered. + """ + return status == 429 or 500 <= status < 600 + + +# --------------------------------------------------------------------------- # +# Redaction +# --------------------------------------------------------------------------- # + +_SECRET_HEADERS = {"unstract-key", "authorization", "apikey"} +_SECRET_HEADER_PREFIXES = ("x-",) +_SECRET_KEY_HINTS = ("key", "token", "secret", "password", "credential", "auth") +REDACTED = "***REDACTED***" + +#: Credentials resolved during this run. Registered where they are resolved, so +#: no emitter has to remember to opt into scrubbing. +_KNOWN_SECRETS: set[str] = set() + + +def remember_secret(value: Any) -> None: + """Record a resolved credential so no stream can print it later.""" + if isinstance(value, str) and len(value) >= 8: + _KNOWN_SECRETS.add(value) + + +def known_secrets() -> list[str]: + """Every credential resolved so far, longest first. + + Longest first so a key that contains another as a prefix is replaced whole + rather than leaving its tail behind. + """ + return sorted(_KNOWN_SECRETS, key=len, reverse=True) + + +def redact_headers(headers: dict[str, Any]) -> dict[str, Any]: + """Redact credential-bearing headers.""" + out: dict[str, Any] = {} + for key, value in headers.items(): + low = key.lower() + secret = low in _SECRET_HEADERS or ( + low.startswith(_SECRET_HEADER_PREFIXES) + and any(hint in low for hint in _SECRET_KEY_HINTS) + ) + out[key] = REDACTED if secret else value + return out + + +def redact_value(value: Any) -> Any: + """Recursively redact secret-looking keys in a payload.""" + if isinstance(value, dict): + return { + k: ( + REDACTED + if any(hint in str(k).lower() for hint in _SECRET_KEY_HINTS) + and isinstance(v, str) + else redact_value(v) + ) + for k, v in value.items() + } + if isinstance(value, list): + return [redact_value(v) for v in value] + return value + + +def scrub(text: str, secrets: list[str]) -> str: + """Remove known secret literals from free text. + + Last line of defence: a credential that reaches a message body via an + upstream error string still must not be printed. Short values are skipped -- + redacting a 3-character "key" would mangle unrelated text. + """ + for secret in secrets: + if secret and len(secret) >= 8: + text = re.sub(re.escape(secret), REDACTED, text) + return text + + +# --------------------------------------------------------------------------- # +# CLIError +# --------------------------------------------------------------------------- # + + +@dataclass +class CLIError(Exception): + """A failure that maps onto an exit code and a structured error payload.""" + + message: str + exit_code: ExitCode = ExitCode.GENERIC + http_status: int | None = None + details: Any = None + endpoint: str | None = None + hint: str | None = None + retryable: bool = False + code: str | None = None + extra: dict[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + super().__init__(self.message) + if self.exit_code is ExitCode.SUCCESS: + raise ValueError("a CLIError cannot carry the success exit code") + + def to_dict(self) -> dict[str, Any]: + payload: dict[str, Any] = { + "code": self.code or _ERROR_CODES.get(self.exit_code, "error"), + "message": self.message, + "exit_code": int(self.exit_code), + "retryable": self.retryable, + } + if self.http_status is not None: + payload["http_status"] = self.http_status + if self.details is not None: + # Structural, not opt-in: the details come from a server body that + # can echo the request, headers and key included. + payload["details"] = redact_value(self.details) + if self.endpoint: + payload["endpoint"] = self.endpoint + if self.hint: + payload["hint"] = self.hint + payload.update(self.extra) + return payload + + +def error_from_status( + status: int, message: str, *, details: Any = None, endpoint: str | None = None +) -> CLIError: + """Build a CLIError from an HTTP status, with its exit code, hint and retryability.""" + return CLIError( + message, + exit_code_for_status(status), + http_status=status, + details=details, + endpoint=endpoint, + hint=hint_for(status), + retryable=is_retryable(status), + ) + + +def undeclared_status_error( + status: int, body: Any, endpoint: str | None = None +) -> CLIError: + """Report a status the spec does not declare, verbatim. + + A guessed message for an unknown status is worse than none: it sends the + reader after the wrong cause. The body is passed through untouched. + """ + return CLIError( + f"Undeclared status {status} with body {body!r}", + exit_code_for_status(status), + http_status=status, + details=body, + endpoint=endpoint, + retryable=is_retryable(status), + ) + + +def hint_for(status: int) -> str | None: + """A short, actionable next step for a common failure.""" + match status: + case 400: + return ( + "The service rejected the request. Check the ids and parameter " + "values passed; `details` carries the service's own response." + ) + case 401 | 403: + # Wrong, revoked, foreign-organisation and not-permitted all arrive + # as the same response, so the hint cannot settle on one of them. + return ( + "The key was rejected. Keys are per-product: `unstract config " + "doctor` reports which one resolved and from where. A key that " + "works elsewhere can still be rejected here -- it may not cover " + "this deployment, or may belong to another organisation." + ) + case 404: + return ( + "Verify the resource id, and that the organisation matches the " + "resource's own. For deployments, confirm the API name." + ) + case 406: + return ( + "This execution result was already retrieved. A deployment serves " + "its result exactly once; re-running the status call cannot " + "recover it. Pass --save to `deployment run` to keep the next one." + ) + case 409: + return "The resource is in use, or conflicts with an existing one." + case 429: + return "Rate limited. Back off and retry." + if 500 <= status < 600: + return "Server-side failure. If it persists, check service status." + return None + + +__all__ = [ + "REDACTED", + "CLIError", + "ExitCode", + "known_secrets", + "remember_secret", + "error_from_status", + "exit_code_for_status", + "hint_for", + "is_retryable", + "redact_headers", + "redact_value", + "scrub", + "undeclared_status_error", +] diff --git a/src/unstract_cli/core/output.py b/src/unstract_cli/core/output.py new file mode 100644 index 0000000..b0c1966 --- /dev/null +++ b/src/unstract_cli/core/output.py @@ -0,0 +1,329 @@ +"""Output rendering, and choosing which rendering to use. + +The contract a caller depends on: + +* ``-o json`` writes **one envelope to stdout and nothing else** -- ``{ok, data, + error, meta}`` -- on success and on failure alike, so a failed run still + yields a valid object rather than an empty stream. +* What ``-o json`` produces depends on nothing but the command and its + arguments: not on a terminal, not on configuration, not on who is calling. +* Human-facing notes, warnings and progress all go to stderr. +* Without ``-o`` the output is ``table``, which is for people to read and is + free to change. Anything parsing this CLI passes ``-o json`` explicitly. + +The one thing an unflagged run reads from its environment is which *default* to +use: a coding agent gets ``json``, because an agent that has to be told twice is +an agent that parses a table. Detection is never allowed to reach past the +default -- see ``resolve_format``. +""" + +from __future__ import annotations + +import json +import os +import shutil +import sys +import textwrap +from collections.abc import Mapping +from enum import StrEnum +from fnmatch import fnmatch +from typing import Any + +from unstract_cli.core.errors import CLIError, ExitCode, known_secrets, scrub + +#: Major version of the stdout envelope, published in every ``meta``. +CONTRACT_VERSION = 1 + +#: Environment markers the coding agents set for the tools they drive. Patterns, +#: so a family of variables can be named once. +AGENT_ENV = ("CLAUDECODE", "CURSOR_AGENT", "CODEX_*", "AI_AGENT") + + +class OutputFormat(StrEnum): + JSON = "json" + TABLE = "table" + RAW = "raw" + + +class AgentMode(StrEnum): + AUTO = "auto" + YES = "yes" + NO = "no" + + +def agent_detected(env: Mapping[str, str] | None = None) -> bool: + """Whether the environment looks like a coding agent's.""" + names = os.environ if env is None else env + return any( + names[name] and fnmatch(name, pattern) for name in names for pattern in AGENT_ENV + ) + + +def resolve_format( + explicit: str | None, + agent: str = AgentMode.AUTO, + env: Mapping[str, str] | None = None, +) -> OutputFormat: + """The format to render in. + + An explicit ``-o`` wins outright, so detection can only ever pick the + default: two runs of ``-o json`` in different environments render the same + bytes, which is the property a script is relying on. + """ + if explicit: + return OutputFormat(explicit) + if agent == AgentMode.YES or (agent == AgentMode.AUTO and agent_detected(env)): + return OutputFormat.JSON + return OutputFormat.TABLE + + +def envelope( + *, + data: Any = None, + error: dict[str, Any] | None = None, + meta: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Build the stdout envelope. ``ok`` is derived, never passed in.""" + return { + "ok": error is None, + "data": data, + "error": error, + "meta": {**(meta or {}), "contract_version": CONTRACT_VERSION}, + } + + +def _flatten(value: Any) -> str: + """Render a cell. Nested structures become compact JSON, not Python reprs.""" + if value is None: + return "" + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (dict, list)): + return json.dumps(value, default=str) + return str(value) + + +def _rows_and_columns( + data: Any, columns: tuple[str, ...] = () +) -> tuple[list[str], list[list[str]]]: + """Derive table columns and rows from arbitrary JSON. + + List of objects -> columns from the union of keys in first-seen order; + single object -> a two-column key/value listing; anything else -> one + ``value`` column. ``columns`` overrides the selection where the generic rule + reads poorly. + """ + if isinstance(data, dict): + # Unwrap a single list-valued envelope, e.g. {"results": [...]}. + for key in ("results", "message", "members", "data", "highlights"): + inner = data.get(key) + if isinstance(inner, list) and inner: + data = inner + break + + if isinstance(data, list): + if not data: + return [], [] + if all(isinstance(item, dict) for item in data): + if columns: + headers = list(columns) + else: + headers = [] + for item in data: + headers.extend(k for k in item if k not in headers) + return headers, [[_flatten(item.get(h)) for h in headers] for item in data] + return ["value"], [[_flatten(item)] for item in data] + + if isinstance(data, dict): + keys = list(columns) if columns else list(data) + return ["key", "value"], [[k, _flatten(data.get(k))] for k in keys] + + return ["value"], [[_flatten(data)]] + + +def _terminal_width(default: int = 100) -> int: + try: + return max(shutil.get_terminal_size((default, 24)).columns, 40) + except Exception: # pragma: no cover - detached terminal + return default + + +def render_table( + data: Any, columns: tuple[str, ...] = (), *, max_width: int | None = None +) -> str: + """Render as an aligned plain-text table. + + Plain text rather than box drawing: tables end up in logs and terminals of + varying width, and ASCII survives both. + + Long cells are **wrapped, never truncated**: a table is a view of the data, + not a lossy summary, and a silently dropped tail is the kind of thing you + only notice after acting on it. + """ + headers, rows = _rows_and_columns(data, columns) + if not headers: + return "(no results)" + + gutter = 2 + total_width = max_width or _terminal_width() + + natural = [len(h) for h in headers] + for row in rows: + for i, cell in enumerate(row): + if i < len(natural): + natural[i] = max( + natural[i], max((len(p) for p in cell.split("\n")), default=0) + ) + + # Shrink only the widest columns, and only as far as the terminal requires, + # so a narrow column is never squeezed on behalf of a wide neighbour. + widths = list(natural) + budget = total_width - gutter * (len(headers) - 1) + while sum(widths) > budget and max(widths) > 8: + widest = widths.index(max(widths)) + widths[widest] -= 1 + + def fmt(cells: list[str]) -> list[str]: + """Lay one logical row out over as many physical lines as it needs.""" + wrapped = [ + textwrap.wrap(cell, width=w, break_long_words=True, break_on_hyphens=False) + or [""] + for cell, w in zip(cells, widths, strict=False) + ] + height = max(len(parts) for parts in wrapped) + lines = [] + for line_no in range(height): + pieces = [ + (parts[line_no] if line_no < len(parts) else "").ljust(w) + for parts, w in zip(wrapped, widths, strict=False) + ] + lines.append((" " * gutter).join(pieces).rstrip()) + return lines + + out = fmt(headers) + out.append((" " * gutter).join("-" * w for w in widths).rstrip()) + for row in rows: + out.extend(fmt(row)) + return "\n".join(out) + + +def render( + env: dict[str, Any], + fmt: OutputFormat = OutputFormat.JSON, + *, + columns: tuple[str, ...] = (), + raw_field: str | None = None, +) -> str: + """Render an envelope. ``table`` and ``raw`` show ``data``, or the error.""" + if fmt is OutputFormat.JSON: + return json.dumps(env, indent=2, default=str) + + payload = env["data"] if env["ok"] else env["error"] + if fmt is OutputFormat.TABLE: + return render_table(payload, columns) + + if isinstance(payload, dict) and raw_field and raw_field in payload: + payload = payload[raw_field] + if isinstance(payload, bytes): + return payload.decode("utf-8", errors="replace") + if isinstance(payload, str): + return payload + return json.dumps(payload, indent=2, default=str) + + +def emit( + env: dict[str, Any], + fmt: OutputFormat = OutputFormat.JSON, + *, + columns: tuple[str, ...] = (), + raw_field: str | None = None, + secrets: list[str] | None = None, +) -> None: + """Write one envelope to stdout -- and nothing else to stdout. + + Every credential resolved during the run is scrubbed whether or not the + caller passed one: an emitter that has to remember is an emitter that + eventually forgets. + """ + emit_text(render(env, fmt, columns=columns, raw_field=raw_field), secrets=secrets) + + +def emit_text(text: str, *, secrets: list[str] | None = None) -> None: + """Write already-rendered text to stdout, scrubbed the way an envelope is. + + A command that renders its own table is still writing to the stream no + credential may reach, and scrubbing it by hand is the arrangement that + eventually forgets. + """ + if to_hide := [*(secrets or []), *known_secrets()]: + text = scrub(text, to_hide) + print(text) + + +def emit_result( + data: Any, + fmt: OutputFormat = OutputFormat.JSON, + *, + meta: dict[str, Any] | None = None, + columns: tuple[str, ...] = (), + raw_field: str | None = None, + secrets: list[str] | None = None, +) -> None: + """Write a successful result.""" + emit( + envelope(data=data, meta=meta), + fmt, + columns=columns, + raw_field=raw_field, + secrets=secrets, + ) + + +def emit_error( + error: CLIError, + fmt: OutputFormat = OutputFormat.JSON, + *, + meta: dict[str, Any] | None = None, + secrets: list[str] | None = None, +) -> ExitCode: + """Write a failure envelope to stdout and a one-line summary to stderr. + + Returns the exit code so the caller can hand it straight to the shell. + """ + emit(envelope(error=error.to_dict(), meta=meta), fmt, secrets=secrets) + summary = error.message + if to_hide := [*(secrets or []), *known_secrets()]: + summary = scrub(summary, to_hide) + print(f"error: {summary}", file=sys.stderr) + return error.exit_code + + +def diagnostic( + message: str, *, quiet: bool = False, verbosity: int = 0, level: int = 0 +) -> None: + """Write a human-facing note to **stderr**, keeping stdout parseable. + + ``level`` is the minimum ``-v`` count required: 0 always shows (unless + ``--quiet``), 1 needs ``-v``, 2 needs ``-vv``. + """ + if quiet or verbosity < level: + return + print(message, file=sys.stderr) + + +__all__ = [ + "AGENT_ENV", + "CONTRACT_VERSION", + "AgentMode", + "OutputFormat", + "agent_detected", + "diagnostic", + "emit", + "emit_error", + "emit_result", + "emit_text", + "envelope", + "render", + "render_table", + "resolve_format", +] diff --git a/src/unstract_cli/core/overlay.py b/src/unstract_cli/core/overlay.py new file mode 100644 index 0000000..337f1d6 --- /dev/null +++ b/src/unstract_cli/core/overlay.py @@ -0,0 +1,38 @@ +"""What the specs cannot say about a flag. + +The committed specs are generated from server code, so they carry names, types +and defaults but no allowed-value lists, no short flags and, today, no parameter +descriptions. Those live here rather than in the derivation, so adding one is an +edit to a data file instead of a special case in code. + +TOML, read with the stdlib, for the same reason the config file is TOML: no +parser dependency, and the file stays editable without a code change. + +Anything not overridden falls through to the spec, so an empty overlay is a +valid overlay. +""" + +from __future__ import annotations + +import tomllib +from functools import cache +from importlib import resources +from typing import Any + +OVERLAY_FILE = "overlay.toml" + + +@cache +def load_overlay() -> dict[str, Any]: + """Read the packaged overlay.""" + text = (resources.files("unstract_cli") / OVERLAY_FILE).read_text(encoding="utf-8") + return tomllib.loads(text) + + +def overlay_for(product: str, operation_id: str) -> dict[str, dict[str, Any]]: + """Per-parameter overrides for one operation, keyed by parameter name.""" + entries = load_overlay().get(product, {}).get(operation_id, {}) + return {name: entry for name, entry in entries.items() if isinstance(entry, dict)} + + +__all__ = ["OVERLAY_FILE", "load_overlay", "overlay_for"] diff --git a/src/unstract_cli/core/params.py b/src/unstract_cli/core/params.py new file mode 100644 index 0000000..817e017 --- /dev/null +++ b/src/unstract_cli/core/params.py @@ -0,0 +1,415 @@ +"""Command flags, derived from the committed API specs. + +The specs are the same artifacts the published clients are generated from, so a +parameter the API gains reaches the CLI by refreshing a JSON file rather than by +hand-editing a flag list that drifts the moment nobody looks at it. + +Two rules make the derivation safe to hand to a caller: + +* **A flag not passed is not sent.** Every option defaults to ``None``, which + means "absent", and the server's own default applies. Writing the spec's + default into the request instead would pin a value the server would otherwise + choose, and the two diverge the moment the server's default moves. +* **A falsy value is a choice, not an absence.** ``0``, ``false`` and ``""`` all + travel; only ``None`` is filtered. + +What the spec cannot express -- allowed values, short flags, wording -- comes +from the overlay, never from a guess made here. +""" + +from __future__ import annotations + +import inspect +import json +import re +from collections.abc import Callable +from dataclasses import dataclass, replace +from functools import cache +from importlib import resources +from typing import Any + +import click + +from unstract_cli.core.overlay import overlay_for + +#: Spec file per product, vendored so flags derive with no network and no +#: dependency on where the client happens to be installed from. +SPEC_FILES = {"llmwhisperer": "llmwhisperer.json", "docstudio": "docstudio.json"} + +_HTTP_METHODS = frozenset({"get", "post", "put", "patch", "delete"}) + +#: OpenAPI type -> Click type. `array` is handled separately, as repetition. +_TYPES: dict[str, click.ParamType] = { + "string": click.STRING, + "integer": click.INT, + "number": click.FLOAT, +} + + +@cache +def load_spec(product: str) -> dict[str, Any]: + """Read one vendored spec.""" + try: + filename = SPEC_FILES[product] + except KeyError: + raise KeyError(f"No spec vendored for product {product!r}") from None + text = (resources.files("unstract_cli.specs") / filename).read_text(encoding="utf-8") + return json.loads(text) + + +def find_operation(product: str, operation_id: str) -> dict[str, Any]: + """Look one operation up by its operationId.""" + for path, methods in load_spec(product)["paths"].items(): + for method, operation in methods.items(): + if method in _HTTP_METHODS and operation.get("operationId") == operation_id: + return {"path": path, "method": method, **operation} + raise KeyError(f"{product} spec declares no operation {operation_id!r}") + + +@dataclass(frozen=True) +class Param: + """One request parameter, as the spec describes it.""" + + name: str + type: str = "string" + default: Any = None + description: str = "" + array: bool = False + nullable: bool = False + required: bool = False + choices: tuple[str, ...] = () + + @property + def flag(self) -> str: + return "--" + self.name.replace("_", "-") + + +def _from_schema( + name: str, schema: dict[str, Any], description: str, *, required: bool = False +) -> Param: + """Read one parameter out of its JSON schema. + + Nullability has two spellings -- a `null` branch in a type union, and 3.0's + `nullable` keyword, which is what the deployment spec uses. The null branch + carries no information for a flag, so the other branch decides the type. + """ + types = schema.get("type") + if isinstance(types, list): + nullable = "null" in types + remaining = [t for t in types if t != "null"] + type_name = remaining[0] if remaining else "string" + else: + nullable = bool(schema.get("nullable")) + type_name = types or "string" + + array = type_name == "array" + if array: + item = schema.get("items") or {} + type_name = item.get("type", "string") + + return Param( + name=name, + type=type_name, + default=schema.get("default"), + description=(description or schema.get("description") or "").strip(), + array=array, + nullable=nullable, + required=required, + choices=tuple(schema.get("enum") or ()), + ) + + +def operation_params(product: str, operation_id: str) -> list[Param]: + """Every parameter one operation accepts: query, then request body. + + Path parameters are excluded: they are the route, supplied by the command + from configuration, not by the caller as a flag. + """ + operation = find_operation(product, operation_id) + params = [ + _from_schema( + p["name"], + p.get("schema") or {}, + p.get("description", ""), + required=bool(p.get("required")), + ) + for p in operation.get("parameters", []) + if p.get("in") == "query" + ] + + body = operation.get("requestBody", {}).get("content", {}) + for media_type, content in body.items(): + # A binary body is the document itself, passed as an argument. + if media_type == "application/octet-stream": + continue + schema = content.get("schema") or {} + if ref := schema.get("$ref"): + schema = _resolve_ref(product, ref) + mandatory = set(schema.get("required") or ()) + for name, prop in (schema.get("properties") or {}).items(): + params.append( + _from_schema( + name, + prop, + prop.get("description", ""), + required=name in mandatory, + ) + ) + + return params + + +def client_params(method: Callable[..., Any]) -> dict[str, inspect.Parameter]: + """The parameters a client method accepts, by name. + + The published clients are frozen, so a spec parameter the client's signature + does not name cannot be reached at all: passing it raises ``TypeError`` + rather than sending it. Flags are intersected with this to keep the CLI's + surface equal to what actually works. + """ + return { + name: p + for name, p in inspect.signature(method).parameters.items() + if name not in ("self", "cls") + } + + +#: Python annotation -> OpenAPI type. A source-derived spec describes the wire, +#: which can differ from what the client method takes. +_ANNOTATIONS: dict[Any, str] = { + bool: "boolean", + int: "integer", + float: "number", + str: "string", +} + + +def _is_unset(value: Any) -> bool: + """Whether a default is a generated client's "absent" sentinel. + + Matched by name rather than by import: each client ships its own ``Unset`` + inside its generated tree, and that path is regenerated wholesale. + """ + return type(value).__name__ == "Unset" + + +def _from_signature(param: Param, signature: inspect.Parameter) -> Param: + """Reconcile a spec parameter with the client signature that will carry it.""" + updates: dict[str, Any] = {} + if (mapped := _ANNOTATIONS.get(signature.annotation)) is not None: + updates["type"] = mapped + if signature.default is inspect.Parameter.empty: + # No default in the signature means the call cannot omit it. + updates["required"] = True + elif not _is_unset(signature.default): + # What omitting the flag gets you: an `Unset` default sends nothing, so + # the spec's default is the one that applies. + updates["default"] = signature.default + return replace(param, **updates) + + +#: `name (type, optional): description` -- the Args entry of a Google-style +#: docstring, which is how both clients document their parameters. +_ARG_LINE = re.compile(r"^\s*(\w+)\s*(\([^)]*\))?\s*:\s*(.*)$") + +#: Sentences a description restates from elsewhere, stripped in the order they +#: appear. Each pattern ends at its own sentence, so prose after it survives. +_RESTATED = ( + re.compile(r"\s*Defaults to .*?\.(?=\s+[A-Z]|\s*$)"), + re.compile(r'\s*Can be ".*?"\s*\.'), +) + + +def docstring_params(method: Callable[..., Any]) -> dict[str, str]: + """Parameter descriptions from a client method's own docstring. + + The specs are generated from server code and carry no parameter + descriptions, while the published clients document every parameter. Reading + the docstring keeps one description per parameter, maintained where the + parameter is implemented, instead of a second copy here that goes stale + quietly. + """ + doc = inspect.getdoc(method) or "" + _, _, args = doc.partition("Args:") + if not args: + return {} + + out: dict[str, str] = {} + current: str | None = None + for line in args.splitlines(): + if not line.strip(): + continue + if line[:1] not in " \t" or re.match(r"^\s{0,4}(Returns|Raises|Yields):", line): + break + if (match := _ARG_LINE.match(line)) and (match.group(2) or current is None): + current = match.group(1) + out[current] = match.group(3).strip() + elif current: + out[current] = f"{out[current]} {line.strip()}".strip() + # Default and allowed values are rendered from the signature and the spec; + # the docstring's own copy of them would disagree as soon as either moves. + return {name: _strip_restated(text) for name, text in out.items() if text} + + +def _strip_restated(text: str) -> str: + text = " ".join(text.split()) + for pattern in _RESTATED: + text = pattern.sub("", text) + return text.strip() + + +def _resolve_ref(product: str, ref: str) -> dict[str, Any]: + node: Any = load_spec(product) + for part in ref.lstrip("#/").split("/"): + node = node[part] + return node + + +def _help_text(param: Param, choices: tuple[str, ...]) -> str: + """Help for one flag: what it does, what it accepts, what omitting it means. + + The default is reported but never applied. It answers "what happens if I + leave this out", which is the only question a default can honestly answer + here: the CLI does not resend it, the client or the server does. + """ + parts = [param.description] if param.description else [] + if choices: + parts.append(f"One of: {', '.join(choices)}.") + if param.default not in (None, "") and not param.required: + rendered = ( + str(param.default).lower() + if isinstance(param.default, bool) + else str(param.default) + ) + parts.append(f"[default: {rendered}]") + return " ".join(parts) + + +def click_option(param: Param, spec_overlay: dict[str, Any]) -> click.Option: + """Build one Click option from a spec parameter and its overlay entry.""" + entry = spec_overlay.get(param.name, {}) + # Falling back to the spec's own enum, so a hand-written list is needed only + # to narrow one on purpose -- a copy of it goes stale as the service grows. + choices = tuple(entry.get("choices", ())) or param.choices + help_text = entry.get("help") or _help_text(param, choices) + short = entry.get("short") + + if param.type == "boolean": + # A paired flag, not `is_flag`: a default-true parameter cannot be + # turned off by an on-only flag, and `None` keeps "not passed" apart + # from "passed false". + decls = [f"{param.flag}/--no-{param.name.replace('_', '-')}"] + if short: + decls.insert(0, short) + return click.Option(decls, default=None, required=param.required, help=help_text) + + decls = [param.flag] + if short: + decls.insert(0, short) + return click.Option( + decls, + type=click.Choice(choices) if choices else _TYPES.get(param.type, click.STRING), + default=None, + required=param.required, + multiple=param.array, + help=help_text, + ) + + +def derive_params( + product: str, + operation_id: str, + *, + client_method: Callable[..., Any] | None = None, + exclude: tuple[str, ...] = (), +) -> list[Param]: + """The parameters one command exposes, in spec order. + + With ``client_method``, the spec is intersected with what that method + accepts and the method's own defaults win, because that is the value the + caller gets by omitting the flag. A spec parameter the method does not name + is dropped rather than offered and then rejected at the call. + """ + spec_overlay = overlay_for(product, operation_id) + hidden = {name for name, entry in spec_overlay.items() if entry.get("hidden")} + accepted = client_params(client_method) if client_method is not None else None + described = docstring_params(client_method) if client_method is not None else {} + + out: list[Param] = [] + for param in operation_params(product, operation_id): + if param.name in exclude or param.name in hidden: + continue + if accepted is not None: + if param.name not in accepted: + continue + param = _from_signature(param, accepted[param.name]) + if not param.description and (text := described.get(param.name)): + param = replace(param, description=text) + out.append(param) + return out + + +def spec_options( + product: str, + operation_id: str, + *, + client_method: Callable[..., Any] | None = None, + exclude: tuple[str, ...] = (), +) -> Callable[[Any], Any]: + """Decorator: hang one operation's parameters off a command as options. + + ``exclude`` drops parameters the command supplies itself -- the document to + extract is an argument, not a flag, and the CLI owns the polling that + ``use_webhook`` would bypass. + + Applies either above or below ``@group.command()``: above it decorates a + built command, below it a bare function that Click has yet to build. + """ + spec_overlay = overlay_for(product, operation_id) + + def decorate(target: Any) -> Any: + options = [ + click_option(param, spec_overlay) + for param in derive_params( + product, operation_id, client_method=client_method, exclude=exclude + ) + ] + if isinstance(target, click.Command): + target.params.extend(options) + else: + # Click reads this list back in reverse, so the help lists the + # parameters in the order the spec declares them. + pending = getattr(target, "__click_params__", []) + target.__click_params__ = list(reversed(options)) + pending + return target + + return decorate + + +def requested(values: dict[str, Any], *, drop: tuple[str, ...] = ()) -> dict[str, Any]: + """Keep the parameters the caller actually passed. + + ``None`` is the only absence. An empty tuple from a repeatable option is one + too -- Click spells "not passed" that way for ``multiple=True`` -- but ``0``, + ``False`` and ``""`` are values the caller chose and must survive. + """ + return { + name: list(value) if isinstance(value, tuple) else value + for name, value in values.items() + if name not in drop and value is not None and value != () + } + + +__all__ = [ + "SPEC_FILES", + "Param", + "click_option", + "client_params", + "derive_params", + "docstring_params", + "find_operation", + "load_spec", + "operation_params", + "requested", + "spec_options", +] diff --git a/src/unstract_cli/core/poll.py b/src/unstract_cli/core/poll.py new file mode 100644 index 0000000..a4e9e2f --- /dev/null +++ b/src/unstract_cli/core/poll.py @@ -0,0 +1,272 @@ +"""`--wait` state machine and one-shot result persistence. + +Both products follow execute -> poll -> retrieve, and a caller should not have to +script that loop. + +**The load-bearing rule:** terminal state is decided by the ``status`` field in +the *response body*, never by the HTTP status code. The deployment API returns +HTTP 422 for the in-progress states, so reading the body means this behaves +identically before and after that is fixed server-side. + +The engine takes callables rather than owning any transport: the clients issue +every request, and the clock is injected so the whole thing tests offline. +""" + +from __future__ import annotations + +import json +import os +import time +from collections.abc import Callable +from contextlib import suppress +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from unstract_cli.core.errors import CLIError, ExitCode + + +@dataclass(frozen=True) +class PollSpec: + """How to read progress out of one operation's responses.""" + + #: Where the job handle lives in the initial response. Echoed back on + #: timeout so a caller can resume rather than reprocess the document. + handle_field: str + terminal_success: tuple[str, ...] + terminal_failure: tuple[str, ...] + #: One name, or candidates tried in order: the run POST and the status GET + #: spell the state differently. + status_field: str | tuple[str, ...] = "status" + + +def _dig(payload: Any, field: str) -> Any: + """Find a field, looking one level into the common envelopes.""" + if not isinstance(payload, dict): + return None + if field in payload: + return payload[field] + for envelope in ("message", "data", "result"): + inner = payload.get(envelope) + if isinstance(inner, dict) and field in inner: + return inner[field] + return None + + +def extract_status(payload: Any, field: str | tuple[str, ...] = "status") -> str | None: + """Read the status from a response body; first candidate that resolves wins.""" + fields = (field,) if isinstance(field, str) else field + for candidate in fields: + value = _dig(payload, candidate) + if value is not None: + return str(value) + return None + + +def extract_handle(payload: Any, field: str) -> str | None: + """Read the job handle out of a response body.""" + value = _dig(payload, field) + return str(value) if value is not None else None + + +def preflight(path: str | Path) -> Path: + """Prove the save target is writable, before anything destructive runs. + + `--save` exists to protect a read the server serves exactly once, so + discovering an unwritable path *after* that read is the one failure the + flag must not have. + """ + target = Path(path).expanduser() + try: + target.parent.mkdir(parents=True, exist_ok=True) + existed = target.exists() + with target.open("a", encoding="utf-8"): + pass + if not existed: + target.unlink() + except OSError as exc: + raise CLIError( + f"Cannot write to --save target {path!r}: {exc}.", + ExitCode.USAGE, + hint="Pick a writable path; nothing has been read yet, so nothing is lost.", + ) from exc + return target + + +def persist(path: str | Path, payload: Any) -> Path: + """Write a result to disk and return where it landed. + + Some results can be read exactly once. Callers must persist **before** the + read is acknowledged to the user, so a crash between the two cannot destroy + a result the server will not serve again. + + Written through a temporary file so a full disk leaves the previous copy + intact rather than a truncated one. A failure here raises with the payload + attached: by this point the only surviving copy is in memory, and it has to + reach stdout somehow. + """ + target = Path(path).expanduser() + text = ( + payload + if isinstance(payload, str) + else json.dumps(payload, indent=2, default=str) + ) + tmp = target.with_name(target.name + ".tmp") + try: + target.parent.mkdir(parents=True, exist_ok=True) + with tmp.open("w", encoding="utf-8") as handle: + handle.write(text) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp, target) + except OSError as exc: + with suppress(OSError): + tmp.unlink(missing_ok=True) + raise CLIError( + f"The result could not be written to {path!r}: {exc}.", + ExitCode.SAVE_FAILED, + details=payload, + hint=( + "`details` carries the result. It has already been read from the " + "service, which will not serve it again -- save it from here." + ), + ) from exc + return target + + +def classify(payload: Any, spec: PollSpec) -> str: + """`success`, `failure`, `pending` or `unknown` for one poll response. + + Shared with the standalone status commands: a finished-and-failed execution + is reported inside an HTTP 200, so a command that only checks the status + code calls it a success. + """ + status = (extract_status(payload, spec.status_field) or "").lower() + if status in {state.lower() for state in spec.terminal_failure}: + return "failure" + if status in {state.lower() for state in spec.terminal_success}: + return "success" + if not status or _dig(payload, "error"): + # Not progress: polling on regardless reports a server fault as "still + # running" until the deadline. + return "unknown" + return "pending" + + +def wait_for_completion( + *, + initial: Any, + spec: PollSpec, + poll: Callable[[str], Any], + retrieve: Callable[[str], Any] | None = None, + save: str | Path | None = None, + interval: float = 3.0, + timeout: float = 300.0, + on_status: Callable[[str | None], None] | None = None, + #: Called with the path once a result is on disk, before the caller sees + #: anything. The ordering it observes is the whole point of --save. + on_saved: Callable[[Path], None] | None = None, + sleep: Callable[[float], None] = time.sleep, + now: Callable[[], float] = time.monotonic, +) -> Any: + """Poll until terminal, then retrieve if the operation has a retrieve step. + + On timeout, raises with the job handle attached, so a caller can resume with + a plain status/retrieve call rather than resubmitting the document. + """ + handle = extract_handle(initial, spec.handle_field) + if not handle: + return initial + + deadline = now() + timeout + last_status: str | None = None + payload: Any = initial + + def naming_the_job(call: Callable[[str], Any]) -> Any: + """Run one step of the loop, ensuring any failure names the job. + + The handle is the difference between resuming and paying to process the + document a second time, so it is attached here rather than left to + whatever the caller wrapped the loop in. + """ + try: + return call(handle) + except CLIError as exc: + exc.extra.setdefault(spec.handle_field, handle) + raise + except Exception as exc: + raise CLIError( + str(exc) or type(exc).__name__, + ExitCode.SERVER_ERROR, + retryable=True, + extra={spec.handle_field: handle}, + ) from exc + + while True: + payload = naming_the_job(poll) + status = extract_status(payload, spec.status_field) + + if status != last_status: + if on_status is not None: + on_status(status) + last_status = status + + state = classify(payload, spec) + if state == "failure": + raise CLIError( + f"Operation finished with status {status!r}.", + ExitCode.VALIDATION, + details=payload, + hint="Inspect `details` for the per-file error, or check the execution logs.", + extra={spec.handle_field: handle}, + ) + if state == "unknown": + raise CLIError( + "The service answered with neither a status nor progress.", + ExitCode.SERVER_ERROR, + details=payload, + retryable=True, + hint=( + "The response carries no usable state, so polling on would " + "only repeat it. Retry with the handle below." + ), + extra={spec.handle_field: handle}, + ) + if state == "success": + break + + remaining = deadline - now() + if remaining <= 0: + raise CLIError( + f"Timed out after {timeout:g}s waiting for completion " + f"(last status: {status!r}).", + ExitCode.TIMEOUT, + hint=( + f"The job is still running. Resume with the {spec.handle_field} " + f"below rather than resubmitting the document." + ), + extra={spec.handle_field: handle, "last_status": status}, + ) + + # Never sleep past the deadline: --wait 30 that returns at 35s has lied, + # and the last poll should land on the deadline, not after it. + sleep(min(interval, remaining)) + + if retrieve is not None: + payload = naming_the_job(retrieve) + if save is not None: + written = persist(save, payload) + if on_saved is not None: + on_saved(written) + return payload + + +__all__ = [ + "PollSpec", + "classify", + "extract_handle", + "extract_status", + "persist", + "preflight", + "wait_for_completion", +] diff --git a/src/unstract_cli/overlay.toml b/src/unstract_cli/overlay.toml new file mode 100644 index 0000000..f563cac --- /dev/null +++ b/src/unstract_cli/overlay.toml @@ -0,0 +1,7 @@ +# Per-flag overrides for spec-derived options: [..]. +# +# Only what the spec cannot express belongs here. Names, types, defaults and +# allowed values are read from the spec, and help text falls back to the +# published client's own docstring, so an entry is needed only to add a short +# flag, narrow a value list on purpose, hide a parameter the CLI owns, or reword +# help the client states poorly. diff --git a/src/unstract_cli/specs/README.md b/src/unstract_cli/specs/README.md new file mode 100644 index 0000000..583b66f --- /dev/null +++ b/src/unstract_cli/specs/README.md @@ -0,0 +1,20 @@ +# Vendored API specs + +Copies of the specs the two published clients are generated from, kept here so +flags derive with no network and no assumption about where a client was +installed from. Each one is produced by the service that serves it, never edited +by hand: + +| file | source | +|---|---| +| `llmwhisperer.json` | `specs/llmwhisperer.json` in the LLMWhisperer service repo, generated by `tools/gen_spec.py` | +| `docstudio.json` | `specs/docstudio-oss.json` in the backend, generated by `manage.py generate_docstudio_spec` | + +Refresh one by copying it from the client commit pinned in `pyproject.toml`. +Refreshing it against a different commit is what `tests/test_contract.py` +guards: a spec parameter the pinned client has no argument for cannot become a +flag, and that test names the ones that already cannot. + +A refresh that changes which flags a command offers fails against +`tests/derived_flags.json`. Read the difference before refreshing that file -- +a flag missing from it is a flag the CLI has stopped offering. diff --git a/src/unstract_cli/specs/docstudio.json b/src/unstract_cli/specs/docstudio.json new file mode 100644 index 0000000..edf3196 --- /dev/null +++ b/src/unstract_cli/specs/docstudio.json @@ -0,0 +1,479 @@ +{ + "components": { + "schemas": { + "ErrorResponse": { + "properties": { + "message": { + "nullable": true + }, + "status": { + "type": "string" + } + }, + "type": "object" + }, + "ExecuteRequest": { + "description": "The documents to run, and the options that shape the result.\n\nSupply `files`, `presigned_urls`, or both.", + "properties": { + "custom_data": { + "nullable": true + }, + "files": { + "items": { + "format": "binary", + "type": "string" + }, + "type": "array" + }, + "hitl_packet_id": { + "nullable": true, + "type": "string" + }, + "hitl_queue_name": { + "nullable": true, + "type": "string" + }, + "include_extracted_text": { + "default": false, + "type": "boolean" + }, + "include_metadata": { + "default": false, + "type": "boolean" + }, + "include_metrics": { + "default": false, + "type": "boolean" + }, + "llm_profile_id": { + "nullable": true, + "type": "string" + }, + "presigned_urls": { + "items": { + "format": "uri", + "type": "string" + }, + "type": "array" + }, + "tags": { + "default": "", + "description": "Comma-separated list of tag names (EX:'tag1,tag2-name,tag3_name')", + "type": "string" + }, + "timeout": { + "default": -1, + "maximum": 300, + "minimum": -1, + "type": "integer" + }, + "use_file_history": { + "default": false, + "type": "boolean" + } + }, + "type": "object" + }, + "ExecuteResponse": { + "properties": { + "message": { + "$ref": "#/components/schemas/ExecutionMessage" + } + }, + "required": [ + "message" + ], + "type": "object" + }, + "ExecutionMessage": { + "description": "The execution's identity and, once it has finished, its per-file\nresults.", + "properties": { + "error": { + "type": "string" + }, + "execution_id": { + "type": "string" + }, + "execution_status": { + "type": "string" + }, + "result": { + "items": { + "$ref": "#/components/schemas/FileResult" + }, + "nullable": true, + "type": "array" + }, + "status_api": { + "type": "string" + } + }, + "required": [ + "error", + "execution_id", + "execution_status", + "status_api" + ], + "type": "object" + }, + "FileResult": { + "properties": { + "error": { + "nullable": true, + "type": "string" + }, + "file": { + "type": "string" + }, + "file_execution_id": { + "type": "string" + }, + "metadata": {}, + "metrics": {}, + "result": {}, + "status": { + "type": "string" + } + }, + "required": [ + "file" + ], + "type": "object" + }, + "StatusResponse": { + "properties": { + "message": { + "items": { + "$ref": "#/components/schemas/FileResult" + }, + "nullable": true, + "type": "array" + }, + "status": { + "type": "string" + } + }, + "required": [ + "status" + ], + "type": "object" + } + }, + "securitySchemes": { + "deploymentKey": { + "description": "The API deployment's own key.", + "scheme": "bearer", + "type": "http" + } + } + }, + "info": { + "title": "Unstract API", + "version": "v1" + }, + "openapi": "3.0.3", + "paths": { + "/deployment/api/{org_name}/{api_name}/": { + "get": { + "description": "Read the result of a previously started execution.\n\nThis read is one-shot: the first call that observes a completed execution acknowledges it and the stored result is discarded, so every later call for that execution answers 406. Poll while the execution is pending, and keep the payload of the call that returns it \u2014 it cannot be fetched again.", + "operationId": "status", + "parameters": [ + { + "description": "API deployment name.", + "in": "path", + "name": "api_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + }, + { + "in": "query", + "name": "execution_id", + "required": true, + "schema": { + "minLength": 1, + "type": "string" + } + }, + { + "in": "query", + "name": "include_extracted_text", + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "include_metadata", + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "include_metrics", + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "description": "Organization identifier.", + "in": "path", + "name": "org_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StatusResponse" + } + } + }, + "description": "" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The request failed validation." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The API key is not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No API key was supplied." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No such active deployment." + }, + "406": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The result was already consumed by an earlier call." + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StatusResponse" + } + } + }, + "description": "" + }, + "429": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Too many concurrent executions; retry later." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "" + } + }, + "security": [ + { + "deploymentKey": [] + } + ], + "tags": [ + "deployment" + ] + }, + "post": { + "description": "Execute an API deployment against one or more documents.\n\nSupply the documents either as `files` (multipart upload) or as `presigned_urls` (HTTPS S3 URLs), or both \u2014 a request carrying neither is rejected, and the two together may not exceed 32 documents.\n\nWith the default `timeout` of -1 the call returns as soon as the execution is queued; read the outcome from the status endpoint.", + "operationId": "execute", + "parameters": [ + { + "description": "API deployment name.", + "in": "path", + "name": "api_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + }, + { + "description": "Organization identifier.", + "in": "path", + "name": "org_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/ExecuteRequest" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExecuteResponse" + } + } + }, + "description": "" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The request failed validation." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The API key is not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No API key was supplied." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No such active deployment." + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The deployment has no active API key." + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExecuteResponse" + } + } + }, + "description": "" + }, + "429": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Too many concurrent executions; retry later." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "" + } + }, + "security": [ + { + "deploymentKey": [] + } + ], + "tags": [ + "deployment" + ] + } + } + }, + "tags": [ + { + "description": "Run an API deployment against one or more documents and poll the result.", + "name": "deployment" + } + ] +} diff --git a/src/unstract_cli/specs/llmwhisperer.json b/src/unstract_cli/specs/llmwhisperer.json new file mode 100644 index 0000000..8cc8dfe --- /dev/null +++ b/src/unstract_cli/specs/llmwhisperer.json @@ -0,0 +1,1977 @@ +{ + "components": { + "schemas": { + "Error": { + "properties": { + "message": { + "type": "string" + } + }, + "type": "object" + }, + "WebhookConfig": { + "properties": { + "auth_token": { + "type": "string" + }, + "url": { + "format": "uri", + "type": "string" + }, + "webhook_name": { + "type": "string" + } + }, + "required": [ + "url", + "auth_token", + "webhook_name" + ], + "type": "object" + }, + "WhisperAccepted": { + "properties": { + "message": { + "type": "string" + }, + "status": { + "type": "string" + }, + "whisper_hash": { + "type": "string" + } + }, + "type": "object" + }, + "WhisperResult": { + "properties": { + "confidence_metadata": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + "line_metadata": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + "metadata": { + "additionalProperties": true, + "type": "object" + }, + "result_text": { + "type": "string" + }, + "webhook_metadata": { + "type": "string" + }, + "whisper_metadata": { + "additionalProperties": true, + "type": "object" + } + }, + "type": "object" + }, + "WhisperStatus": { + "properties": { + "detail": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + "message": { + "type": "string" + }, + "status": { + "type": "string" + } + }, + "type": "object" + } + }, + "securitySchemes": { + "unstract_key": { + "in": "header", + "name": "unstract-key", + "type": "apiKey" + } + } + }, + "info": { + "description": "The hosted regions are listed under `servers`; a self-hosted deployment serves the same API from its own URL, which every client takes as a configuration option.", + "title": "Unstract LLMWhisperer", + "version": "v2" + }, + "openapi": "3.0.3", + "paths": { + "/api/v2/convert-to-pdf": { + "post": { + "operationId": "convert_to_pdf", + "parameters": [ + { + "in": "query", + "name": "mode", + "required": false, + "schema": { + "default": "form", + "type": "string" + } + }, + { + "in": "query", + "name": "url", + "required": false, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "url_in_post", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + } + ], + "requestBody": { + "content": { + "application/octet-stream": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "required": false + }, + "responses": { + "200": { + "content": { + "application/pdf": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + } + }, + "summary": "Convert a document to PDF", + "tags": [ + "convert" + ] + } + }, + "/api/v2/convert-xlsb-to-xlsx": { + "post": { + "operationId": "convert_xlsb_to_xlsx", + "parameters": [ + { + "in": "query", + "name": "mode", + "required": false, + "schema": { + "default": "form", + "type": "string" + } + }, + { + "in": "query", + "name": "url", + "required": false, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "url_in_post", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + } + ], + "requestBody": { + "content": { + "application/octet-stream": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "required": false + }, + "responses": { + "200": { + "content": { + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + } + }, + "summary": "Convert an XLSB workbook to XLSX", + "tags": [ + "convert" + ] + } + }, + "/api/v2/document-insights": { + "post": { + "operationId": "document_insights", + "parameters": [ + { + "in": "query", + "name": "file_name", + "required": false, + "schema": { + "default": "sample.pdf", + "type": "string" + } + }, + { + "in": "query", + "name": "mode", + "required": false, + "schema": { + "default": "form", + "type": "string" + } + }, + { + "in": "query", + "name": "pages_to_extract", + "required": false, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "tag", + "required": false, + "schema": { + "default": "default", + "type": "string" + } + }, + { + "in": "query", + "name": "url", + "required": false, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "url_in_post", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "use_webhook", + "required": false, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "webhook_metadata", + "required": false, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/octet-stream": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "required": false + }, + "responses": { + "202": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WhisperAccepted" + } + } + }, + "description": "Accepted" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + } + }, + "summary": "Run document insights over a file", + "tags": [ + "insights" + ] + } + }, + "/api/v2/document-insights-retrieve": { + "get": { + "operationId": "document_insights_retrieve", + "parameters": [ + { + "in": "query", + "name": "whisper_hash", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + } + }, + "summary": "Retrieve document insights result (destructive \u2014 one shot)", + "tags": [ + "insights" + ] + } + }, + "/api/v2/get-usage-info": { + "get": { + "operationId": "usage_info", + "parameters": [], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + } + }, + "summary": "Subscription usage summary", + "tags": [ + "account" + ] + } + }, + "/api/v2/highlights": { + "get": { + "operationId": "highlights", + "parameters": [ + { + "in": "query", + "name": "extract_all_lines", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "description": "Line numbers or ranges, e.g. `1-5,9`. Not required when `extract_all_lines=true`.", + "in": "query", + "name": "lines", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "mode", + "required": false, + "schema": { + "default": "form", + "type": "string" + } + }, + { + "in": "query", + "name": "whisper_hash", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + } + }, + "summary": "Line-level highlight geometry for an extraction", + "tags": [ + "whisper" + ] + } + }, + "/api/v2/pdf-to-images": { + "post": { + "operationId": "pdf_to_images", + "parameters": [ + { + "in": "query", + "name": "file_name", + "required": false, + "schema": { + "default": "sample.pdf", + "type": "string" + } + }, + { + "in": "query", + "name": "format", + "required": false, + "schema": { + "default": "png", + "type": "string" + } + }, + { + "in": "query", + "name": "mode", + "required": false, + "schema": { + "default": "form", + "type": "string" + } + }, + { + "in": "query", + "name": "tag", + "required": false, + "schema": { + "default": "default", + "type": "string" + } + }, + { + "in": "query", + "name": "url", + "required": false, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "url_in_post", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + } + ], + "requestBody": { + "content": { + "application/octet-stream": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "required": false + }, + "responses": { + "202": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WhisperAccepted" + } + } + }, + "description": "Accepted" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + } + }, + "summary": "Render a PDF's pages as images", + "tags": [ + "convert" + ] + } + }, + "/api/v2/pdf-to-images-retrieve": { + "get": { + "operationId": "pdf_to_images_retrieve", + "parameters": [ + { + "in": "query", + "name": "whisper_hash", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/zip": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + } + }, + "summary": "Retrieve rendered images as a zip (destructive \u2014 one shot)", + "tags": [ + "convert" + ] + } + }, + "/api/v2/pdf-to-images-status": { + "get": { + "operationId": "pdf_to_images_status", + "parameters": [ + { + "in": "query", + "name": "whisper_hash", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + } + }, + "summary": "Poll PDF-to-images status", + "tags": [ + "convert" + ] + } + }, + "/api/v2/test-connection": { + "get": { + "operationId": "test_connection", + "parameters": [], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + } + }, + "summary": "Verify credentials", + "tags": [ + "account" + ] + } + }, + "/api/v2/usage": { + "get": { + "operationId": "usage", + "parameters": [ + { + "in": "query", + "name": "from_date", + "required": false, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "tag", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "to_date", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + } + }, + "summary": "Detailed usage statistics", + "tags": [ + "account" + ] + } + }, + "/api/v2/whisper": { + "post": { + "operationId": "extract", + "parameters": [ + { + "in": "query", + "name": "add_line_nos", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "allow_rotated_text", + "required": false, + "schema": { + "default": true, + "type": "boolean" + } + }, + { + "in": "query", + "name": "checkbox_confidence_threshold", + "required": false, + "schema": { + "default": 0.3, + "type": "number" + } + }, + { + "in": "query", + "name": "derotate_threshold", + "required": false, + "schema": { + "default": 10.0, + "type": "number" + } + }, + { + "in": "query", + "name": "file_name", + "required": false, + "schema": { + "default": "sample.pdf", + "type": "string" + } + }, + { + "in": "query", + "name": "gaussian_blur_radius", + "required": false, + "schema": { + "default": 0, + "type": "number" + } + }, + { + "in": "query", + "name": "horizontal_stretch_factor", + "required": false, + "schema": { + "default": 1.0, + "type": "number" + } + }, + { + "in": "query", + "name": "ignore_vertical_text", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "include_line_confidence", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "lang", + "required": false, + "schema": { + "default": "eng", + "type": "string" + } + }, + { + "in": "query", + "name": "line_splitter_strategy", + "required": false, + "schema": { + "default": "left-priority", + "enum": [ + "left-priority", + "mid-priority", + "right-priority" + ], + "type": "string" + } + }, + { + "in": "query", + "name": "line_splitter_tolerance", + "required": false, + "schema": { + "default": 0.75, + "type": "number" + } + }, + { + "in": "query", + "name": "mark_horizontal_lines", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "mark_vertical_lines", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "median_filter_size", + "required": false, + "schema": { + "default": 0, + "type": "integer" + } + }, + { + "in": "query", + "name": "min_table_width", + "required": false, + "schema": { + "default": 0.0, + "type": "number" + } + }, + { + "in": "query", + "name": "mode", + "required": false, + "schema": { + "default": "form", + "enum": [ + "document_insights", + "excel", + "form", + "high_quality", + "low_cost", + "native_text", + "pdf_to_images", + "table" + ], + "type": "string" + } + }, + { + "in": "query", + "name": "output_mode", + "required": false, + "schema": { + "default": "layout_preserving", + "enum": [ + "dump-text", + "layout_preserving", + "line-printer", + "text" + ], + "type": "string" + } + }, + { + "in": "query", + "name": "page_separator", + "required": false, + "schema": { + "default": "<<<", + "type": "string" + } + }, + { + "in": "query", + "name": "pages_to_extract", + "required": false, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "tag", + "required": false, + "schema": { + "default": "default", + "type": "string" + } + }, + { + "description": "Fetch the document from this URL instead of sending a body.", + "in": "query", + "name": "url", + "required": false, + "schema": { + "format": "uri", + "type": "string" + } + }, + { + "description": "Read the URL to fetch from the request body.", + "in": "query", + "name": "url_in_post", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "use_webhook", + "required": false, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "watermark_angle_threshold", + "required": false, + "schema": { + "default": 25.0, + "type": "number" + } + }, + { + "in": "query", + "name": "webhook_metadata", + "required": false, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "word_confidence_threshold", + "required": false, + "schema": { + "type": "number" + } + } + ], + "requestBody": { + "content": { + "application/octet-stream": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "required": false + }, + "responses": { + "202": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WhisperAccepted" + } + } + }, + "description": "Accepted" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + } + }, + "summary": "Submit a document for text extraction", + "tags": [ + "whisper" + ] + } + }, + "/api/v2/whisper-detail": { + "get": { + "operationId": "detail", + "parameters": [ + { + "in": "query", + "name": "whisper_hash", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + } + }, + "summary": "Metadata about a whisper job", + "tags": [ + "whisper" + ] + } + }, + "/api/v2/whisper-manage-callback": { + "delete": { + "operationId": "webhook_delete", + "parameters": [ + { + "in": "query", + "name": "webhook_name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + } + }, + "summary": "Manage extraction webhooks", + "tags": [ + "webhook" + ] + }, + "get": { + "operationId": "webhook_get", + "parameters": [ + { + "in": "query", + "name": "webhook_name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + } + }, + "summary": "Manage extraction webhooks", + "tags": [ + "webhook" + ] + }, + "post": { + "operationId": "webhook_post", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookConfig" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "Created" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + } + }, + "summary": "Manage extraction webhooks", + "tags": [ + "webhook" + ] + }, + "put": { + "operationId": "webhook_put", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhookConfig" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "additionalProperties": true, + "type": "object" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + } + }, + "summary": "Manage extraction webhooks", + "tags": [ + "webhook" + ] + } + }, + "/api/v2/whisper-retrieve": { + "get": { + "operationId": "retrieve", + "parameters": [ + { + "in": "query", + "name": "text_only", + "required": false, + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "whisper_hash", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WhisperResult" + } + }, + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + } + }, + "summary": "Retrieve extraction result (destructive \u2014 one shot)", + "tags": [ + "whisper" + ] + } + }, + "/api/v2/whisper-status": { + "get": { + "operationId": "status", + "parameters": [ + { + "in": "query", + "name": "whisper_hash", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WhisperStatus" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The request was rejected -- see `message`." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The API key is missing or not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "The key is valid but not entitled to this operation." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + }, + "description": "No such resource." + } + }, + "summary": "Poll extraction status", + "tags": [ + "whisper" + ] + } + } + }, + "security": [ + { + "unstract_key": [] + } + ], + "servers": [ + { + "description": "US region (the default of the published clients).", + "url": "https://llmwhisperer-api.us-central.unstract.com" + }, + { + "description": "EU region.", + "url": "https://llmwhisperer-api.eu-west.unstract.com" + } + ] +} diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..5d33b03 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import os +from fnmatch import fnmatch + +import pytest + +from unstract_cli import config as config_mod +from unstract_cli.core.output import AGENT_ENV + +#: Every variable the loader consults. Cleared per test so a developer's real +#: shell environment cannot change a result. +_ENV_VARS = sorted( + {var for vars_ in config_mod.ENV_VARS.values() for var in vars_} + | {"UNSTRACT_CONFIG", "UNSTRACT_PROFILE"} +) + + +@pytest.fixture(autouse=True) +def clean_env(monkeypatch, tmp_path): + for var in _ENV_VARS: + monkeypatch.delenv(var, raising=False) + # These decide the default output format, and this suite is as likely to be + # run by an agent as by a person. + for var in [ + name + for name in os.environ + if any(fnmatch(name, pattern) for pattern in AGENT_ENV) + ]: + monkeypatch.delenv(var, raising=False) + config_mod.set_config_path(None) + # Both discovery fallbacks are redirected into the tmp dir: an upward search + # from a real cwd could otherwise find a developer's own .unstract.toml. + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(config_mod, "HOME_CONFIG", tmp_path / "home" / "config.toml") + yield + config_mod.set_config_path(None) + + +@pytest.fixture +def write_config(tmp_path, monkeypatch): + """Write a config file and point the CLI at it.""" + + def _write(text: str): + path = tmp_path / "config.toml" + path.write_text(text, encoding="utf-8") + monkeypatch.setenv("UNSTRACT_CONFIG", str(path)) + return path + + return _write diff --git a/tests/derived_flags.json b/tests/derived_flags.json new file mode 100644 index 0000000..bca07c9 --- /dev/null +++ b/tests/derived_flags.json @@ -0,0 +1,458 @@ +{ + "llmwhisperer:extract": { + "--add-line-nos": { + "name": "add_line_nos", + "type": "boolean", + "default": false, + "description": "Adds line numbers to the extracted text and saves line metadata, which can be queried later using the highlights API.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--allow-rotated-text": { + "name": "allow_rotated_text", + "type": "boolean", + "default": true, + "description": "Whether to keep words whose own orientation is rotated. With this off, a word angled further than watermark_angle_threshold is treated as a watermark and excluded.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--checkbox-confidence-threshold": { + "name": "checkbox_confidence_threshold", + "type": "number", + "default": 0.3, + "description": "The minimum confidence a detected checkbox mark must have to be reported as marked. Accepts a value in the range [0.0, 1.0].", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--derotate-threshold": { + "name": "derotate_threshold", + "type": "number", + "default": 10.0, + "description": "The page rotation in degrees beyond which the page is straightened and re-read.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--file-name": { + "name": "file_name", + "type": "string", + "default": null, + "description": "The name of the file to store in reports.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--gaussian-blur-radius": { + "name": "gaussian_blur_radius", + "type": "integer", + "default": 0, + "description": "The radius of the Gaussian blur.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--horizontal-stretch-factor": { + "name": "horizontal_stretch_factor", + "type": "number", + "default": 1.0, + "description": "The horizontal stretch factor.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--ignore-vertical-text": { + "name": "ignore_vertical_text", + "type": "boolean", + "default": false, + "description": "Whether to drop vertically oriented text instead of extracting it.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--include-line-confidence": { + "name": "include_line_confidence", + "type": "boolean", + "default": false, + "description": "Adds line confidence to the line metadata returned by the highlights API. Requires add_line_nos to be enabled.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--lang": { + "name": "lang", + "type": "string", + "default": "eng", + "description": "The language of the document.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--line-splitter-strategy": { + "name": "line_splitter_strategy", + "type": "string", + "default": null, + "description": "The line splitter strategy.", + "array": false, + "nullable": false, + "required": false, + "choices": [ + "left-priority", + "mid-priority", + "right-priority" + ] + }, + "--line-splitter-tolerance": { + "name": "line_splitter_tolerance", + "type": "number", + "default": 0.4, + "description": "The line splitter tolerance.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--mark-horizontal-lines": { + "name": "mark_horizontal_lines", + "type": "boolean", + "default": false, + "description": "Whether to mark horizontal lines.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--mark-vertical-lines": { + "name": "mark_vertical_lines", + "type": "boolean", + "default": false, + "description": "Whether to mark vertical lines.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--median-filter-size": { + "name": "median_filter_size", + "type": "integer", + "default": 0, + "description": "The size of the median filter.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--min-table-width": { + "name": "min_table_width", + "type": "number", + "default": 0.0, + "description": "The minimum width a table must span, as a fraction of the page width, to be extracted as a table.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--mode": { + "name": "mode", + "type": "string", + "default": "form", + "description": "The processing mode.", + "array": false, + "nullable": false, + "required": false, + "choices": [ + "document_insights", + "excel", + "form", + "high_quality", + "low_cost", + "native_text", + "pdf_to_images", + "table" + ] + }, + "--output-mode": { + "name": "output_mode", + "type": "string", + "default": "layout_preserving", + "description": "The output mode.", + "array": false, + "nullable": false, + "required": false, + "choices": [ + "dump-text", + "layout_preserving", + "line-printer", + "text" + ] + }, + "--page-separator": { + "name": "page_separator", + "type": "string", + "default": null, + "description": "The page separator.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--pages-to-extract": { + "name": "pages_to_extract", + "type": "string", + "default": "", + "description": "The pages to extract.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--tag": { + "name": "tag", + "type": "string", + "default": "default", + "description": "The tag for the document.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--url": { + "name": "url", + "type": "string", + "default": "", + "description": "Fetch the document from this URL instead of sending a body.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--use-webhook": { + "name": "use_webhook", + "type": "string", + "default": "", + "description": "Webhook name to call. If not provided, then no webhook will be called.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--watermark-angle-threshold": { + "name": "watermark_angle_threshold", + "type": "number", + "default": 25.0, + "description": "The angle in degrees beyond which a rotated word counts as a watermark. Only applies when allow_rotated_text is off.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--webhook-metadata": { + "name": "webhook_metadata", + "type": "string", + "default": "", + "description": "The webhook metadata. This data will be passed to the webhook if webhooks are used", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--word-confidence-threshold": { + "name": "word_confidence_threshold", + "type": "number", + "default": 0.3, + "description": "The minimum OCR confidence score a word must have to be included in the extracted text. Accepts a value in the range [0.0, 1.0], where higher values are stricter. Any word whose confidence value falls below the configured threshold is ignored and excluded from the final output. This parameter works only with \"form\", \"high_quality\" and \"table\" modes.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + } + }, + "llmwhisperer:highlights": { + "--extract-all-lines": { + "name": "extract_all_lines", + "type": "boolean", + "default": false, + "description": "", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--lines": { + "name": "lines", + "type": "string", + "default": null, + "description": "Line numbers or ranges, e.g. `1-5,9`. Not required when `extract_all_lines=true`.", + "array": false, + "nullable": false, + "required": true, + "choices": [] + }, + "--whisper-hash": { + "name": "whisper_hash", + "type": "string", + "default": null, + "description": "The hash of the whisper operation.", + "array": false, + "nullable": false, + "required": true, + "choices": [] + } + }, + "docstudio:execute": { + "--custom-data": { + "name": "custom_data", + "type": "string", + "default": null, + "description": "Arbitrary data echoed back with the result.", + "array": false, + "nullable": true, + "required": false, + "choices": [] + }, + "--hitl-packet-id": { + "name": "hitl_packet_id", + "type": "string", + "default": null, + "description": "Human-in-the-loop packet to attach the file to.", + "array": false, + "nullable": true, + "required": false, + "choices": [] + }, + "--hitl-queue-name": { + "name": "hitl_queue_name", + "type": "string", + "default": null, + "description": "Human-in-the-loop queue to route the file to.", + "array": false, + "nullable": true, + "required": false, + "choices": [] + }, + "--include-extracted-text": { + "name": "include_extracted_text", + "type": "boolean", + "default": false, + "description": "Include the extracted text.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--include-metadata": { + "name": "include_metadata", + "type": "boolean", + "default": false, + "description": "Include metadata in the result.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--include-metrics": { + "name": "include_metrics", + "type": "boolean", + "default": false, + "description": "Include metrics in the result.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--llm-profile-id": { + "name": "llm_profile_id", + "type": "string", + "default": null, + "description": "LLM profile to override the deployment's.", + "array": false, + "nullable": true, + "required": false, + "choices": [] + }, + "--presigned-urls": { + "name": "presigned_urls", + "type": "string", + "default": null, + "description": "URLs to fetch the inputs from.", + "array": true, + "nullable": false, + "required": false, + "choices": [] + }, + "--tags": { + "name": "tags", + "type": "string", + "default": "", + "description": "Comma-separated list of tag names (EX:'tag1,tag2-name,tag3_name')", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--timeout": { + "name": "timeout", + "type": "integer", + "default": -1, + "description": "Execution mode \u2014 ``0`` or below runs asynchronously.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--use-file-history": { + "name": "use_file_history", + "type": "boolean", + "default": false, + "description": "Reuse a previous result for the same file.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + } + }, + "docstudio:status": { + "--include-extracted-text": { + "name": "include_extracted_text", + "type": "boolean", + "default": false, + "description": "Include the extracted text.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--include-metadata": { + "name": "include_metadata", + "type": "boolean", + "default": false, + "description": "Include metadata in the result.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + }, + "--include-metrics": { + "name": "include_metrics", + "type": "boolean", + "default": false, + "description": "Include metrics in the result.", + "array": false, + "nullable": false, + "required": false, + "choices": [] + } + } +} diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..7184189 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,203 @@ +"""End-to-end through the entry point: exit codes reach the shell, stdout parses.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from unstract_cli import app +from unstract_cli.__main__ import main +from unstract_cli.app import cli, command_tree +from unstract_cli.core.errors import ExitCode + + +def run(capsys, *args): + """Invoke the CLI as the console script does, returning (code, stdout json). + + `-o json` is passed the way any consumer has to pass it: the default format + is human-facing, and a test that relied on it would be pinning the wrong + thing. + """ + code = main(["-o", "json", *args]) + captured = capsys.readouterr() + payload = json.loads(captured.out) if captured.out.strip() else None + return code, payload, captured.err + + +def test_v1_groups_are_registered(): + tree = command_tree() + assert set(tree) >= {"config", "whisper", "docstudio"} + assert "deployment" in tree["docstudio"]["commands"] + assert set(tree["config"]["commands"]) == {"doctor", "get", "init", "list", "set"} + + +def test_help_exits_zero(capsys): + assert main(["--help"]) == int(ExitCode.SUCCESS) + + +def test_unknown_command_is_a_usage_error_with_an_envelope(capsys): + code, payload, err = run(capsys, "nope") + assert code == int(ExitCode.USAGE) + assert payload["ok"] is False + assert payload["error"]["exit_code"] == int(ExitCode.USAGE) + assert err.startswith("error:") + + +def test_an_interrupt_exits_one_thirty_with_an_envelope(capsys, monkeypatch): + """Ctrl-C is not a failure of the command. Reporting it as a generic error + tells a supervisor to retry what the user deliberately stopped.""" + + def interrupted(): + raise KeyboardInterrupt + + monkeypatch.setattr("unstract_cli.commands.config_cmd.load_config", interrupted) + + code, payload, _ = run(capsys, "config", "doctor") + + assert code == int(ExitCode.INTERRUPTED) == 130 + assert payload["ok"] is False + assert payload["error"]["code"] == "interrupted" + + +def test_unknown_config_target_exits_two(capsys): + code, payload, _ = run(capsys, "config", "get", "nosuchproduct", "base_url") + assert code == int(ExitCode.USAGE) + assert "llmwhisperer" in payload["error"]["hint"] + + +def test_set_then_get_round_trip(capsys, tmp_path, monkeypatch): + monkeypatch.setenv("UNSTRACT_CONFIG", str(tmp_path / "c.toml")) + + code, payload, _ = run(capsys, "config", "set", "docstudio", "org_id", "org_A") + assert code == 0 and payload["ok"] is True + + code, payload, _ = run(capsys, "config", "get", "docstudio", "org_id") + assert code == 0 + assert payload["data"]["value"] == "org_A" + + +def test_set_warns_when_a_credential_is_stored_literally(capsys, tmp_path, monkeypatch): + monkeypatch.setenv("UNSTRACT_CONFIG", str(tmp_path / "c.toml")) + _, payload, _ = run(capsys, "config", "set", "llmwhisperer", "api_key", "literal-key") + assert "env:VAR_NAME" in payload["data"]["warning"] + + +def test_get_never_echoes_a_credential(capsys, tmp_path, monkeypatch): + monkeypatch.setenv("UNSTRACT_CONFIG", str(tmp_path / "c.toml")) + run(capsys, "config", "set", "llmwhisperer", "api_key", "super-secret-value") + _, payload, _ = run(capsys, "config", "get", "llmwhisperer", "api_key") + assert payload["data"]["value"] == "***SET***" + assert "super-secret-value" not in json.dumps(payload) + + +def test_init_refuses_to_clobber_without_force(capsys, tmp_path, monkeypatch): + monkeypatch.setenv("UNSTRACT_CONFIG", str(tmp_path / "c.toml")) + assert run(capsys, "config", "init")[0] == 0 + + code, payload, _ = run(capsys, "config", "init") + assert code == int(ExitCode.USAGE) + assert "--force" in payload["error"]["hint"] + + assert run(capsys, "config", "init", "--force")[0] == 0 + + +def test_doctor_reports_sources_without_leaking_values(capsys, monkeypatch): + monkeypatch.setenv("LLMWHISPERER_API_KEY", "super-secret-value") + code, payload, _ = run(capsys, "config", "doctor") + assert code == 0 + products = payload["data"]["products"] + assert products["llmwhisperer"]["api_key"] == { + "resolved": True, + "source": "env:LLMWHISPERER_API_KEY", + } + assert products["docstudio"]["api_key"]["resolved"] is False + assert "super-secret-value" not in json.dumps(payload) + + +def doctor(capsys, *args) -> str: + """`config doctor` -- a command with no network -- and its raw stdout.""" + main([*args, "config", "doctor"]) + return capsys.readouterr().out + + +def is_table(out: str) -> bool: + try: + json.loads(out) + except json.JSONDecodeError: + return "active_profile" in out + return False + + +class TestOutputFormatEndToEnd: + """One rule: `-o` decides, and where it is absent the environment picks the + default only. Everything here is a way of getting that wrong.""" + + def test_the_default_is_a_table_in_a_terminal_and_in_a_pipe( + self, capsys, monkeypatch + ): + monkeypatch.setattr("sys.stdout.isatty", lambda: True, raising=False) + assert is_table(doctor(capsys)) + monkeypatch.setattr("sys.stdout.isatty", lambda: False, raising=False) + assert is_table(doctor(capsys)) + + def test_no_isatty_call_decides_a_format(self): + """A format that depends on a terminal makes a script's output depend on + how it was launched.""" + source = Path(app.__file__).parent + offenders = [ + path.name + for path in source.rglob("*.py") + if "isatty" in path.read_text(encoding="utf-8") + ] + assert offenders == [] + + def test_an_agent_environment_makes_json_the_default(self, capsys, monkeypatch): + monkeypatch.setenv("CLAUDECODE", "1") + assert json.loads(doctor(capsys))["ok"] is True + + def test_an_explicit_format_wins_over_a_detected_agent(self, capsys, monkeypatch): + monkeypatch.setenv("CLAUDECODE", "1") + assert is_table(doctor(capsys, "-o", "table")) + + def test_agent_no_forces_the_human_default(self, capsys, monkeypatch): + monkeypatch.setenv("CLAUDECODE", "1") + assert is_table(doctor(capsys, "--agent", "no")) + + def test_json_is_byte_identical_however_it_was_asked_for(self, capsys, monkeypatch): + monkeypatch.setattr("sys.stdout.isatty", lambda: True, raising=False) + on_a_tty = doctor(capsys, "-o", "json") + + monkeypatch.setattr("sys.stdout.isatty", lambda: False, raising=False) + monkeypatch.setenv("CLAUDECODE", "1") + piped_under_an_agent = doctor(capsys, "-o", "json") + + assert on_a_tty == piped_under_an_agent + + def test_every_envelope_carries_the_contract_version(self, capsys): + assert run(capsys, "config", "doctor")[1]["meta"]["contract_version"] == 1 + assert run(capsys, "nope")[1]["meta"]["contract_version"] == 1 + + +def test_the_config_group_says_what_it_withheld(capsys, tmp_path, monkeypatch): + """`config list` is one of the commands run *to understand* the config. + + It loads the file itself rather than through the root context, so it has to + report the file's warnings on its own or stay silent about its own subject. + """ + work = tmp_path / "checkout" + work.mkdir() + (work / ".unstract.toml").write_text( + '[profiles.p.llmwhisperer]\napi_key = "planted"\n', encoding="utf-8" + ) + monkeypatch.chdir(work) + + _, _, err = run(capsys, "config", "list") + assert err.count("Ignoring p.llmwhisperer.api_key") == 1 + + +def test_click_parameter_info_dict_keeps_the_keys_discovery_reads(): + # Discovery derives flags from Click's own introspection; a Click bump that + # reshaped this dict would silently degrade it. + param = next(p for p in cli.params if p.name == "output") + info = param.to_info_dict() + assert {"name", "opts", "help", "type", "required"} <= set(info) diff --git a/tests/test_commands.py b/tests/test_commands.py new file mode 100644 index 0000000..6d55531 --- /dev/null +++ b/tests/test_commands.py @@ -0,0 +1,1038 @@ +"""The product commands, with the clients replaced. No network. + +The seam is the client factory, not the transport: what matters here is which +arguments a command hands the client, what it does with the reply, and what a +caller sees on stdout and in the exit code. +""" + +from __future__ import annotations + +import json + +import pytest +from requests.exceptions import ConnectionError +from unstract.clone.report import CloneReport, Endpoint, PhaseResult +from unstract.llmwhisperer.client_v2 import ( + LLMWhispererClientException, + LLMWhispererClientV2, +) + +from unstract_cli.__main__ import main +from unstract_cli.app import command_tree +from unstract_cli.commands import clone_cmd, docstudio_cmd, whisper_cmd +from unstract_cli.config import LLMWHISPERER +from unstract_cli.core.errors import CLIError, ExitCode + + +def run(capsys, *args): + """Invoke the CLI as the console script does, returning (code, stdout, stderr). + + `-o json` explicitly: these assert on the parseable output, which is what a + caller opts into rather than what an unflagged run happens to print. + """ + code = main(["-o", "json", *args]) + captured = capsys.readouterr() + return code, captured.out, captured.err + + +def envelope(out: str) -> dict: + return json.loads(out) + + +class FakeWhisper: + """Records calls; returns whatever the test queued.""" + + def __init__(self, **replies): + self.replies = replies + self.calls: list[tuple[str, tuple, dict]] = [] + + def _reply(self, name, *args, **kwargs): + self.calls.append((name, args, kwargs)) + reply = self.replies.get(name) + if isinstance(reply, Exception): + raise reply + if isinstance(reply, list): + return reply.pop(0) if len(reply) > 1 else reply[0] + return reply + + #: Pure geometry on a reply, so the real implementation is used rather than + #: a queued answer. + get_highlight_rect = LLMWhispererClientV2.get_highlight_rect + + def __getattr__(self, name): + def call(*args, **kwargs): + return self._reply(name, *args, **kwargs) + + return call + + def kwargs_for(self, name) -> dict: + return next(kw for called, _, kw in self.calls if called == name) + + +@pytest.fixture +def whisper_client(monkeypatch): + """Install a fake LLMWhisperer client and hand it back to the test.""" + + def install(**replies): + client = FakeWhisper(**replies) + # Resolving the credential is what registers it for scrubbing, so the + # fake factory has to do it too or the seam hides a production path. + monkeypatch.setattr( + whisper_cmd, + "llmwhisperer", + lambda config: (config.get(LLMWHISPERER, "api_key"), client)[1], + ) + return client + + return install + + +@pytest.fixture +def deployment_client(monkeypatch): + """Install a fake deployment client and hand it back to the test.""" + + def install(**replies): + client = FakeWhisper(**replies) + client.api_url = "https://api.example.com/deployment/api/org/api-name/" + client.built_with = {} + + def build(_config, _target, transport_timeout=None): + client.built_with["transport_timeout"] = transport_timeout + return client + + monkeypatch.setattr(docstudio_cmd, "deployment", build) + return client + + return install + + +# --------------------------------------------------------------------------- # +# The command surface +# --------------------------------------------------------------------------- # + + +def test_the_v1_commands_are_registered(): + tree = command_tree() + assert set(tree["whisper"]["commands"]) == { + "detail", + "extract", + "highlights", + "retrieve", + "status", + "usage", + "webhook", + } + assert set(tree["whisper"]["commands"]["webhook"]["commands"]) == { + "create", + "delete", + "get", + "update", + } + assert set(tree["docstudio"]["commands"]["deployment"]["commands"]) == { + "run", + "status", + } + + +# --------------------------------------------------------------------------- # +# whisper extract +# --------------------------------------------------------------------------- # + + +def test_extract_without_wait_returns_the_handle(capsys, whisper_client, tmp_path): + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + client = whisper_client(whisper={"whisper_hash": "h1", "status_code": 202}) + + code, out, _ = run(capsys, "whisper", "extract", str(doc), "--no-wait") + + assert code == int(ExitCode.SUCCESS) + assert envelope(out)["data"]["whisper_hash"] == "h1" + assert client.kwargs_for("whisper")["file_path"] == str(doc) + + +def test_only_the_flags_that_were_passed_reach_the_client( + capsys, whisper_client, tmp_path +): + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + client = whisper_client(whisper={"whisper_hash": "h1"}) + + run(capsys, "whisper", "extract", str(doc), "--no-wait", "--mode", "table") + + sent = client.kwargs_for("whisper") + assert sent["mode"] == "table" + assert "lang" not in sent and "median_filter_size" not in sent + + +def test_a_falsy_flag_still_reaches_the_client(capsys, whisper_client, tmp_path): + """`--median-filter-size 0` is a choice; a truthiness filter would drop it + and silently leave the client's own default in place.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + client = whisper_client(whisper={"whisper_hash": "h1"}) + + run( + capsys, + "whisper", + "extract", + str(doc), + "--no-wait", + "--median-filter-size", + "0", + "--no-add-line-nos", + ) + + sent = client.kwargs_for("whisper") + assert sent["median_filter_size"] == 0 + assert sent["add_line_nos"] is False + + +def test_a_url_source_is_sent_as_a_url(capsys, whisper_client): + client = whisper_client(whisper={"whisper_hash": "h1"}) + run(capsys, "whisper", "extract", "https://example.com/a.pdf", "--no-wait") + sent = client.kwargs_for("whisper") + assert sent["url"] == "https://example.com/a.pdf" and "file_path" not in sent + + +def test_the_cli_owns_the_wait_loop(capsys, whisper_client, tmp_path): + """The client has a blocking loop of its own; using it would make --interval, + --timeout and the handle-on-timeout behaviour product-specific.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + client = whisper_client( + whisper={"whisper_hash": "h1"}, + whisper_status=[{"status": "processing"}, {"status": "processed"}], + whisper_retrieve={"extraction": {"result_text": "hello"}}, + ) + + code, out, _ = run(capsys, "-q", "whisper", "extract", str(doc), "--interval", "0") + + assert code == int(ExitCode.SUCCESS) + assert client.kwargs_for("whisper")["wait_for_completion"] is False + assert envelope(out)["data"] == {"result_text": "hello"} + + +def test_raw_output_prints_the_extracted_text(capsys, whisper_client, tmp_path): + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + whisper_client( + whisper={"whisper_hash": "h1"}, + whisper_status={"status": "processed"}, + whisper_retrieve={"extraction": {"result_text": "hello"}}, + ) + + _, out, _ = run( + capsys, "-q", "-o", "raw", "whisper", "extract", str(doc), "--interval", "0" + ) + assert out.strip() == "hello" + + +def test_wait_and_use_webhook_are_mutually_exclusive(capsys, whisper_client, tmp_path): + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + whisper_client(whisper={"whisper_hash": "h1"}) + + code, out, _ = run( + capsys, "whisper", "extract", str(doc), "--use-webhook", "wh1", "--wait" + ) + assert code == int(ExitCode.USAGE) + assert "webhook" in envelope(out)["error"]["hint"] + + +def test_a_failed_extraction_carries_the_handle(capsys, whisper_client, tmp_path): + """A caller can resume from the handle rather than resubmitting.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + whisper_client( + whisper={"whisper_hash": "h1"}, + whisper_status={"status": "error", "message": "bad scan"}, + ) + + code, out, _ = run(capsys, "-q", "whisper", "extract", str(doc), "--interval", "0") + assert code == int(ExitCode.VALIDATION) + assert envelope(out)["error"]["whisper_hash"] == "h1" + + +def test_a_transport_failure_mid_poll_carries_the_handle( + capsys, whisper_client, tmp_path +): + """The document is submitted and billed by this point. Without the handle the + only way on is to send it again and pay for it twice.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + whisper_client( + whisper={"whisper_hash": "h1"}, + whisper_status=ConnectionError("connection dropped"), + ) + + code, out, _ = run(capsys, "-q", "whisper", "extract", str(doc), "--interval", "0") + assert code == int(ExitCode.SERVER_ERROR) + assert envelope(out)["error"]["whisper_hash"] == "h1" + + +def test_a_failed_retrieve_carries_the_handle(capsys, whisper_client, tmp_path): + """Retrieve is the acknowledging read: a failure here can lose the text and + the handle at once, and the handle is the only way back to either.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + whisper_client( + whisper={"whisper_hash": "h1"}, + whisper_status={"status": "processed"}, + whisper_retrieve=ConnectionError("connection dropped"), + ) + + code, out, _ = run(capsys, "-q", "whisper", "extract", str(doc), "--interval", "0") + assert code == int(ExitCode.SERVER_ERROR) + assert envelope(out)["error"]["whisper_hash"] == "h1" + + +# --------------------------------------------------------------------------- # +# Retrieval is one-shot +# --------------------------------------------------------------------------- # + + +def test_retrieve_saves_before_it_prints(capsys, whisper_client, tmp_path): + """A result can be read once. Persisting after printing loses it to a broken + pipe or a full terminal buffer.""" + target = tmp_path / "out" / "result.json" + whisper_client(whisper_retrieve={"extraction": {"result_text": "hello"}}) + + code, out, _ = run(capsys, "whisper", "retrieve", "h1", "--save", str(target)) + + assert code == int(ExitCode.SUCCESS) + assert json.loads(target.read_text())["result_text"] == "hello" + assert envelope(out)["data"]["result_text"] == "hello" + + +def test_an_already_consumed_result_has_its_own_exit_code(capsys, whisper_client): + whisper_client(whisper_retrieve=LLMWhispererClientException("already retrieved", 406)) + code, out, _ = run(capsys, "whisper", "retrieve", "h1") + assert code == int(ExitCode.ALREADY_CONSUMED) + assert "once" in envelope(out)["error"]["hint"] + + +# --------------------------------------------------------------------------- # +# Errors from the client +# --------------------------------------------------------------------------- # + + +def test_an_auth_failure_maps_onto_its_exit_code(capsys, whisper_client): + whisper_client(get_usage_info=LLMWhispererClientException("bad key", 401)) + code, out, _ = run(capsys, "whisper", "usage") + assert code == int(ExitCode.AUTH) + assert envelope(out)["error"]["message"] == "bad key" + + +def test_an_error_body_keeps_its_own_wording(capsys, whisper_client): + whisper_client( + whisper_detail=LLMWhispererClientException( + {"message": "no such hash", "status_code": 404} + ) + ) + code, out, _ = run(capsys, "whisper", "detail", "h1") + assert code == int(ExitCode.NOT_FOUND) + error = envelope(out)["error"] + assert error["message"] == "no such hash" + assert error["details"]["status_code"] == 404 + + +# --------------------------------------------------------------------------- # +# highlights +# --------------------------------------------------------------------------- # + + +def test_highlights_scales_line_metadata_when_a_page_size_is_given( + capsys, whisper_client +): + """Pure arithmetic on the reply, so it is folded into this command rather + than being a command that makes no request.""" + whisper_client(get_highlight_data={"1": [1, 100, 20, 1000]}) + code, out, _ = run( + capsys, + "whisper", + "highlights", + "h1", + "--lines", + "1-5", + "--target-width", + "600", + "--target-height", + "800", + ) + assert code == int(ExitCode.SUCCESS) + data = envelope(out)["data"] + assert data["rects"]["1"] == [1, 0, 64, 600, 80] + + +def test_highlights_reads_the_named_metadata_object(capsys, whisper_client): + """The service returns the list inside an object; the client's geometry takes + the bare list.""" + whisper_client(get_highlight_data={"1": {"raw": [1, 100, 20, 1000], "page": 1}}) + _, out, _ = run( + capsys, + "whisper", + "highlights", + "h1", + "--lines", + "1-5", + "--target-width", + "600", + "--target-height", + "800", + ) + assert envelope(out)["data"]["rects"]["1"] == [1, 0, 64, 600, 80] + + +def test_a_line_without_geometry_gets_no_box(capsys, whisper_client): + """The service reports a line it has no geometry for as all zeros, and the + page height is a divisor in the scaling.""" + whisper_client( + get_highlight_data={"1": {"raw": [0, 0, 0, 0]}, "2": {"raw": [1, 100, 20, 1000]}} + ) + code, out, _ = run( + capsys, + "whisper", + "highlights", + "h1", + "--lines", + "1-5", + "--target-width", + "600", + "--target-height", + "800", + ) + assert code == int(ExitCode.SUCCESS) + assert set(envelope(out)["data"]["rects"]) == {"2"} + + +def test_highlights_returns_the_metadata_alone_without_a_page_size( + capsys, whisper_client +): + whisper_client(get_highlight_data={"1": [1, 100, 20, 1000]}) + _, out, _ = run(capsys, "whisper", "highlights", "h1", "--lines", "1-5") + assert envelope(out)["data"] == {"1": [1, 100, 20, 1000]} + + +# --------------------------------------------------------------------------- # +# Deployments +# --------------------------------------------------------------------------- # + + +def test_run_queues_the_execution_and_polls_it(capsys, deployment_client, tmp_path): + """`timeout=0` queues, so the CLI holds the poll loop instead of the request + holding a connection open for the length of the job.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + client = deployment_client( + structure_file={ + "status_code": 200, + "pending": True, + "execution_status": "PENDING", + "status_check_api_endpoint": "/status?execution_id=e1", + }, + check_execution_status=[ + {"status_code": 200, "pending": True, "execution_status": "EXECUTING"}, + { + "status_code": 200, + "pending": False, + "execution_status": "COMPLETED", + "extraction_result": [{"file": "doc.pdf"}], + }, + ], + ) + + code, out, _ = run( + capsys, + "-q", + "docstudio", + "deployment", + "run", + "my-api", + str(doc), + "--interval", + "0", + ) + + assert code == int(ExitCode.SUCCESS) + assert client.kwargs_for("structure_file")["timeout"] == 0 + assert envelope(out)["data"]["execution_status"] == "COMPLETED" + + +@pytest.mark.parametrize( + ("flag", "expected"), [([], None), (["--transport-timeout", "12.5"], 12.5)] +) +def test_the_transport_timeout_flag_reaches_the_client( + capsys, deployment_client, tmp_path, flag, expected +): + """Unset means a stalled connection is never given up on, which is what + the client has always done.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + client = deployment_client( + structure_file={"status_code": 200, "execution_status": "COMPLETED"} + ) + + code, _out, _err = run( + capsys, "-q", "docstudio", *flag, "deployment", "run", "my-api", str(doc) + ) + + assert code == int(ExitCode.SUCCESS) + assert client.built_with["transport_timeout"] == expected + + +def test_run_passes_only_the_flags_that_were_given(capsys, deployment_client, tmp_path): + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + client = deployment_client( + structure_file={"status_code": 200, "execution_status": "COMPLETED"} + ) + + run( + capsys, + "docstudio", + "deployment", + "run", + "my-api", + str(doc), + "--no-wait", + "--tags", + "a,b", + "--no-include-metrics", + ) + + sent = client.kwargs_for("structure_file") + assert sent["tags"] == "a,b" + assert sent["include_metrics"] is False + assert "llm_profile_id" not in sent + + +def test_an_error_status_from_a_run_is_a_failure(capsys, deployment_client, tmp_path): + """The client reports the status code instead of raising, so an error would + otherwise be reported as a successful run with an error inside it.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + deployment_client( + structure_file={ + "status_code": 422, + "pending": False, + "execution_status": "ERROR", + "error": "no such API", + } + ) + + code, out, _ = run( + capsys, "docstudio", "deployment", "run", "my-api", str(doc), "--no-wait" + ) + assert code == int(ExitCode.VALIDATION) + assert envelope(out)["error"]["message"] == "no such API" + + +def test_deployment_status_reports_a_running_execution(capsys, deployment_client): + client = deployment_client( + check_execution_status={ + "status_code": 200, + "pending": True, + "execution_status": "EXECUTING", + } + ) + code, out, _ = run(capsys, "docstudio", "deployment", "status", "my-api", "e1") + assert code == int(ExitCode.SUCCESS) + assert envelope(out)["data"]["execution_status"] == "EXECUTING" + assert "execution_id=e1" in client.calls[0][1][0] + + +@pytest.mark.parametrize( + ("flag", "name", "value"), + [ + ("--include-metadata", "include_metadata", True), + ("--no-include-metadata", "include_metadata", False), + ("--include-metrics", "include_metrics", True), + ("--no-include-metrics", "include_metrics", False), + ("--include-extracted-text", "include_extracted_text", True), + ("--no-include-extracted-text", "include_extracted_text", False), + ], +) +def test_a_status_flag_reaches_the_client(capsys, deployment_client, flag, name, value): + """A derived flag that is collected and never forwarded is indistinguishable + from one that works: the command still succeeds and the payload still parses.""" + client = deployment_client( + check_execution_status={"status_code": 200, "execution_status": "COMPLETED"} + ) + run(capsys, "docstudio", "deployment", "status", flag, "my-api", "e1") + assert client.kwargs_for("check_execution_status")[name] is value + + +def test_status_sends_only_the_flags_that_were_given(capsys, deployment_client): + client = deployment_client( + check_execution_status={"status_code": 200, "execution_status": "COMPLETED"} + ) + run(capsys, "docstudio", "deployment", "status", "my-api", "e1") + assert client.kwargs_for("check_execution_status") == {} + + +def test_a_waited_run_reads_its_result_with_the_flags_it_was_given( + capsys, deployment_client, tmp_path +): + """Otherwise --wait silently returns less than the same flags return without + it: the run is asked for metrics and the read that fetches them is not.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + client = deployment_client( + structure_file={ + "status_code": 200, + "pending": True, + "execution_status": "PENDING", + "status_check_api_endpoint": "/status?execution_id=e1", + }, + check_execution_status={ + "status_code": 200, + "pending": False, + "execution_status": "COMPLETED", + }, + ) + + run( + capsys, + "-q", + "docstudio", + "deployment", + "run", + "my-api", + str(doc), + "--interval", + "0", + "--include-metrics", + "--no-include-metadata", + ) + + polled = client.kwargs_for("check_execution_status") + assert polled["include_metrics"] is True + assert polled["include_metadata"] is False + # `tags` is a run-time parameter the status endpoint does not accept. + assert "tags" not in polled + + +def test_a_waited_run_reports_which_execution_it_was(capsys, deployment_client, tmp_path): + """The waited payload names the execution nowhere, so without this a caller + has no id to correlate the result against the service.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + deployment_client( + structure_file={ + "status_code": 200, + "pending": True, + "execution_status": "PENDING", + "status_check_api_endpoint": "/status?execution_id=e1", + }, + check_execution_status={ + "status_code": 200, + "pending": False, + "execution_status": "COMPLETED", + }, + ) + + _, out, _ = run( + capsys, + "-q", + "docstudio", + "deployment", + "run", + "my-api", + str(doc), + "--interval", + "0", + ) + assert envelope(out)["meta"]["execution_id"] == "e1" + + +def test_a_run_only_parameter_is_not_forwarded_to_the_status_read( + capsys, deployment_client, tmp_path +): + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + client = deployment_client( + structure_file={ + "status_code": 200, + "pending": True, + "execution_status": "PENDING", + "status_check_api_endpoint": "/status?execution_id=e1", + }, + check_execution_status={ + "status_code": 200, + "pending": False, + "execution_status": "COMPLETED", + }, + ) + + run( + capsys, + "-q", + "docstudio", + "deployment", + "run", + "my-api", + str(doc), + "--interval", + "0", + "--tags", + "a,b", + ) + + assert client.kwargs_for("structure_file")["tags"] == "a,b" + assert client.kwargs_for("check_execution_status") == {} + + +# --------------------------------------------------------------------------- # +# The flag tier of flag > env > profile > default +# --------------------------------------------------------------------------- # + + +def test_a_connection_flag_beats_the_environment(capsys, monkeypatch, tmp_path): + monkeypatch.setenv("LLMWHISPERER_BASE_URL", "https://from-env.test") + monkeypatch.setenv("LLMWHISPERER_API_KEY", "env-key") + seen = {} + monkeypatch.setattr( + whisper_cmd, + "llmwhisperer", + lambda config: ( + seen.update( + base_url=config.get("llmwhisperer", "base_url"), + api_key=config.get("llmwhisperer", "api_key"), + ) + or FakeWhisper(get_usage_info={}) + ), + ) + + code, _, err = run( + capsys, + "whisper", + "--base-url", + "https://from-flag.test", + "--api-key", + "flag-key", + "usage", + ) + + assert code == int(ExitCode.SUCCESS) + assert seen == {"base_url": "https://from-flag.test", "api_key": "flag-key"} + # A key on the command line lands in shell history and the process list. + assert "shell history" in err + + +def test_the_environment_still_wins_over_a_profile(capsys, monkeypatch, write_config): + write_config( + """ + default_profile = "p" + [profiles.p.llmwhisperer] + base_url = "https://from-profile.test" + """ + ) + monkeypatch.setenv("LLMWHISPERER_BASE_URL", "https://from-env.test") + seen = {} + monkeypatch.setattr( + whisper_cmd, + "llmwhisperer", + lambda config: ( + seen.update(base_url=config.get("llmwhisperer", "base_url")) + or FakeWhisper(get_usage_info={}) + ), + ) + + run(capsys, "whisper", "usage") + assert seen == {"base_url": "https://from-env.test"} + + +def test_a_deployment_org_can_come_from_a_flag(capsys, monkeypatch): + monkeypatch.setenv("UNSTRACT_DEPLOYMENT_KEY", "key") + seen = {} + monkeypatch.setattr( + docstudio_cmd, + "deployment", + lambda config, target, transport_timeout=None: ( + seen.update(org=config.get("docstudio", "org_id")) or _deployment_fake() + ), + ) + run(capsys, "docstudio", "--org-id", "org_A", "deployment", "status", "api", "e1") + assert seen == {"org": "org_A"} + + +def _deployment_fake(): + client = FakeWhisper( + check_execution_status={"status_code": 200, "execution_status": "COMPLETED"} + ) + client.api_url = "https://api.example.com/deployment/api/org/api-name/" + return client + + +# --------------------------------------------------------------------------- # +# The one-shot data path +# --------------------------------------------------------------------------- # + + +def test_a_waited_extract_keeps_a_result_that_is_not_wrapped( + capsys, whisper_client, tmp_path +): + """A bare `.get("extraction")` returned None here and printed + `ok: true, data: null` for a document that had been processed and billed.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + whisper_client( + whisper={"whisper_hash": "h1", "status_code": 202}, + whisper_status={"status": "processed"}, + # No `extraction` key -- the shape the sibling command already tolerated. + whisper_retrieve={"status_code": 200, "result_text": "THE REAL TEXT"}, + ) + + code, out, _ = run(capsys, "whisper", "extract", str(doc), "--interval", "0") + + assert code == int(ExitCode.SUCCESS) + assert envelope(out)["data"]["result_text"] == "THE REAL TEXT" + + +def test_a_waited_extract_calls_an_empty_result_a_failure( + capsys, whisper_client, tmp_path +): + """The read is acknowledged either way, so an empty result is a consumed + document with nothing to show for it.""" + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + whisper_client( + whisper={"whisper_hash": "h1", "status_code": 202}, + whisper_status={"status": "processed"}, + whisper_retrieve={"extraction": {}}, + ) + + code, out, _ = run(capsys, "whisper", "extract", str(doc), "--interval", "0") + + assert code == int(ExitCode.SERVER_ERROR) + assert envelope(out)["ok"] is False + + +def test_a_waited_extract_reads_the_result_when_it_is_not_wrapped( + capsys, whisper_client, tmp_path +): + doc = tmp_path / "doc.pdf" + doc.write_bytes(b"%PDF-") + whisper_client( + whisper={"whisper_hash": "h1", "status_code": 202}, + whisper_status={"status": "processed"}, + whisper_retrieve={"extraction": {"result_text": "hello"}}, + ) + + code, out, _ = run(capsys, "whisper", "extract", str(doc), "--interval", "0") + + assert code == int(ExitCode.SUCCESS) + assert envelope(out)["data"]["result_text"] == "hello" + + +def test_retrieve_writes_the_result_before_it_prints( + capsys, whisper_client, tmp_path, monkeypatch +): + """Ordering, not outcome: asserting after the command returns passes for + either order, which is how this went unnoticed.""" + order: list[str] = [] + target = tmp_path / "out" / "result.json" + whisper_client(whisper_retrieve={"extraction": {"result_text": "hello"}}) + + real_persist = whisper_cmd.persist + monkeypatch.setattr( + whisper_cmd, + "persist", + lambda path, payload: (order.append("persist"), real_persist(path, payload))[1], + ) + real_finish = whisper_cmd.finish + monkeypatch.setattr( + whisper_cmd, + "finish", + lambda *a, **kw: (order.append("finish"), real_finish(*a, **kw))[1], + ) + + run(capsys, "whisper", "retrieve", "h1", "--save", str(target)) + + assert order == ["persist", "finish"] + + +def test_retrieve_refuses_an_unwritable_target_before_reading( + capsys, whisper_client, tmp_path +): + """Nothing has been consumed yet at this point, so this failure is cheap -- + the same failure after the read is not recoverable at all.""" + blocker = tmp_path / "not-a-dir" + blocker.write_text("") + client = whisper_client(whisper_retrieve={"extraction": {"result_text": "hello"}}) + + code, out, _ = run( + capsys, "whisper", "retrieve", "h1", "--save", str(blocker / "r.json") + ) + + assert code == int(ExitCode.USAGE) + assert client.calls == [] + + +def test_a_save_failure_after_the_read_still_emits_the_result( + capsys, whisper_client, tmp_path, monkeypatch +): + target = tmp_path / "result.json" + whisper_client(whisper_retrieve={"extraction": {"result_text": "IRREPLACEABLE"}}) + + def explode(path, payload): + # What `persist` itself raises when the write fails: the payload rides + # out on the error because there is no other copy left. + raise CLIError( + "The result could not be written.", + ExitCode.SAVE_FAILED, + details=payload, + ) + + monkeypatch.setattr(whisper_cmd, "persist", explode) + + code, out, _ = run(capsys, "whisper", "retrieve", "h1", "--save", str(target)) + + assert code == int(ExitCode.SAVE_FAILED) + assert envelope(out)["error"]["details"]["result_text"] == "IRREPLACEABLE" + + +def test_a_failed_execution_inside_a_200_is_not_a_success(capsys, deployment_client): + deployment_client( + check_execution_status={ + "status_code": 200, + "pending": False, + "execution_status": "ERROR", + "error": "tool crashed", + } + ) + + code, out, _ = run(capsys, "docstudio", "deployment", "status", "api", "e1") + + assert code != int(ExitCode.SUCCESS) + assert envelope(out)["ok"] is False + + +def test_the_key_never_reaches_stdout_or_stderr(capsys, whisper_client, monkeypatch): + """Scrubbing is not a keyword argument a call site can forget.""" + key = "lw-live-ABCDEF0123456789" + monkeypatch.setenv("LLMWHISPERER_API_KEY", key) + whisper_client( + whisper_retrieve=LLMWhispererClientException( + {"message": f"invalid key {key}", "status_code": 401}, 401 + ) + ) + + code, out, err = run(capsys, "whisper", "retrieve", "h1") + + assert code == int(ExitCode.AUTH) + assert key not in out + assert key not in err + + +def test_clone_maps_its_flags_and_reports_a_partial_failure(capsys, monkeypatch): + """Migration flags decide what is copied where, with two admin keys in play.""" + captured: dict = {} + + def fake_clone(source, target, options): + captured.update(source=source, target=target, options=options) + return CloneReport( + source=Endpoint(source.base_url, source.organization_id), + target=Endpoint(target.base_url, target.organization_id), + phases=[ + PhaseResult(name="adapters", created=1, failed=2), + PhaseResult(name="files", created=1, skipped=3), + ], + oversize_files=[{"name": "big.pdf"}, {"name": "bigger.pdf"}], + ) + + monkeypatch.setattr(clone_cmd, "run_clone", fake_clone) + monkeypatch.setenv("UNSTRACT_SRC_PLATFORM_KEY", "src-key-0123456789") + monkeypatch.setenv("UNSTRACT_TGT_PLATFORM_KEY", "tgt-key-0123456789") + + code, out, err = run( + capsys, + "clone", + "--source-url", + "https://dev.example.com", + "--source-org", + "org_dev", + "--target-url", + "https://qa.example.com", + "--target-org", + "org_qa", + "--dry-run", + "--exclude", + "files, groups", + "--skip-files", + "--max-file-size", + "2MB", + "--api-prefix", + "api/v2", + "--on-name-conflict", + "abort", + ) + + assert captured["source"].platform_key == "src-key-0123456789" + assert captured["target"].organization_id == "org_qa" + assert captured["target"].api_path_prefix == "api/v2" + assert captured["options"].dry_run is True + assert captured["options"].exclude == ("files", "groups") + assert captured["options"].file_strategy == "skip" + assert captured["options"].max_file_size == 2 * 1024 * 1024 + # adopt and abort decide what is written into a live target organisation. + assert captured["options"].on_name_conflict == "abort" + + # A phase that failed is not a successful migration, whatever else worked. + assert code == int(ExitCode.GENERIC) + body = envelope(out) + assert body["ok"] is False + assert "adapters" in body["error"]["message"] + # Documents that never arrived are counted where a consumer reads first. + assert body["error"]["details"]["skipped"] == { + "total": 3, + "by_phase": {"files": 3}, + "oversize_files": 2, + "unsupported_files": 0, + } + for key in ("src-key-0123456789", "tgt-key-0123456789"): + assert key not in out and key not in err + + +def test_a_key_quoted_in_a_clone_report_does_not_survive_the_table(capsys, monkeypatch): + """The table is the output a person gets, and the report renders itself. + + A platform key quoted back by a failing service lands in a terminal buffer + and in whatever scrapes one, so the rendered report is scrubbed on the same + path as every envelope rather than by hand. + """ + key = "src-key-0123456789" + + def fake_clone(source, target, options): + return CloneReport( + source=Endpoint(source.base_url, source.organization_id), + target=Endpoint(target.base_url, target.organization_id), + phases=[PhaseResult(name="adapters", created=1)], + warnings=[f"target refused the request for {key}"], + ) + + monkeypatch.setattr(clone_cmd, "run_clone", fake_clone) + monkeypatch.setenv("UNSTRACT_SRC_PLATFORM_KEY", key) + monkeypatch.setenv("UNSTRACT_TGT_PLATFORM_KEY", "tgt-key-0123456789") + + code = main( + [ + "-o", + "table", + "clone", + "--source-url", + "https://dev.example.com", + "--source-org", + "org_dev", + "--target-url", + "https://qa.example.com", + "--target-org", + "org_qa", + ] + ) + captured = capsys.readouterr() + + assert code == int(ExitCode.SUCCESS) + assert "adapters" in captured.out + assert key not in captured.out and key not in captured.err diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..739d192 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,353 @@ +"""Config resolution: flag > env > profile > built-in default.""" + +from __future__ import annotations + +import stat +from pathlib import Path + +import pytest + +from unstract_cli.config import ( + DEFAULT_BASE_URLS, + DOCSTUDIO, + LLMWHISPERER, + PROJECT_CONFIG_NAME, + ConfigError, + ConfigFile, + ResolvedConfig, + config_path, + find_project_config, + load_config, + save_config, + set_config_path, + starter_profiles, +) + +PROFILE_TOML = """ +default_profile = "p" + +[profiles.p.llmwhisperer] +base_url = "https://profile.example/api/v2" +api_key = "profile-key" + +[profiles.p.docstudio] +org_id = "org_from_profile" +api_key = "env:UNSTRACT_DEPLOYMENT_KEY" + +[profiles.p.deployments.invoices] +api_name = "invoice-parser" + +[profiles.p.deployments.receipts] +api_name = "receipt-parser" +org_id = "org_alias" +api_key = "alias-key" +""" + + +def resolved(overrides=None, profile=None): + return ResolvedConfig( + file=load_config(), profile_name=profile, overrides=overrides or {} + ) + + +def test_default_when_nothing_configured(): + assert resolved().get(LLMWHISPERER, "base_url") == DEFAULT_BASE_URLS[LLMWHISPERER] + assert resolved().get(LLMWHISPERER, "api_key") is None + + +def test_profile_beats_default(write_config): + write_config(PROFILE_TOML) + assert resolved().get(LLMWHISPERER, "base_url") == "https://profile.example/api/v2" + + +def test_env_beats_profile(write_config, monkeypatch): + write_config(PROFILE_TOML) + monkeypatch.setenv("LLMWHISPERER_BASE_URL", "https://env.example/api/v2") + assert resolved().get(LLMWHISPERER, "base_url") == "https://env.example/api/v2" + + +def test_override_beats_env(write_config, monkeypatch): + write_config(PROFILE_TOML) + monkeypatch.setenv("LLMWHISPERER_BASE_URL", "https://env.example/api/v2") + cfg = resolved(overrides={"llmwhisperer.base_url": "https://flag.example"}) + assert cfg.get(LLMWHISPERER, "base_url") == "https://flag.example" + + +def test_env_indirection_resolves_and_missing_var_reads_as_unset( + write_config, monkeypatch +): + write_config(PROFILE_TOML) + assert resolved().get(DOCSTUDIO, "api_key") is None + monkeypatch.setenv("UNSTRACT_DEPLOYMENT_KEY", "secret-value") + assert resolved().get(DOCSTUDIO, "api_key") == "secret-value" + + +def test_require_names_every_way_to_supply_the_setting(): + with pytest.raises(ConfigError) as excinfo: + resolved().require(DOCSTUDIO, "api_key") + message = str(excinfo.value) + assert "UNSTRACT_DEPLOYMENT_KEY" in message + assert "[profiles..docstudio]" in message + # Credentials get no flag, so none may be suggested. + assert "--api-key" not in message + + +def test_placeholder_is_not_a_value(write_config): + """`config init` writes `org_id = ""`, and that must not satisfy `require`.""" + write_config('default_profile = "p"\n\n[profiles.p.docstudio]\norg_id = ""\n') + assert resolved().get(DOCSTUDIO, "org_id") is None + assert resolved().resolution_source(DOCSTUDIO, "org_id")["resolved"] is False + with pytest.raises(ConfigError): + resolved().require(DOCSTUDIO, "org_id") + + +def test_starter_profile_org_id_does_not_satisfy_require(write_config): + path = write_config("") + save_config(ConfigFile(default_profile="cloud-us", profiles=starter_profiles()), path) + with pytest.raises(ConfigError): + resolved().require(DOCSTUDIO, "org_id") + + +def test_unknown_profile_is_an_error_not_a_silent_empty_block(write_config): + write_config(PROFILE_TOML) + with pytest.raises(ConfigError, match="not found"): + resolved(profile="nope").get(DOCSTUDIO, "org_id") + + +def test_profile_selected_by_env_var(write_config, monkeypatch): + write_config(PROFILE_TOML.replace('default_profile = "p"', "")) + monkeypatch.setenv("UNSTRACT_PROFILE", "p") + assert resolved().get(DOCSTUDIO, "org_id") == "org_from_profile" + + +def test_deployment_alias_falls_back_to_the_product_block(write_config, monkeypatch): + write_config(PROFILE_TOML) + monkeypatch.setenv("UNSTRACT_DEPLOYMENT_KEY", "secret-value") + alias = resolved().deployment("invoices") + assert alias == { + "api_name": "invoice-parser", + "org_id": "org_from_profile", + "api_key": "secret-value", + } + + +def test_deployment_alias_overrides_win(write_config): + write_config(PROFILE_TOML) + alias = resolved().deployment("receipts") + assert alias["org_id"] == "org_alias" + assert alias["api_key"] == "alias-key" + + +def test_unknown_deployment_alias_lists_the_known_ones(write_config): + write_config(PROFILE_TOML) + with pytest.raises(ConfigError, match="invoices, receipts"): + resolved().deployment("nope") + + +def test_resolution_source_reports_the_winner(write_config, monkeypatch): + write_config(PROFILE_TOML) + cfg = resolved() + assert ( + cfg.resolution_source(LLMWHISPERER, "base_url")["source"] == "profile (literal)" + ) + assert cfg.resolution_source(DOCSTUDIO, "base_url")["source"] == "built-in default" + assert cfg.resolution_source(DOCSTUDIO, "api_key") == { + "resolved": False, + "source": "profile -> env:UNSTRACT_DEPLOYMENT_KEY", + "detail": "$UNSTRACT_DEPLOYMENT_KEY is not set in this process's environment", + } + monkeypatch.setenv("LLMWHISPERER_API_KEY", "k") + assert resolved().resolution_source(LLMWHISPERER, "api_key") == { + "resolved": True, + "source": "env:LLMWHISPERER_API_KEY", + } + + +# --------------------------------------------------------------------------- # +# File discovery and writing +# --------------------------------------------------------------------------- # + + +def test_discovery_order(tmp_path, monkeypatch): + from unstract_cli import config as config_mod + + home_default = config_mod.HOME_CONFIG + assert config_path() == home_default + + project = tmp_path / "proj" / "nested" + project.mkdir(parents=True) + (tmp_path / "proj" / ".unstract.toml").touch() + monkeypatch.chdir(project) + assert config_path() == tmp_path / "proj" / ".unstract.toml" + + monkeypatch.setenv("UNSTRACT_CONFIG", str(tmp_path / "env.toml")) + assert config_path() == tmp_path / "env.toml" + + set_config_path(tmp_path / "flag.toml") + assert config_path() == tmp_path / "flag.toml" + + +def test_project_search_stops_at_home(tmp_path, monkeypatch): + home = tmp_path / "home" + work = home / "work" + work.mkdir(parents=True) + monkeypatch.setattr("pathlib.Path.home", lambda: home) + # Above $HOME, so it must not be picked up. + (tmp_path / ".unstract.toml").touch() + assert find_project_config(work) is None + + +def test_missing_file_is_not_an_error(): + cfg = load_config() + assert cfg.exists is False and cfg.profiles == {} + + +def test_saved_config_is_owner_only(tmp_path): + path = tmp_path / "nested" / "config.toml" + written = save_config( + ConfigFile(default_profile="cloud-us", profiles=starter_profiles()), path + ) + assert stat.S_IMODE(written.stat().st_mode) == 0o600 + assert load_config(written).default_profile == "cloud-us" + + +def test_loose_permissions_warn_rather_than_fail(write_config): + path = write_config(PROFILE_TOML) + path.chmod(0o644) + assert any("readable by other users" in w for w in load_config().warnings) + + +#: What a repository could commit: a host of its own choosing, and a key. +PROJECT_TOML = """ +default_profile = "p" + +[profiles.p.llmwhisperer] +base_url = "https://elsewhere.example/api/v2" +api_key = "project-literal-key" + +[profiles.p.docstudio] +org_id = "org_from_project" + +[profiles.p.deployments.invoices] +api_name = "invoice-parser" +api_key = "alias-literal-key" +""" + + +def _plant_project_config(tmp_path, monkeypatch): + work = tmp_path / "checkout" + work.mkdir() + path = work / ".unstract.toml" + path.write_text(PROJECT_TOML, encoding="utf-8") + monkeypatch.chdir(work) + return path + + +def test_a_discovered_project_config_supplies_no_key_and_no_host(tmp_path, monkeypatch): + path = _plant_project_config(tmp_path, monkeypatch) + cfg = resolved() + + assert cfg.get(LLMWHISPERER, "base_url") == DEFAULT_BASE_URLS[LLMWHISPERER] + assert cfg.get(LLMWHISPERER, "api_key") is None + assert cfg.deployment("invoices")["api_key"] is None + # Everything the file is legitimately for still applies. + assert cfg.get(DOCSTUDIO, "org_id") == "org_from_project" + assert cfg.deployment("invoices")["api_name"] == "invoice-parser" + assert any(str(path) in w and "Ignoring" in w for w in cfg.file.warnings) + assert cfg.resolution_source(LLMWHISPERER, "api_key")["detail"] + + +def test_the_same_file_named_explicitly_is_honoured(tmp_path, monkeypatch): + path = _plant_project_config(tmp_path, monkeypatch) + monkeypatch.setenv("UNSTRACT_CONFIG", str(path)) + cfg = resolved() + + assert cfg.get(LLMWHISPERER, "base_url") == "https://elsewhere.example/api/v2" + assert cfg.get(LLMWHISPERER, "api_key") == "project-literal-key" + assert not any("Ignoring" in w for w in cfg.file.warnings) + + +def test_writing_back_a_project_config_keeps_the_keys_it_withheld(tmp_path, monkeypatch): + path = _plant_project_config(tmp_path, monkeypatch) + cfg = load_config() + cfg.profiles["p"]["docstudio"]["org_id"] = "org_edited" + save_config(cfg) + + monkeypatch.setenv("UNSTRACT_CONFIG", str(path)) + reloaded = load_config() + assert reloaded.profiles["p"]["docstudio"]["org_id"] == "org_edited" + assert reloaded.profiles["p"]["llmwhisperer"]["api_key"] == "project-literal-key" + assert reloaded.profiles["p"]["deployments"]["invoices"]["api_key"] == ( + "alias-literal-key" + ) + + +def test_withheld_keys_are_not_carried_into_a_file_the_user_names(tmp_path, monkeypatch): + _plant_project_config(tmp_path, monkeypatch) + elsewhere = tmp_path / "named.toml" + save_config(load_config(), elsewhere) + + monkeypatch.setenv("UNSTRACT_CONFIG", str(elsewhere)) + assert "api_key" not in load_config().profiles["p"]["llmwhisperer"] + + +def test_naming_the_discovered_file_does_not_make_it_trusted(tmp_path, monkeypatch): + path = _plant_project_config(tmp_path, monkeypatch) + work = path.parent + (work / "sub").mkdir() + (tmp_path / "link").symlink_to(work) + + # The outcome first: the flag is only the mechanism, withholding is the point. + cfg = ResolvedConfig(file=load_config(path)) + assert cfg.get(LLMWHISPERER, "api_key") is None + assert cfg.get(LLMWHISPERER, "base_url") == DEFAULT_BASE_URLS[LLMWHISPERER] + + # However the same file is spelled, it is the same file. + for spelling in ( + Path(PROJECT_CONFIG_NAME), + path, + work / "sub" / ".." / PROJECT_CONFIG_NAME, + tmp_path / "link" / PROJECT_CONFIG_NAME, + ): + assert load_config(spelling).is_project_local is True, spelling + + other = tmp_path / "elsewhere.toml" + other.write_text(PROJECT_TOML, encoding="utf-8") + assert load_config(other).is_project_local is False + + +def test_a_symlinked_project_candidate_is_not_discovered(tmp_path, monkeypatch): + work = tmp_path / "checkout" + work.mkdir() + victim = tmp_path / "victim.toml" + victim.write_text("keep = true\n", encoding="utf-8") + (work / ".unstract.toml").symlink_to(victim) + monkeypatch.chdir(work) + + assert find_project_config(work) is None + assert config_path() != work / ".unstract.toml" + + +def test_a_write_through_a_symlink_fails_without_touching_its_target(tmp_path): + victim = tmp_path / "victim.toml" + victim.write_text("keep = true\n", encoding="utf-8") + link = tmp_path / "config.toml" + link.symlink_to(victim) + + with pytest.raises(ConfigError, match="symlink"): + save_config(ConfigFile(profiles=starter_profiles()), link) + assert victim.read_text(encoding="utf-8") == "keep = true\n" + + +def test_a_withheld_alias_key_is_reported_against_the_alias(tmp_path, monkeypatch): + _plant_project_config(tmp_path, monkeypatch) + cfg = resolved() + assert cfg.withheld_detail("deployments", "invoices", "api_key") + assert cfg.withheld_detail("deployments", "invoices", "org_id") is None + + +def test_starter_profiles_hold_no_literal_secrets(): + for blocks in starter_profiles().values(): + for settings in blocks.values(): + key = settings.get("api_key") + assert key is None or key.startswith("env:") diff --git a/tests/test_contract.py b/tests/test_contract.py new file mode 100644 index 0000000..b0766df --- /dev/null +++ b/tests/test_contract.py @@ -0,0 +1,124 @@ +"""What the CLI can reach of what the APIs offer. + +The vendored specs and the pinned clients move independently: a refreshed spec +can declare a parameter the published client has no argument for, and such a +parameter is dropped from the CLI rather than offered and then rejected at the +call. Dropping it silently is the failure mode this file exists to prevent -- +the gap is written down, so widening it is a decision someone makes on purpose. +""" + +from __future__ import annotations + +import inspect +import json +import os +from dataclasses import asdict +from pathlib import Path +from typing import Any + +import pytest +from unstract.api_deployments.client import APIDeploymentsClient +from unstract.llmwhisperer.client_v2 import LLMWhispererClientV2 + +from unstract_cli.core.params import derive_params, operation_params + +#: (product, operationId, client method) per command that derives its flags, +#: with the spec parameters that method cannot accept. Most are parameters the +#: client owns rather than lacks; `highlights.mode` is the exception, and the +#: CLI cannot offer it without the call failing. +COMMANDS = [ + ( + "llmwhisperer", + "extract", + LLMWhispererClientV2.whisper, + {"url_in_post"}, + ), + ("llmwhisperer", "highlights", LLMWhispererClientV2.get_highlight_data, {"mode"}), + ("docstudio", "execute", APIDeploymentsClient.structure_file, {"files"}), + ( + "docstudio", + "status", + APIDeploymentsClient.check_execution_status, + {"execution_id"}, + ), +] + + +@pytest.mark.parametrize( + ("product", "operation", "method", "unreachable"), + COMMANDS, + ids=[f"{p}:{o}" for p, o, _, _ in COMMANDS], +) +def test_the_parameters_no_command_can_reach_are_the_known_ones( + product, operation, method, unreachable +): + declared = {p.name for p in operation_params(product, operation)} + derived = {p.name for p in derive_params(product, operation, client_method=method)} + assert declared - derived == unreachable + assert derived <= declared + + +@pytest.mark.parametrize( + ("product", "operation", "method"), + [(p, o, m) for p, o, m, _ in COMMANDS], + ids=[f"{p}:{o}" for p, o, _, _ in COMMANDS], +) +def test_every_derived_flag_is_an_argument_the_client_accepts(product, operation, method): + """The check the CLI cannot make at runtime: a flag the client has no + parameter for raises TypeError at the call, after the document is read.""" + accepted = set(inspect.signature(method).parameters) + for param in derive_params(product, operation, client_method=method): + assert param.name in accepted + + +#: The flags the specs derive today, written down rather than read from the +#: spec, so a parameter lost upstream fails here instead of vanishing quietly. +SNAPSHOT = Path(__file__).parent / "derived_flags.json" + +#: Refreshing the snapshot is a decision, not a side effect of running the suite. +REFRESH = "UNSTRACT_CLI_REFRESH_FLAG_SNAPSHOT" + + +def _derived_flags() -> dict[str, dict[str, Any]]: + """Every flag the specs derive, with the whole of what each one accepts. + + Names alone would let a spec narrow an enum, or change a type or a default, + without moving the snapshot -- and the CLI would start rejecting a value it + used to take, with nothing here to say so. + """ + return { + f"{product}:{operation}": { + # Choices as a list: JSON has no tuple, and the snapshot is compared + # against what a JSON reader gives back. + param.flag: {**asdict(param), "choices": list(param.choices)} + for param in sorted( + derive_params(product, operation, client_method=method), + key=lambda param: param.flag, + ) + } + for product, operation, method, _ in COMMANDS + } + + +def _changed(current: dict[str, Any], expected: dict[str, Any]) -> list[str]: + """The flags that moved, named. Comparing whole payloads reports neither.""" + return sorted( + f"{operation} {flag}" + for operation in current.keys() | expected.keys() + for flag in current.get(operation, {}).keys() | expected.get(operation, {}).keys() + if current.get(operation, {}).get(flag) != expected.get(operation, {}).get(flag) + ) + + +def test_the_derived_flags_are_the_ones_last_reviewed(): + current = _derived_flags() + if os.environ.get(REFRESH): + SNAPSHOT.write_text(json.dumps(current, indent=2) + "\n", encoding="utf-8") + expected = json.loads(SNAPSHOT.read_text(encoding="utf-8")) + assert current == expected, ( + "What the vendored specs derive has changed: " + f"{', '.join(_changed(current, expected))}. A flag that disappears here " + "disappears from the CLI, and a choice or a type that narrows here " + "rejects a value the CLI used to take. Review the difference, then " + f"refresh the snapshot with {REFRESH}=1." + ) diff --git a/tests/test_discover.py b/tests/test_discover.py new file mode 100644 index 0000000..5916c9e --- /dev/null +++ b/tests/test_discover.py @@ -0,0 +1,165 @@ +"""`--discover`, and the live half of `config doctor`. + +Discovery is what an agent reads before it runs anything, so the tiers have to +stay cheap-then-detailed, and everything reported has to be read back from the +parser rather than described separately. +""" + +from __future__ import annotations + +import json + +import pytest + +from unstract_cli.__main__ import main +from unstract_cli.commands import config_cmd +from unstract_cli.core.errors import CLIError, ExitCode +from unstract_cli.core.output import CONTRACT_VERSION + + +def run(capsys, *args): + code = main(["-o", "json", *args]) + out = capsys.readouterr().out + return code, json.loads(out)["data"] if out.strip() else None + + +def test_groups_names_the_products_and_stops_there(capsys): + """The cheap question stays cheap: no command list, no flags.""" + code, data = run(capsys, "--discover", "groups") + assert code == int(ExitCode.SUCCESS) + assert {g["name"] for g in data["groups"]} == {"config", "docstudio", "whisper"} + # A leaf listed among the groups is a group a consumer finds empty. + assert [c["name"] for c in data["commands"]] == ["clone"] + assert all(entry["help"] for entry in [*data["groups"], *data["commands"]]) + assert all("commands" not in entry for entry in data["groups"]) + + +def test_summary_lists_commands_without_their_flags(capsys): + _, data = run(capsys, "--discover", "summary") + whisper = data["commands"]["whisper"]["commands"] + assert "extract" in whisper + assert whisper["extract"]["help"] + assert "params" not in whisper["extract"] + + +def test_full_carries_enough_to_build_a_call(capsys): + _, data = run(capsys, "--discover", "full") + extract = data["commands"]["whisper"]["commands"]["extract"] + params = {p["name"]: p for p in extract["params"]} + + assert params["source"]["kind"] == "argument" and params["source"]["required"] + assert params["mode"]["choices"] == [ + "document_insights", + "excel", + "form", + "high_quality", + "low_cost", + "native_text", + "pdf_to_images", + "table", + ] + assert params["wait"]["flags"] == ["--wait", "--no-wait"] + assert params["interval"]["type"] == "float" + assert extract["raw_field"] == "result_text" + + +def test_full_carries_the_exit_code_table(capsys): + """A caller branches on these; they are part of the contract, not prose.""" + _, data = run(capsys, "--discover", "full") + codes = {entry["name"]: entry["code"] for entry in data["exit_codes"]} + assert codes["already_consumed"] == int(ExitCode.ALREADY_CONSUMED) + assert codes["success"] == 0 + + +def test_full_publishes_how_to_consume_the_output(capsys): + """The compatibility bargain is only binding if the consumer can read it.""" + _, data = run(capsys, "--discover", "full") + contract = data["contract"] + assert contract["version"] == CONTRACT_VERSION + assert contract["envelope"] == ["ok", "data", "error", "meta"] + rules = " ".join(contract["rules"]).lower() + assert "-o json" in rules + assert "ignore fields you do not recognise" in rules + assert "contract_version" in rules + + +def test_discovery_needs_no_configuration(capsys, tmp_path, monkeypatch): + """It is how a caller finds out what to run, so it must work before anything + is set up.""" + monkeypatch.setenv("UNSTRACT_CONFIG", str(tmp_path / "nonexistent.toml")) + code, data = run(capsys, "--discover", "summary") + assert code == int(ExitCode.SUCCESS) and data["commands"] + + +def test_an_unknown_tier_is_a_usage_error(capsys): + code = main(["--discover", "sideways"]) + capsys.readouterr() + assert code == int(ExitCode.USAGE) + + +# --------------------------------------------------------------------------- # +# config doctor --probe +# --------------------------------------------------------------------------- # + + +@pytest.fixture +def probe_client(monkeypatch): + def install(reply=None): + class Fake: + def get_usage_info(self): + if isinstance(reply, Exception): + raise reply + return reply or {} + + monkeypatch.setattr(config_cmd, "llmwhisperer", lambda _config: Fake()) + + return install + + +def test_doctor_makes_no_call_without_probe(capsys, probe_client): + probe_client(CLIError("must not be called")) + code, data = run(capsys, "config", "doctor") + assert code == int(ExitCode.SUCCESS) + assert "probe" not in data + + +def test_probe_verifies_the_whisperer_key(capsys, probe_client): + probe_client({"quota": 1}) + _, data = run(capsys, "config", "doctor", "--probe") + assert data["probe"]["llmwhisperer"] == { + "checked": True, + "ok": True, + "detail": "The key was accepted by the usage endpoint.", + } + + +def test_a_rejected_key_reports_why(capsys, probe_client): + """A probe that failed exits non-zero: --probe is run from setup scripts, + and a script branches on the exit code, not on the payload.""" + probe_client(CLIError("bad key", ExitCode.AUTH)) + code = main(["-o", "json", "config", "doctor", "--probe"]) + report = json.loads(capsys.readouterr().out)["error"]["details"] + assert code == int(ExitCode.GENERIC) + entry = report["probe"]["llmwhisperer"] + assert entry == { + "checked": True, + "ok": False, + "detail": "bad key", + "exit_code": int(ExitCode.AUTH), + } + + +def test_the_deployment_probe_says_it_verified_nothing(capsys, probe_client, monkeypatch): + """The only deployment endpoint is an execution, so there is nothing + side-effect-free to call. Saying otherwise would be worse than not checking. + """ + probe_client({}) + monkeypatch.setenv("UNSTRACT_ORG_ID", "org_A") + monkeypatch.setenv("UNSTRACT_DEPLOYMENT_KEY", "key") + _, data = run(capsys, "config", "doctor", "--probe") + entry = data["probe"]["docstudio"] + # `ok` is null rather than true: a true beside `checked: false` is read as a + # live check that passed, which is the one thing this probe cannot claim. + assert entry["checked"] is False and entry["ok"] is None + assert entry["resolved"] is True + assert "NOT verified" in entry["detail"] diff --git a/tests/test_errors.py b/tests/test_errors.py new file mode 100644 index 0000000..50a8d42 --- /dev/null +++ b/tests/test_errors.py @@ -0,0 +1,125 @@ +"""The exit-code table, retry policy and redaction.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from unstract_cli.core.errors import ( + REDACTED, + ExitCode, + error_from_status, + exit_code_for_status, + hint_for, + is_retryable, + redact_headers, + redact_value, + scrub, + undeclared_status_error, +) + + +@pytest.mark.parametrize( + ("status", "expected"), + [ + # Only a failure ever reaches this map: a 2xx or an unfollowed 3xx here + # means something answered outside the contract, which is not success. + (200, ExitCode.GENERIC), + (302, ExitCode.GENERIC), + (400, ExitCode.VALIDATION), + (401, ExitCode.AUTH), + (403, ExitCode.AUTH), + (404, ExitCode.NOT_FOUND), + (406, ExitCode.ALREADY_CONSUMED), + (408, ExitCode.TIMEOUT), + (409, ExitCode.VALIDATION), + (418, ExitCode.GENERIC), + (422, ExitCode.VALIDATION), + (429, ExitCode.RATE_LIMITED), + (500, ExitCode.SERVER_ERROR), + (503, ExitCode.SERVER_ERROR), + ], +) +def test_status_to_exit_code(status, expected): + assert exit_code_for_status(status) is expected + + +def test_exit_codes_are_stable_integers(): + # A caller branches on these numbers, so they are an API, not an enum detail. + assert [int(c) for c in ExitCode] == [*range(11), 130] + assert int(ExitCode.ALREADY_CONSUMED) == 9 + assert int(ExitCode.SAVE_FAILED) == 10 + # 128 + SIGINT, which every shell and job runner already reads as + # "stopped", rather than the next number in this CLI's own sequence. + assert int(ExitCode.INTERRUPTED) == 130 + + +@pytest.mark.parametrize("status", [429, 500, 502, 503]) +def test_retryable(status): + assert is_retryable(status) + + +@pytest.mark.parametrize("status", [400, 401, 403, 404, 406, 409, 422]) +def test_not_retryable(status): + # A 4xx retry re-sends what the server already rejected, and for a one-shot + # read it can consume a result the first attempt already delivered. + assert not is_retryable(status) + + +def test_one_shot_status_carries_its_own_hint(): + assert "already retrieved" in hint_for(406) + assert "--save" in hint_for(406) + + +def test_error_from_status_fills_code_hint_and_retryability(): + err = error_from_status(429, "slow down", endpoint="POST /whisper") + assert err.exit_code is ExitCode.RATE_LIMITED + assert err.retryable is True + assert err.to_dict()["endpoint"] == "POST /whisper" + + +def test_undeclared_status_is_reported_verbatim_never_guessed(): + err = undeclared_status_error(418, {"detail": "teapot"}) + assert "Undeclared status 418" in err.message + assert "teapot" in err.message + assert err.to_dict()["details"] == {"detail": "teapot"} + + +def test_redact_headers(): + out = redact_headers( + { + "unstract-key": "abc", + "Authorization": "Bearer x", + "X-Api-Key": "y", + "Content-Type": "application/json", + } + ) + assert out["unstract-key"] == out["Authorization"] == out["X-Api-Key"] == REDACTED + assert out["Content-Type"] == "application/json" + + +def test_redact_value_walks_nested_payloads(): + out = redact_value({"a": {"api_key": "secret", "n": 1}, "b": [{"token": "t"}]}) + assert out == {"a": {"api_key": REDACTED, "n": 1}, "b": [{"token": REDACTED}]} + + +def test_scrub_ignores_short_values(): + # Redacting a 3-character "key" would mangle unrelated text. + assert scrub("the key is abc", ["abc"]) == "the key is abc" + assert scrub("the key is abcdefghij", ["abcdefghij"]) == f"the key is {REDACTED}" + + +def test_the_readme_table_lists_every_exit_code(): + """The README table is a copy of the enum, and the only one users read.""" + readme = (Path(__file__).resolve().parents[1] / "README.md").read_text() + documented = { + int(row.split("|")[1]) for row in readme.splitlines() if _is_code_row(row) + } + + assert documented == {int(code) for code in ExitCode} + + +def _is_code_row(row: str) -> bool: + cells = row.split("|") + return len(cells) > 2 and cells[1].strip().isdigit() diff --git a/tests/test_output.py b/tests/test_output.py new file mode 100644 index 0000000..9293c11 --- /dev/null +++ b/tests/test_output.py @@ -0,0 +1,140 @@ +"""The stdout envelope and its renderings.""" + +from __future__ import annotations + +import json + +from unstract_cli.core.errors import CLIError, ExitCode +from unstract_cli.core.output import ( + CONTRACT_VERSION, + AgentMode, + OutputFormat, + emit_error, + emit_result, + envelope, + render, + render_table, + resolve_format, +) + +ENVELOPE_KEYS = {"ok", "data", "error", "meta"} + + +def test_success_envelope_shape(): + env = envelope(data={"a": 1}, meta={"took": 2}) + assert set(env) == ENVELOPE_KEYS + assert env == { + "ok": True, + "data": {"a": 1}, + "error": None, + "meta": {"took": 2, "contract_version": CONTRACT_VERSION}, + } + + +def test_error_envelope_shape(): + err = CLIError("boom", ExitCode.AUTH, http_status=401, hint="check the key") + env = envelope(error=err.to_dict()) + assert set(env) == ENVELOPE_KEYS + assert env["ok"] is False and env["data"] is None + assert env["error"] == { + "code": "auth_error", + "message": "boom", + "exit_code": 3, + "retryable": False, + "http_status": 401, + "hint": "check the key", + } + + +def test_meta_defaults_to_an_object_not_null(): + # A caller reading meta. should not have to null-check the container. + assert envelope(data=1)["meta"] == {"contract_version": CONTRACT_VERSION} + + +def test_every_envelope_is_versioned(): + """A consumer cannot refuse a shape it was not written for without this.""" + for env in (envelope(data=1, meta={"job": "x"}), envelope(error={"code": "x"})): + assert env["meta"]["contract_version"] == CONTRACT_VERSION + + +def test_stdout_carries_the_envelope_on_success(capsys): + emit_result({"text": "hello"}, OutputFormat.JSON) + out = capsys.readouterr() + assert json.loads(out.out) == { + "ok": True, + "data": {"text": "hello"}, + "error": None, + "meta": {"contract_version": CONTRACT_VERSION}, + } + assert out.err == "" + + +def test_stdout_carries_the_envelope_on_failure_and_stderr_gets_a_summary(capsys): + code = emit_error(CLIError("nope", ExitCode.NOT_FOUND)) + out = capsys.readouterr() + parsed = json.loads(out.out) + assert parsed["ok"] is False and parsed["error"]["code"] == "not_found" + assert out.err.strip() == "error: nope" + assert code == ExitCode.NOT_FOUND + + +def test_secrets_are_scrubbed_from_both_streams(capsys): + secret = "sk-supersecret-value" + emit_error(CLIError(f"rejected token {secret}"), secrets=[secret]) + out = capsys.readouterr() + assert secret not in out.out and secret not in out.err + assert "***REDACTED***" in out.out + + +def test_table_and_raw_render_the_payload_not_the_envelope(): + env = envelope(data={"text": "hello"}) + assert "hello" in render(env, OutputFormat.TABLE) + assert "ok" not in render(env, OutputFormat.TABLE) + assert render(env, OutputFormat.RAW, raw_field="text") == "hello" + + +def test_raw_renders_the_error_when_the_run_failed(): + env = envelope(error=CLIError("boom").to_dict()) + assert "boom" in render(env, OutputFormat.RAW) + + +def test_table_wraps_long_cells_rather_than_truncating(): + long = "word " * 40 + rendered = render_table([{"text": long.strip()}], max_width=40) + assert rendered.count("\n") > 2 + assert "".join(rendered.split()).count("word") == 40 + + +def test_table_of_an_empty_list_says_so(): + assert render_table([]) == "(no results)" + + +class TestFormatSelection: + """Which rendering a run gets, and what is allowed to influence it.""" + + AGENT = {"CLAUDECODE": "1"} + + def test_the_default_is_a_table(self): + assert resolve_format(None, env={}) is OutputFormat.TABLE + + def test_an_agent_environment_moves_the_default_to_json(self): + for var in ("CLAUDECODE", "CURSOR_AGENT", "CODEX_SANDBOX", "AI_AGENT"): + assert resolve_format(None, env={var: "1"}) is OutputFormat.JSON + + def test_an_unset_marker_is_not_an_agent(self): + """An exported-but-empty variable is how a shell spells 'no'.""" + assert resolve_format(None, env={"CLAUDECODE": ""}) is OutputFormat.TABLE + + def test_an_explicit_format_beats_detection_in_both_directions(self): + assert resolve_format("table", env=self.AGENT) is OutputFormat.TABLE + assert resolve_format("json", env={}) is OutputFormat.JSON + + def test_the_agent_flag_overrides_what_the_environment_says(self): + assert resolve_format(None, AgentMode.NO, self.AGENT) is OutputFormat.TABLE + assert resolve_format(None, AgentMode.YES, {}) is OutputFormat.JSON + + def test_json_renders_the_same_bytes_wherever_it_is_asked_for(self): + env = envelope(data={"text": "hello"}) + one = render(env, resolve_format("json", AgentMode.NO, {})) + two = render(env, resolve_format("json", AgentMode.YES, self.AGENT)) + assert one == two diff --git a/tests/test_params.py b/tests/test_params.py new file mode 100644 index 0000000..47911bc --- /dev/null +++ b/tests/test_params.py @@ -0,0 +1,258 @@ +"""Flag derivation: what the spec says, what the client accepts, what is sent. + +Each test here corresponds to a way derived flags can be wrong while still +looking right: a value silently dropped, a default silently pinned, a flag +offered that the client cannot accept. +""" + +from __future__ import annotations + +import click +import pytest +from unstract.api_deployments.client import APIDeploymentsClient +from unstract.llmwhisperer.client_v2 import LLMWhispererClientV2 + +from unstract_cli.core import params as params_module +from unstract_cli.core.params import ( + Param, + click_option, + derive_params, + docstring_params, + find_operation, + operation_params, + requested, +) + + +def _by_name(params: list[Param]) -> dict[str, Param]: + return {p.name: p for p in params} + + +# --------------------------------------------------------------------------- # +# Reading the spec +# --------------------------------------------------------------------------- # + + +def test_query_parameters_carry_type_and_default(): + params = _by_name(operation_params("llmwhisperer", "extract")) + assert params["mode"].type == "string" + assert params["add_line_nos"].type == "boolean" + assert params["median_filter_size"].type == "integer" + assert params["horizontal_stretch_factor"].default == 1.0 + + +def test_body_parameters_are_derived_too(): + """The deployment declares its parameters in a multipart body, not a query.""" + params = _by_name(operation_params("docstudio", "execute")) + assert params["tags"].type == "string" + assert params["timeout"].type == "integer" + assert params["presigned_urls"].array is True + # `null | string` in the spec: the null branch carries nothing for a flag. + assert params["llm_profile_id"].type == "string" + assert params["llm_profile_id"].nullable is True + + +def test_a_required_body_parameter_stays_required(): + params = _by_name(operation_params("llmwhisperer", "webhook_post")) + assert {p.name for p in params.values() if p.required} == { + "url", + "auth_token", + "webhook_name", + } + + +def test_the_uploaded_document_is_not_a_flag(): + """The binary body is the document itself, which the command takes as an + argument.""" + assert "body" not in _by_name(operation_params("llmwhisperer", "extract")) + assert find_operation("llmwhisperer", "extract")["method"] == "post" + + +def test_an_unknown_operation_names_itself(): + with pytest.raises(KeyError, match="whisper_sideways"): + find_operation("llmwhisperer", "whisper_sideways") + + +# --------------------------------------------------------------------------- # +# Intersecting the spec with the published client +# --------------------------------------------------------------------------- # + + +def test_only_parameters_the_client_accepts_become_flags(): + """A flag the client cannot accept raises TypeError at the call instead of + reaching the API, so it is not offered at all.""" + spec = set(_by_name(operation_params("llmwhisperer", "extract"))) + derived = set( + _by_name( + derive_params( + "llmwhisperer", "extract", client_method=LLMWhispererClientV2.whisper + ) + ) + ) + assert derived < spec + # In URL mode the URL travels in the body, and saying so is the client's + # decision, not a caller's. + assert spec - derived == {"url_in_post"} + + +def test_the_clients_default_wins_over_the_specs(): + """What a caller gets by omitting a flag is the client's default, since the + client sends its own value regardless of the spec's.""" + spec = _by_name(operation_params("llmwhisperer", "extract")) + derived = _by_name( + derive_params( + "llmwhisperer", "extract", client_method=LLMWhispererClientV2.whisper + ) + ) + assert spec["line_splitter_tolerance"].default == 0.75 + assert derived["line_splitter_tolerance"].default == 0.4 + + +def test_every_deployment_parameter_survives_the_intersection(): + derived = _by_name( + derive_params( + "docstudio", "execute", client_method=APIDeploymentsClient.structure_file + ) + ) + assert "tags" in derived and "hitl_queue_name" in derived + + +def test_excluded_parameters_do_not_become_flags(): + derived = _by_name( + derive_params( + "llmwhisperer", + "extract", + client_method=LLMWhispererClientV2.whisper, + exclude=("use_webhook",), + ) + ) + assert "use_webhook" not in derived + + +# --------------------------------------------------------------------------- # +# Help text +# --------------------------------------------------------------------------- # + + +def test_help_comes_from_the_clients_docstring(): + """The specs carry no parameter descriptions; the clients document every + parameter, so that is where the text comes from.""" + derived = _by_name( + derive_params( + "llmwhisperer", "extract", client_method=LLMWhispererClientV2.whisper + ) + ) + assert "language" in derived["lang"].description.lower() + + +def test_the_docstrings_own_restated_sentences_are_dropped(): + """The default and the allowed values are rendered from the signature and the + spec; printing the docstring's copies too shows each twice and disagrees the + moment either drifts.""" + described = docstring_params(LLMWhispererClientV2.whisper) + assert not described["lang"].endswith('Defaults to "eng".') + # A default that itself contains a period, which is where a sentence-shaped + # match stops early and leaves half of it behind. + assert not described["checkbox_confidence_threshold"].endswith("Defaults to 0.3.") + assert described["mode"] == "The processing mode." + assert described["tag"] == "The tag for the document." + + +def test_the_spec_wins_over_the_docstring_and_the_overlay_wins_over_both(monkeypatch): + """Three sources can describe one flag, and only the most specific should + show. Today no spec parameter carries a description, so the precedence is + unexercised until one does -- which is when it would silently invert.""" + described = Param("lang", "string", description="From the spec.") + monkeypatch.setattr(params_module, "operation_params", lambda *_: [described]) + + derived = derive_params( + "llmwhisperer", "extract", client_method=LLMWhispererClientV2.whisper + ) + assert derived[0].description == "From the spec." + assert click_option(derived[0], {}).help.startswith("From the spec.") + assert click_option(derived[0], {"lang": {"help": "From the overlay."}}).help == ( + "From the overlay." + ) + + +def test_a_multi_line_description_is_joined(): + text = docstring_params(LLMWhispererClientV2.whisper)["word_confidence_threshold"] + assert "\n" not in text and "confidence" in text + + +def test_the_default_is_reported_in_help(): + param = Param("mode", "string", default="form", description="The mode.") + assert click_option(param, {}).help == "The mode. [default: form]" + + +# --------------------------------------------------------------------------- # +# Building Click options +# --------------------------------------------------------------------------- # + + +def test_a_boolean_gets_a_paired_flag_defaulting_to_neither(): + """`is_flag` cannot turn off a parameter that defaults to on, and cannot + distinguish "not passed" from "passed false".""" + option = click_option(Param("allow_rotated_text", "boolean", default=True), {}) + assert option.secondary_opts == ["--no-allow-rotated-text"] + assert option.default is None + + +def test_no_option_carries_a_value_by_default(): + """A default written into the option would be sent on every call, pinning a + value the client or server would otherwise choose.""" + for param in derive_params( + "llmwhisperer", "extract", client_method=LLMWhispererClientV2.whisper + ): + assert click_option(param, {}).default is None + + +def test_choices_come_from_the_spec_unless_the_overlay_narrows_them(): + """A wrong value must fail before the request, not after -- and the list it + is checked against is the service's own, not a copy that can fall behind.""" + spec_declared = _by_name(operation_params("llmwhisperer", "extract"))["mode"] + assert "excel" in spec_declared.choices + assert click_option(spec_declared, {}).type.choices == spec_declared.choices + + option = click_option(spec_declared, {"mode": {"choices": ["form", "table"]}}) + assert isinstance(option.type, click.Choice) + assert option.type.choices == ("form", "table") + + +def test_an_array_becomes_a_repeatable_option(): + option = click_option(Param("presigned_urls", "string", array=True), {}) + assert option.multiple is True + + +def test_types_map_onto_click_types(): + assert click_option(Param("n", "integer"), {}).type is click.INT + assert click_option(Param("x", "number"), {}).type is click.FLOAT + assert click_option(Param("s", "string"), {}).type is click.STRING + + +def test_a_required_parameter_stays_required(): + assert click_option(Param("url", "string", required=True), {}).required is True + + +# --------------------------------------------------------------------------- # +# Choosing what to send +# --------------------------------------------------------------------------- # + + +def test_falsy_values_are_sent(): + """0, false and "" are choices. A truthiness filter eats them and hands the + decision back to the server without telling anyone.""" + assert requested({"a": 0, "b": False, "c": "", "d": 0.0}) == { + "a": 0, + "b": False, + "c": "", + "d": 0.0, + } + + +def test_unpassed_values_are_not_sent(): + assert requested({"a": None, "b": (), "c": 1}) == {"c": 1} + + +def test_dropped_names_are_not_sent(): + assert requested({"a": 1, "b": 2}, drop=("b",)) == {"a": 1} diff --git a/tests/test_poll.py b/tests/test_poll.py new file mode 100644 index 0000000..3c08471 --- /dev/null +++ b/tests/test_poll.py @@ -0,0 +1,247 @@ +"""The `--wait` engine, driven by a fake clock and fake responses. No network.""" + +from __future__ import annotations + +import json + +import pytest + +from unstract_cli.core.errors import ExitCode +from unstract_cli.core.poll import ( + CLIError, + PollSpec, + extract_handle, + extract_status, + persist, + preflight, + wait_for_completion, +) + +SPEC = PollSpec( + handle_field="whisper_hash", + terminal_success=("processed",), + terminal_failure=("error",), + status_field=("status", "execution_status"), +) + + +class Clock: + """Monotonic clock that only advances when the engine sleeps.""" + + def __init__(self) -> None: + self.t = 0.0 + self.slept: list[float] = [] + + def now(self) -> float: + return self.t + + def sleep(self, seconds: float) -> None: + self.slept.append(seconds) + self.t += seconds + + +def responses(*payloads): + """A poll callable returning each payload in turn, then repeating the last.""" + queue = list(payloads) + calls: list[str] = [] + + def poll(handle: str): + calls.append(handle) + return queue.pop(0) if len(queue) > 1 else queue[0] + + poll.calls = calls + return poll + + +def test_polls_until_terminal_success(): + clock = Clock() + poll = responses( + {"status": "processing"}, + {"status": "processing"}, + {"status": "processed", "n": 1}, + ) + out = wait_for_completion( + initial={"whisper_hash": "h1"}, + spec=SPEC, + poll=poll, + interval=3, + sleep=clock.sleep, + now=clock.now, + ) + assert out == {"status": "processed", "n": 1} + assert poll.calls == ["h1", "h1", "h1"] + assert clock.slept == [3, 3] + + +def test_terminal_state_comes_from_the_body_not_the_http_status(): + # The deployment API returns HTTP 422 while still executing; only the body's + # status decides, so this reaches COMPLETED without any status-code input. + spec = PollSpec( + handle_field="execution_id", + terminal_success=("COMPLETED",), + terminal_failure=("ERROR",), + status_field=("status", "execution_status"), + ) + clock = Clock() + out = wait_for_completion( + initial={"message": {"execution_id": "e1", "execution_status": "PENDING"}}, + spec=spec, + poll=responses({"status": "EXECUTING"}, {"status": "COMPLETED"}), + sleep=clock.sleep, + now=clock.now, + ) + assert out == {"status": "COMPLETED"} + + +def test_terminal_failure_raises_with_the_handle_attached(): + clock = Clock() + with pytest.raises(CLIError) as excinfo: + wait_for_completion( + initial={"whisper_hash": "h1"}, + spec=SPEC, + poll=responses({"status": "error", "detail": "bad page"}), + sleep=clock.sleep, + now=clock.now, + ) + err = excinfo.value + assert err.exit_code is ExitCode.VALIDATION + assert err.to_dict()["whisper_hash"] == "h1" + assert err.to_dict()["details"]["detail"] == "bad page" + + +def test_timeout_carries_the_handle_so_work_is_resumable(): + clock = Clock() + with pytest.raises(CLIError) as excinfo: + wait_for_completion( + initial={"whisper_hash": "h1"}, + spec=SPEC, + poll=responses({"status": "processing"}), + interval=5, + timeout=12, + sleep=clock.sleep, + now=clock.now, + ) + err = excinfo.value + assert err.exit_code is ExitCode.TIMEOUT + payload = err.to_dict() + assert payload["whisper_hash"] == "h1" + assert payload["last_status"] == "processing" + assert "Resume" in payload["hint"] + # The last sleep is clipped so the wait lasts exactly as long as asked. + assert clock.slept == [5, 5, 2] + assert clock.now() == 12 + + +def test_missing_handle_returns_the_initial_response_unpolled(): + poll = responses({"status": "processed"}) + out = wait_for_completion( + initial={"no_handle_here": True}, spec=SPEC, poll=poll, sleep=Clock().sleep + ) + assert out == {"no_handle_here": True} + assert poll.calls == [] + + +def test_status_changes_are_reported_once_each(): + clock = Clock() + seen: list[str | None] = [] + wait_for_completion( + initial={"whisper_hash": "h1"}, + spec=SPEC, + poll=responses( + {"status": "accepted"}, + {"status": "processing"}, + {"status": "processing"}, + {"status": "processed"}, + ), + on_status=seen.append, + sleep=clock.sleep, + now=clock.now, + ) + assert seen == ["accepted", "processing", "processed"] + + +def test_retrieve_step_runs_after_terminal_success(): + clock = Clock() + out = wait_for_completion( + initial={"whisper_hash": "h1"}, + spec=SPEC, + poll=responses({"status": "processed"}), + retrieve=lambda handle: {"result_for": handle}, + sleep=clock.sleep, + now=clock.now, + ) + assert out == {"result_for": "h1"} + + +def test_save_persists_the_retrieved_result_before_returning(tmp_path): + target = tmp_path / "out" / "result.json" + on_disk: list[bool] = [] + + def retrieve(handle): + return {"text": "extracted"} + + out = wait_for_completion( + initial={"whisper_hash": "h1"}, + spec=SPEC, + poll=responses({"status": "processed"}), + retrieve=retrieve, + save=target, + # Observed from inside the engine, before the caller is handed anything: + # asserting after the return passes for either ordering. + on_saved=lambda path: on_disk.append(path.exists()), + sleep=Clock().sleep, + ) + assert on_disk == [True] + assert json.loads(target.read_text()) == out + + +def test_an_unwritable_save_target_is_refused_before_anything_is_read(tmp_path): + blocker = tmp_path / "not-a-dir" + blocker.write_text("") + + with pytest.raises(CLIError) as caught: + preflight(blocker / "result.json") + + assert caught.value.exit_code is ExitCode.USAGE + assert "nothing is lost" in (caught.value.hint or "") + + +def test_a_failed_save_carries_the_result_it_could_not_write(tmp_path): + """By this point the service has served the result and will not again, so + the payload has to leave through the error.""" + blocker = tmp_path / "not-a-dir" + blocker.write_text("") + + with pytest.raises(CLIError) as caught: + persist(blocker / "result.json", {"result_text": "IRREPLACEABLE"}) + + assert caught.value.exit_code is ExitCode.SAVE_FAILED + assert caught.value.details == {"result_text": "IRREPLACEABLE"} + + +def test_a_save_leaves_no_temporary_file_behind(tmp_path): + target = persist(tmp_path / "out.json", {"a": 1}) + assert [p.name for p in tmp_path.iterdir()] == [target.name] + + +def test_persist_writes_text_payloads_unwrapped(tmp_path): + target = persist(tmp_path / "a.txt", "plain extracted text") + assert target.read_text() == "plain extracted text" + + +@pytest.mark.parametrize( + "payload", + [ + {"status": "processed"}, + {"message": {"status": "processed"}}, + {"data": {"status": "processed"}}, + {"result": {"status": "processed"}}, + ], +) +def test_status_is_found_one_level_into_the_common_envelopes(payload): + assert extract_status(payload) == "processed" + + +def test_handle_is_found_one_level_in_too(): + assert extract_handle({"message": {"execution_id": "e1"}}, "execution_id") == "e1" + assert extract_handle({"nothing": 1}, "execution_id") is None diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..70ddebb --- /dev/null +++ b/uv.lock @@ -0,0 +1,376 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "anyio" +version = "4.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/4a/ecbd131485c07fcdfad54e28946d513e3da22ef3b4bd854dcafae54ec739/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0", size = 319300, upload-time = "2026-07-07T14:33:15.666Z" }, + { url = "https://files.pythonhosted.org/packages/ec/96/5d9364e3342d69f3a045e1777bc47c85c383e6e9466d561b33fdb419d1f9/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9", size = 215802, upload-time = "2026-07-07T14:33:17.031Z" }, + { url = "https://files.pythonhosted.org/packages/4b/4c/5361f9aa7f2cb58d94f2ab831b3d493f69efb1d239654b4744e3c09527cb/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44", size = 237171, upload-time = "2026-07-07T14:33:18.576Z" }, + { url = "https://files.pythonhosted.org/packages/50/78/ce342ca4ff30b2eb49fe6d9578df85974f90c67d294113e94efdd9664cbd/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9", size = 233075, upload-time = "2026-07-07T14:33:20.084Z" }, + { url = "https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd", size = 224256, upload-time = "2026-07-07T14:33:21.747Z" }, + { url = "https://files.pythonhosted.org/packages/87/3a/ad914516df7e358a81aae018caa5e0470ba827fa6d763b1d2e87d920a5f6/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84", size = 208784, upload-time = "2026-07-07T14:33:23.313Z" }, + { url = "https://files.pythonhosted.org/packages/d7/74/3c12f9755717dfe5c5c87da63f35d765fa0c00382ec26bf23f7fae34f2ba/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b", size = 219928, upload-time = "2026-07-07T14:33:24.814Z" }, + { url = "https://files.pythonhosted.org/packages/33/9a/895095b83e7907abd6d3d99aad3a38ad0d9686cc186cb0c94c24320fe63e/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde", size = 218489, upload-time = "2026-07-07T14:33:26.42Z" }, + { url = "https://files.pythonhosted.org/packages/a1/34/ef5c05f412f42520d7709b7d3784d19640839eb7366ded1755511585429f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39", size = 210267, upload-time = "2026-07-07T14:33:27.952Z" }, + { url = "https://files.pythonhosted.org/packages/83/dc/9b29fa4412b318bf3bfea985c35d67eb55e04b59a7c3f2237168b0e0be6f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62", size = 226030, upload-time = "2026-07-07T14:33:29.397Z" }, + { url = "https://files.pythonhosted.org/packages/0e/42/6dbc00b8cd16011691203e33570fa42ed5746599a2e878112d16eab403a3/charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642", size = 151185, upload-time = "2026-07-07T14:33:30.781Z" }, + { url = "https://files.pythonhosted.org/packages/80/cc/f920afd1a23c58ccd53c1d36085a71893a4737ff5e66e0371efab6809850/charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0", size = 162557, upload-time = "2026-07-07T14:33:32.176Z" }, + { url = "https://files.pythonhosted.org/packages/f0/e6/0386d43a261ff4e4b30c5857af7df877254b46bec7b9d1b74b6bf969a90b/charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2", size = 152665, upload-time = "2026-07-07T14:33:33.711Z" }, + { url = "https://files.pythonhosted.org/packages/b2/06/97ec2aeae780b31d742b6352218b43841a6871e2564578ca522dce4a45c3/charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614", size = 317688, upload-time = "2026-07-07T14:33:35.408Z" }, + { url = "https://files.pythonhosted.org/packages/d0/39/8ff066c672434225f8d25f8b739f992af250944392173dcc88362681c9bf/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698", size = 214982, upload-time = "2026-07-07T14:33:36.996Z" }, + { url = "https://files.pythonhosted.org/packages/92/8f/3a47a3667c83c2df9483d91644c6c107de3bf8874aa1793da9d3012eb986/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b", size = 236460, upload-time = "2026-07-07T14:33:38.536Z" }, + { url = "https://files.pythonhosted.org/packages/f1/60/b22cdbee7e4013dab8b0d7647fc6181120fbbbc8f7025c226d15bd5a47fc/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9", size = 232003, upload-time = "2026-07-07T14:33:40.059Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f8/72eb13dcabe7257035cea8aefd922caad2f110d252bf9f67c4c2ca763aee/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33", size = 223149, upload-time = "2026-07-07T14:33:41.631Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3e/faee8f9de92b14ee1198e9163252bb15efee7301b31256a3b6d9ebfdd0dd/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63", size = 207901, upload-time = "2026-07-07T14:33:43.209Z" }, + { url = "https://files.pythonhosted.org/packages/3a/25/45f30093ae27dd7b92a793b61882a38685f993700113ca36e0c9c14965e1/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0", size = 219176, upload-time = "2026-07-07T14:33:44.725Z" }, + { url = "https://files.pythonhosted.org/packages/48/18/c8f397329c35e32f6a837e488986f4ae03bd2abebc453b48714991630c2f/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe", size = 217356, upload-time = "2026-07-07T14:33:46.192Z" }, + { url = "https://files.pythonhosted.org/packages/86/7e/5ce0bba863470fd1902d5e5843968951bddf38abe4742fc97116ef4598b3/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35", size = 209614, upload-time = "2026-07-07T14:33:47.705Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ef/2473d3c4d869155be4af1191111d59c4d5c4e0173026f7e85b176e23bf65/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8", size = 224991, upload-time = "2026-07-07T14:33:49.238Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a3/53ddae3db108a088156aa8ddfafd411ebbc1340f48c5573f697b27f69a39/charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9", size = 150622, upload-time = "2026-07-07T14:33:50.711Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ef/6953a77c7cf2c2ff9998e6f575ab3e380119f100223381565a4f94c1f836/charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115", size = 161947, upload-time = "2026-07-07T14:33:52.197Z" }, + { url = "https://files.pythonhosted.org/packages/6e/fb/d560d1d1555debbfe7849d9cac6145c1b537709d79576bf22557ed803b82/charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012", size = 152594, upload-time = "2026-07-07T14:33:53.486Z" }, + { url = "https://files.pythonhosted.org/packages/7e/8d/496817fa0944239ecae662dd57ea765cfeaec6a735f9f025d4b7b72e7143/charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380", size = 317253, upload-time = "2026-07-07T14:33:54.994Z" }, + { url = "https://files.pythonhosted.org/packages/2b/f9/ef4a69ea338ad3c0deceea0f5f7d2380ae8b52132b06d652cb0d2cd86706/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9", size = 215898, upload-time = "2026-07-07T14:33:56.334Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e7/5ddfd76fc061eb52de219658a4aa431cbacadf0a0219c8854f00da50d289/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4", size = 236718, upload-time = "2026-07-07T14:33:57.9Z" }, + { url = "https://files.pythonhosted.org/packages/49/ba/768fa3f36048d81c477a0ce61f813bc1454d80917ccfe550abd9f44f5e24/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a", size = 232519, upload-time = "2026-07-07T14:33:59.811Z" }, + { url = "https://files.pythonhosted.org/packages/f4/c4/b3e049d2aa3766180c78507110543d9d50894cc97f57de543f1be521dcdc/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046", size = 223143, upload-time = "2026-07-07T14:34:01.517Z" }, + { url = "https://files.pythonhosted.org/packages/19/79/55c32d06d76ae4feafe053f061f3e3ab70bcf19f4007797ce8c3efda7830/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81", size = 206742, upload-time = "2026-07-07T14:34:03.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/e0/47c079dd82d217c807479cd59ffd30af56307ea31c108b75758970459ad3/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917", size = 219191, upload-time = "2026-07-07T14:34:04.657Z" }, + { url = "https://files.pythonhosted.org/packages/42/ab/b9bc2e77d6b44a7e46ef62ec5cac1c9a6ba7b9135a5d560f002696ec9995/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41", size = 218328, upload-time = "2026-07-07T14:34:06.115Z" }, + { url = "https://files.pythonhosted.org/packages/f1/78/c9c71d599f5aa2d42bcdd35cbbd46d7f535351a57e40ff7d8e5a7e219401/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1", size = 207406, upload-time = "2026-07-07T14:34:07.554Z" }, + { url = "https://files.pythonhosted.org/packages/f6/39/c914445c321a845097ce4f6ac7de9a18228a77b766272125a1ce00d851eb/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf", size = 225157, upload-time = "2026-07-07T14:34:09.061Z" }, + { url = "https://files.pythonhosted.org/packages/9b/f2/c0d4b8508565a36bc5c624e88ed297f5b0b1095011034d7f5b83a69908b5/charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48", size = 151095, upload-time = "2026-07-07T14:34:10.901Z" }, + { url = "https://files.pythonhosted.org/packages/49/fd/a1d26144398c67486422a72bf5812cda22cb4ccfcd95a290fb41ceb4b8e2/charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b", size = 162796, upload-time = "2026-07-07T14:34:12.47Z" }, + { url = "https://files.pythonhosted.org/packages/20/95/d75e82f8ce9fd323ebf059c16c9aadefb22a1ecde13b7840b35835e4886c/charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519", size = 153334, upload-time = "2026-07-07T14:34:14.044Z" }, + { url = "https://files.pythonhosted.org/packages/00/5e/17398df3a139985ba9d11ed072531986f408c8fca952835ef1ab1820c02b/charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198", size = 338848, upload-time = "2026-07-07T14:34:15.688Z" }, + { url = "https://files.pythonhosted.org/packages/cd/91/7253a32e86b7e1d1239b1b36ba6dd0f021a21107ab33054b53119cc083b9/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32", size = 223022, upload-time = "2026-07-07T14:34:17.248Z" }, + { url = "https://files.pythonhosted.org/packages/cb/32/2e64bd2be10e89c61e57ebe6a93fd98ae88eb7ebe414b5121f22c96c69eb/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632", size = 241590, upload-time = "2026-07-07T14:34:18.813Z" }, + { url = "https://files.pythonhosted.org/packages/3d/ef/d96ec496cfea0c21db43b0ad03891308b02388d054cc902cf0e5a1ad6a88/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf", size = 239584, upload-time = "2026-07-07T14:34:20.52Z" }, + { url = "https://files.pythonhosted.org/packages/d4/ce/9af95f7876194bd7a14e3dfe4a4de2e0bff02666a3910d72beafd06cc297/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990", size = 230224, upload-time = "2026-07-07T14:34:22.189Z" }, + { url = "https://files.pythonhosted.org/packages/52/94/af74dde74a3996bd959c350709bfe50e297823d70a8c1cbd54b838880863/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d", size = 212667, upload-time = "2026-07-07T14:34:23.857Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f0/f1c4fe746c395922961b5916ed1d7d6e7d4c84851d19ed43cc89980ec953/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e", size = 227179, upload-time = "2026-07-07T14:34:25.586Z" }, + { url = "https://files.pythonhosted.org/packages/e4/56/6c745619ac397e8871e2bcd3cea1eec86b877488f33888b3aef5c3ed506e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c", size = 225372, upload-time = "2026-07-07T14:34:27.212Z" }, + { url = "https://files.pythonhosted.org/packages/78/ad/98aae8630ac71f16711968e38a5acfecce41b778bf2f0312851020f565a8/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2", size = 215222, upload-time = "2026-07-07T14:34:28.774Z" }, + { url = "https://files.pythonhosted.org/packages/f7/40/9593d54209765207a7f11073c06494c1721e4ca4a0a426c597679bf7f91e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534", size = 231958, upload-time = "2026-07-07T14:34:30.345Z" }, + { url = "https://files.pythonhosted.org/packages/b1/27/693ee5e8a18191eb38647360c51cd505013e2bd3b366aa43fd5344c21e3c/charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226", size = 155580, upload-time = "2026-07-07T14:34:31.884Z" }, + { url = "https://files.pythonhosted.org/packages/80/3f/bd97d3d9c613013d07cb7733d299385b41df37f0471310f5a73dc359f0b8/charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177", size = 167620, upload-time = "2026-07-07T14:34:33.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c6/eee9dca4439b1061f76373f06ea855678cc4a64c1c3c90b50e479edbb8eb/charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501", size = 158037, upload-time = "2026-07-07T14:34:35.018Z" }, + { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, +] + +[[package]] +name = "click" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "llmwhisperer-client" +version = "2.7.0" +source = { git = "https://github.com/Zipstack/llm-whisperer-python-client?rev=7f64caf#7f64caf5893370e0d472c50df3df39ef198fb37b" } +dependencies = [ + { name = "attrs" }, + { name = "httpx" }, + { name = "requests" }, + { name = "tenacity" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + +[[package]] +name = "ruff" +version = "0.16.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/73/e1/4508a569211b35599016e84ba65c1a992b7a4004b4b6c4bea02a851cba1b/ruff-0.16.2.tar.gz", hash = "sha256:c3d7828d12e8927a6fc65fe38e2c2541b9e762d360a1786d752cb1b8883b3c9c", size = 4885811, upload-time = "2026-08-07T13:31:01.432Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/57/db19951540f98859c956b50bdb4d31089b4d91e9f15e2968e7d5193806d5/ruff-0.16.2-py3-none-linux_armv6l.whl", hash = "sha256:3c8de4cf2181f01d57946d87d777aa52916976fc09942aed89938fab5e013318", size = 10847925, upload-time = "2026-08-07T13:30:14.468Z" }, + { url = "https://files.pythonhosted.org/packages/13/5a/995fe85a8470d3e391ac0f7fa8054bb454eaf33ee138196d6172ed1079c0/ruff-0.16.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9a48cc05c6fbc811ca81b5d7ba95375affea6582d1b8024e455e41afbbf55344", size = 11072662, upload-time = "2026-08-07T13:30:18.143Z" }, + { url = "https://files.pythonhosted.org/packages/32/53/370d767c61c71a971a4ace36703a7ecd8c393956349a7325d7fab2b56827/ruff-0.16.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a2c0d14fcbb26c91f0f867a6dc9bd71bbc30b1b6151829c884f23faeab2e5700", size = 10566771, upload-time = "2026-08-07T13:30:20.899Z" }, + { url = "https://files.pythonhosted.org/packages/85/d6/9d96948caf5a632be62d62202d5ec914d6856f204fd79eb036e5915e79ea/ruff-0.16.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:335c621622c4650330be50842561c6586ac6971bb8ab5407fe34dcc9efb16bbe", size = 10975825, upload-time = "2026-08-07T13:30:23.517Z" }, + { url = "https://files.pythonhosted.org/packages/3b/92/ea87129b3414acb0b5770563779c51804d37ac67675c7ba35447ddb14773/ruff-0.16.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:20e66910f2c37cc753f9ef6580c914a621b80c4fa3549d3e3521e29d0f5bfc3f", size = 10649437, upload-time = "2026-08-07T13:30:26.097Z" }, + { url = "https://files.pythonhosted.org/packages/ac/43/f8f291dcd4af5bb7872b74fdfa41a7cd7c856ca1d4069670971cf1b9f5cb/ruff-0.16.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7e36fbfba65510548156902bcf1350a979a958ce0347ce0f90d73894036b39f", size = 11446761, upload-time = "2026-08-07T13:30:28.752Z" }, + { url = "https://files.pythonhosted.org/packages/71/4a/ef991fb2fcf516ab71f0808adcdd8da5e18c8cde447f4ceaf5f47a5132a5/ruff-0.16.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f0eab35f80df8f134aae5d1630e751901321d317cc8e50dc39e36fa3ed34cd12", size = 12336364, upload-time = "2026-08-07T13:30:31.468Z" }, + { url = "https://files.pythonhosted.org/packages/f3/24/f615e74f307e6ca0e56a482872477b856c70d530aa356abfb6dfe5ca8a80/ruff-0.16.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40ea8c0594feb894e89c8c61ab9c103d38b0ea72dfde6c594107147ca31b1140", size = 11630720, upload-time = "2026-08-07T13:30:34.426Z" }, + { url = "https://files.pythonhosted.org/packages/c5/d3/8ef50149e8412a77f7ab409efdef0e2b23803707a3863da4fc64cb23d459/ruff-0.16.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab3d62dde0b19facdd632008cc4827fc28ada7736c6bd35ab6f1050f0bfed53f", size = 11466130, upload-time = "2026-08-07T13:30:36.958Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a7/a19334985c4dea8c381981fa252cd854c7ee52dc4b1686dc16f4a911c702/ruff-0.16.2-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:e43e1f5b8388da9eca1b9e88328d47a5cec794633ccf6f7484ac2dd15eee92c0", size = 11523634, upload-time = "2026-08-07T13:30:39.822Z" }, + { url = "https://files.pythonhosted.org/packages/6e/6c/96d192b0e742412ceda08c0a50f9669b253dde9fd6a60ea1a10c9fa79a63/ruff-0.16.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c24788a980581e1d7ea3a0cbe4344c4fbeb0a6a9b1f4713aa46bb104f8294690", size = 10949807, upload-time = "2026-08-07T13:30:42.745Z" }, + { url = "https://files.pythonhosted.org/packages/fa/51/e26599ceca11e79ee255c7df515995561edf87e9ca1893284e44d98f5a86/ruff-0.16.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:81806b08329130005dd4a8a8394a0c9da8c6f4cafb16ba438d2a2ee6a18bedf1", size = 10646891, upload-time = "2026-08-07T13:30:45.522Z" }, + { url = "https://files.pythonhosted.org/packages/68/01/800c4b1f97bc8d7c6029e06b1f20473a3cf1e13c4933d8f3342add83fc55/ruff-0.16.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:4ce4e02bad779bef557f541a1b31f20d6abeae1cc05ed1b1ac019d4ffd1044c8", size = 11162063, upload-time = "2026-08-07T13:30:48.131Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d0/1477ea50fc5a0d4b0b71d1d63d50770bdd794d90b43e37a7618e63ec9894/ruff-0.16.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e0422abdf70070255fc4073ce9dfc814cc03db577013761ddd09bc1e4a9a4fbd", size = 11556038, upload-time = "2026-08-07T13:30:50.686Z" }, + { url = "https://files.pythonhosted.org/packages/b8/76/a7776f32048d991e16d4fa8ff91790b877342d3596cc3ed04acdbf1aaedc/ruff-0.16.2-py3-none-win32.whl", hash = "sha256:bf3a63d78fb39f4bf5ac8ae52051c5520505301abe19ba4e204c453b3f09bb0b", size = 10872850, upload-time = "2026-08-07T13:30:53.471Z" }, + { url = "https://files.pythonhosted.org/packages/00/0d/929c800d920e61397d82a01b60bffc68da3052c17d31de59efaad2e4ed75/ruff-0.16.2-py3-none-win_amd64.whl", hash = "sha256:bcabe2f6d0fc7819f1431793005af4e4de7371927d037345bf941252b195b9fa", size = 12023338, upload-time = "2026-08-07T13:30:56.193Z" }, + { url = "https://files.pythonhosted.org/packages/5b/6c/93e26c22c5f78ff87363e07da49c84955affbeb1098bd1936bf3b3f293bf/ruff-0.16.2-py3-none-win_arm64.whl", hash = "sha256:d614e95cedf38a2053fd351c55b103ba30d017d61688fdbfd40ee0412852a99f", size = 11374065, upload-time = "2026-08-07T13:30:58.775Z" }, +] + +[[package]] +name = "tenacity" +version = "9.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, +] + +[[package]] +name = "tomli-w" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/75/241269d1da26b624c0d5e110e8149093c759b7a286138f4efd61a60e75fe/tomli_w-1.2.0.tar.gz", hash = "sha256:2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021", size = 7184, upload-time = "2025-01-15T12:07:24.262Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/18/c86eb8e0202e32dd3df50d43d7ff9854f8e0603945ff398974c1d91ac1ef/tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90", size = 6675, upload-time = "2025-01-15T12:07:22.074Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "unstract-cli" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "click" }, + { name = "llmwhisperer-client" }, + { name = "tomli-w" }, + { name = "unstract-client" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "click", specifier = ">=8.1,<9" }, + { name = "llmwhisperer-client", git = "https://github.com/Zipstack/llm-whisperer-python-client?rev=7f64caf" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, + { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.6" }, + { name = "tomli-w", specifier = ">=1.0" }, + { name = "unstract-client", git = "https://github.com/Zipstack/unstract-python-client?rev=a77ef6a" }, +] +provides-extras = ["dev"] + +[[package]] +name = "unstract-client" +version = "1.5.3" +source = { git = "https://github.com/Zipstack/unstract-python-client?rev=a77ef6a#a77ef6ae65d69a8290b5aa3fb6b13952a2084d45" } +dependencies = [ + { name = "attrs" }, + { name = "click" }, + { name = "httpx" }, + { name = "requests" }, + { name = "rich" }, + { name = "tenacity" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +]