From 91da98f739ac6524ea2fca045ac1a757dc3c65f9 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 02:48:13 +0530 Subject: [PATCH 01/14] fix: resolve linting issues and pyproject.toml duplicate key --- coverage.xml | 3 +-- pyproject.toml | 14 +++++--------- src/stash/cli/output.py | 1 - 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/coverage.xml b/coverage.xml index 95e69aa..2af7b68 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -114,7 +114,6 @@ - diff --git a/pyproject.toml b/pyproject.toml index 6be1b57..a6379c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,9 +40,11 @@ stash = "stash.cli:main" requires = ["setuptools>=68.0", "wheel"] build-backend = "setuptools.build_meta" -[tool.setuptools.packages.find] -where = ["src"] -include = ["stash*"] +[tool.setuptools] +packages = {where = ["src"], include = ["stash*"]} +entry-points = [ + "console_scripts = stash = stash.cli:main" +] [tool.ruff] target-version = "py311" @@ -56,12 +58,6 @@ ignore = [ fixable = ["ALL"] unfixable = ["T20", "ARG001", "ARG002"] -[tool.setuptools] -packages = {where = ["src"], include = ["stash*"]} -entry-points = [ - "console_scripts = stash = stash.cli:main" -] - [tool.ruff.format] quote-style = "double" indent-style = "space" diff --git a/src/stash/cli/output.py b/src/stash/cli/output.py index 2d2e185..841c1be 100644 --- a/src/stash/cli/output.py +++ b/src/stash/cli/output.py @@ -115,7 +115,6 @@ def format_timestamp(ts: float) -> str: def confirm(message: str, default: bool = False) -> bool: """Ask for confirmation.""" - suffix = " (y/n)" response = console.input(f"{message} (y/n): ").strip().lower() if not response: return default From b0fc9d4bec9c0f483312a9f5f68c23b98e049657 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 03:05:13 +0530 Subject: [PATCH 02/14] fix: migrate ruff config to [tool.ruff.lint] section --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 8bf7e9b..3e4bee1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,8 @@ entry-points = [ [tool.ruff] target-version = "py311" line-length = 120 + +[tool.ruff.lint] select = ["E", "F", "I", "UP", "B", "C4", "SIM", "T20", "ARG", "PTH", "ERA", "PD", "PL", "TRY", "NPY", "RSE", "RET"] ignore = [ "S101", "S106", "S311", "B008", From cf7893acc802cdc3201570a94bc490d6e42ce027 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 03:08:03 +0530 Subject: [PATCH 03/14] fix: correct setuptools packages.find config to fix pip install -e . --- pyproject.toml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3e4bee1..c4b8798 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,11 +40,9 @@ stash = "stash.cli:main" requires = ["setuptools>=84.0.0", "wheel"] build-backend = "setuptools.build_meta" -[tool.setuptools] -packages = {where = ["src"], include = ["stash*"]} -entry-points = [ - "console_scripts = stash = stash.cli:main" -] +[tool.setuptools.packages.find] +where = ["src"] +include = ["stash*"] [tool.ruff] target-version = "py311" From 42ec9f60b8fb1e1a3dd8ea4ad28cfa004b8bc010 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 03:09:56 +0530 Subject: [PATCH 04/14] fix: install setuptools and wheel for no-isolation build in Docker --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2d36c9c..22453e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ COPY pyproject.toml README.md ./ COPY src/ ./src/ # Build the package -RUN pip install --no-cache-dir build && \ +RUN pip install --no-cache-dir build setuptools wheel && \ python -m build --wheel --no-isolation # Runtime stage From a2060db1451eec334961844519c5026694a885a6 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 03:17:43 +0530 Subject: [PATCH 05/14] fix: add .dockerignore to exclude local build artifacts from docker context --- .dockerignore | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0853d3d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,23 @@ +.git +.github +.venv +venv +env +build +dist +*.egg-info +__pycache__ +*.py[cod] +.pytest_cache +.mypy_cache +.ruff_cache +.hypothesis +.coverage +coverage.xml +htmlcov +.stash +tests +docs +Dockerfile +.dockerignore +*.log \ No newline at end of file From 6804d1b8f32a5abe2217558ce4d6dac9bf60edf1 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 03:30:41 +0530 Subject: [PATCH 06/14] fix: use isolated build in Docker to avoid stale bundled setuptools --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 22453e8..056a7a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,8 +13,8 @@ COPY pyproject.toml README.md ./ COPY src/ ./src/ # Build the package -RUN pip install --no-cache-dir build setuptools wheel && \ - python -m build --wheel --no-isolation +RUN pip install --no-cache-dir build && \ + python -m build --wheel # Runtime stage FROM python:3.11-slim From 46847b0ea41bf1357f6ce22ada68f2694048566b Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 03:36:43 +0530 Subject: [PATCH 07/14] fix: run docker image without redundant stash arg (entrypoint already set) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2633ea..8cc7ef8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,7 +92,7 @@ jobs: docker build -t stash:test . - name: Test Docker image run: | - docker run --rm stash:test stash --help + docker run --rm stash:test --help concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 1866df8c6c934e47eb8e03b1fc3f813545bf37e9 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 03:03:44 +0530 Subject: [PATCH 08/14] docs: add complete documentation suite --- docs/architecture.md | 170 +++++++++++++++++++++++++++++++++++ docs/cli-reference.md | 166 ++++++++++++++++++++++++++++++++++ docs/configuration.md | 121 +++++++++++++++++++++++++ docs/index.md | 62 +++++++++++++ docs/installation.md | 70 +++++++++++++++ docs/providers/README.md | 102 +++++++++++++++++++++ docs/providers/discord.md | 101 +++++++++++++++++++++ docs/providers/telegram.md | 145 ++++++++++++++++++++++++++++++ docs/security.md | 140 +++++++++++++++++++++++++++++ docs/troubleshooting.md | 180 +++++++++++++++++++++++++++++++++++++ 10 files changed, 1257 insertions(+) create mode 100644 docs/architecture.md create mode 100644 docs/cli-reference.md create mode 100644 docs/configuration.md create mode 100644 docs/index.md create mode 100644 docs/installation.md create mode 100644 docs/providers/README.md create mode 100644 docs/providers/discord.md create mode 100644 docs/providers/telegram.md create mode 100644 docs/security.md create mode 100644 docs/troubleshooting.md diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..b812783 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,170 @@ +# Architecture + +## Overview + +Stash is a privacy-focused CLI storage system that uses third-party platforms as encrypted storage backends. The architecture is designed around a provider-agnostic core with pluggable storage backends. + +## Core Components + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Stash CLI │ +├─────────────────────────────────────────────────────────────┤ +│ Commands (put, get, ls, info, rm, verify, status, etc.) │ +├─────────────────────────────────────────────────────────────┤ +│ Storage Engine │ +│ ┌─────────────┐ ┌──────────────┐ ┌────────────────────┐ │ +│ │ Crypto │ │ Chunking │ │ Manifest │ │ +│ │ Engine │ │ Manager │ │ Manager │ │ +│ └─────────────┘ └──────────────┘ └────────────────────┘ │ +├─────────────────────────────────────────────────────────────┤ +│ Provider Abstraction Layer │ +│ ┌──────────────┐ ┌──────────────┐ ┌────────────────────┐ │ +│ │ Telegram │ │ Discord │ │ S3 / B2 / GDrive │ │ +│ │ Provider │ │ Provider │ │ (Planned) │ │ +│ └──────────────┘ └──────────────┘ └────────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ +``` + +## Core Modules + +### Crypto Engine (`src/stash/core/crypto.py`) +- **AES-256-GCM** encryption for chunks +- **HKDF-SHA256** key derivation +- Per-file encryption keys with HKDF-SHA256 derivation +- Per-chunk keys derived via HKDF from file key +- Password-based key wrapping with Argon2id (planned) + +### Chunking Manager (`src/stash/core/chunking.py`) +- Configurable chunk sizes (default 10MB) +- Streaming chunking for memory efficiency +- Automatic chunk boundary alignment +- Checksum verification (SHA-256) + +### Manifest Manager (`src/stash/core/manifest.py`) +- File metadata: ID, name, size, chunk list +- Chunk mapping: index → provider, remote_id, checksum +- Encryption parameters (algorithm, key size, nonce size) +- Distribution strategy tracking +- JSON serialization with binary data as hex + +### Metadata Store (`src/stash/core/metadata.py`) +- JSON-based local metadata storage +- SQLite backend (planned) +- Provider configuration management +- File indexing and lookup + +### Provider Abstraction (`src/stash/core/storage.py`) +- `StorageProvider` protocol defines interface +- `BaseStorageProvider` base class +- Provider registry for dynamic loading +- Automatic provider discovery + +## Provider Architecture + +Each provider implements `StorageProvider` interface: + +```python +class StorageProvider(Protocol): + async def initialize(self, config: ProviderConfig) -> None: ... + async def upload_chunk(self, chunk: Chunk, remote_path: str) -> RemoteRef: ... + async def download_chunk(self, remote_ref: RemoteRef) -> bytes: ... + async def delete_chunk(self, remote_ref: RemoteRef) -> None: ... + async def list_chunks(self, prefix: str) -> list[RemoteRef]: ... + def get_limits(self) -> ProviderLimits: ... + async def close(self) -> None: ... +``` + +### Provider Implementation Structure +``` +src/stash/providers/ +├── __init__.py # Provider registry +├── base.py # BaseStorageProvider abstract class +├── discord/ +│ ├── __init__.py +│ ├── auth.py # Discord OAuth/bot auth +│ ├── limits.py # Discord-specific limits +│ └── provider.py # DiscordProvider implementation +├── telegram/ +│ ├── __init__.py +│ ├── auth.py # Telegram bot auth +│ ├── limits.py # Telegram-specific limits +│ └── provider.py # TelegramProvider implementation +└── ... +``` + +## Data Flow + +### Upload Flow +``` +File → Encrypt → Chunk → Per-chunk encrypt → Upload to providers → Save manifest +``` + +1. **File Input** → Read file stream +2. **Encryption** → Generate file key, encrypt filename +4. **Chunking** → Split into configurable chunks (default 10MB) +5. **Per-Chunk Encryption** → HKDF-derived per-chunk key + AES-256-GCM +5. **Provider Upload** → Parallel upload to configured providers +7. **Manifest Creation** → Store metadata, chunk mappings, encryption params +8. **Persist** → Save manifest to local metadata store + +### Download Flow +``` +Manifest → Decrypt filename → Resolve chunks → Download from providers → Decrypt → Reconstruct +``` + +1. **Manifest Lookup** → Load file metadata +4. **Key Derivation** → Decrypt file key with password +4. **Filename Decryption** → Decrypt original filename +5. **Chunk Resolution** → Determine providers for each chunk +6. **Parallel Download** → Fetch chunks from providers +7. **Decrypt & Verify** → Decrypt chunks, verify checksums +8. **Reconstruction** → Stream decrypted chunks to output file + +## Distribution Strategies + +| Strategy | Description | +|----------|-------------| +| **Single** | All chunks on one provider | +| **Split** | Round-robin chunks across providers | +| **Balanced** | Distribute based on provider capacity | +| **Replicated** | Store each chunk on multiple providers | + +## Concurrency Model + +- **Async/await** throughout for I/O operations +- **Semaphore-based** concurrency control per provider +- **AsyncIO** for HTTP requests +- **ThreadPoolExecutor** for CPU-bound crypto operations +- Configurable concurrency per provider + +## Security Model + +### Threat Model +- **Trusted**: Local machine, user password +- **Untrusted**: Storage providers, network +- **Assumption**: Provider may be malicious/curious + +### Security Guarantees +- **Confidentiality**: AES-256-GCM encryption +- **Integrity**: GCM authentication tags + SHA-256 checksums +- **Forward Secrecy**: Per-file keys, per-chunk keys +- **Provider Isolation**: Providers cannot decrypt without user password + +### Key Hierarchy +``` +User Password + ↓ Argon2id (planned) / PBKDF2 +Master Key + ↓ HKDF-SHA256 +File Key (per file) + ↓ HKDF-SHA256 +Chunk Key (per chunk) +``` + +## Error Handling + +- **Retry Logic**: Exponential backoff with jitter +- **Circuit Breaker**: Per-provider failure tracking +- **Graceful Degradation**: Continue with available providers +- **Validation**: Input validation, checksum verification \ No newline at end of file diff --git a/docs/cli-reference.md b/docs/cli-reference.md new file mode 100644 index 0000000..a3c0e02 --- /dev/null +++ b/docs/cli-reference.md @@ -0,0 +1,166 @@ +# CLI Reference + +## Global Options + +| Option | Description | +|--------|-------------| +| `--repo, -r` | Repository path (default: current directory) | +| `--verbose, -v` | Verbose output | +| `--help, -h` | Show help | +| `--version` | Show version | + +## Commands + +### `stash init` +Initialize a new Stash repository. + +```bash +stash init [--repo PATH] [--force] +``` + +| Option | Description | +|--------|-------------| +| `--repo, -r` | Repository path (default: current directory) | +| `--force, -f` | Overwrite existing repository | + +### `stash provider` +Manage storage providers. + +#### `stash provider add` +Add a storage provider. + +```bash +stash provider add --type [options] +``` + +**Telegram:** +```bash +stash provider add tg --type telegram --token --chat-id +``` + +**Discord:** +```bash +stash provider add dc --type discord --token --channel-id +``` + +#### Options + +| Option | Description | +|--------|-------------| +| `--type, -t` | Provider type: `telegram`, `discord` | +| `--token` | Bot token (prompt if not provided) | +| `--chat-id` | Telegram chat ID | +| `--channel-id` | Discord channel ID | +| `--is-bot/--is-user` | Discord: bot vs user token | +| `--max-concurrent` | Max concurrent uploads (default: 3) | + +#### `stash provider list` +List configured providers. + +```bash +stash provider list +``` + +#### `stash provider remove` +Remove a storage provider. + +```bash +stash provider remove [--force] +``` + +### `stash put` +Store a file in Stash. + +```bash +stash put [options] +``` + +| Option | Description | +|--------|-------------| +| `--provider, -p` | Specific provider to use | +| `--chunk-size` | Chunk size in bytes (default: provider limit) | +| `--strategy` | Distribution: single, split, balanced, replicated | +| `--password` | Encryption password (prompt if not provided) | +| `--confirm/--no-confirm` | Skip confirmation prompt | + +### `stash get` +Retrieve a file from Stash. + +```bash +stash get [options] +``` + +| Option | Description | +|--------|-------------| +| `--output, -o` | Output path (default: current directory) | +| `--password` | Encryption password (prompt if not provided) | +| `--overwrite` | Overwrite existing file | + +### `stash ls` +List stored files. + +```bash +stash ls [options] +``` + +| Option | Description | +|--------|-------------| +| `--long, -l` | Show detailed information | + +### `stash info` +Show file metadata. + +```bash +stash info +``` + +### `stash rm` +Remove a stored file. + +```bash +stash rm [options] +``` + +| Option | Description | +|--------|-------------| +| `--force, -f` | Force removal without confirmation | +| `--remote/--local-only` | Also delete from remote providers | + +### `stash verify` +Verify file integrity. + +```bash +stash verify [--full] +``` + +| Option | Description | +|--------|-------------| +| `--full` | Download and verify all chunks | + +### `stash status` +Show overall repository status. + +```bash +stash status +``` + +## Global Options + +| Option | Description | +|--------|-------------| +| `--repo, -r` | Repository path (default: current directory) | +| `--verbose, -v` | Verbose output | +| `--help, -h` | Show help | +| `--version` | Show version | + +## Exit Codes + +| Code | Description | +|------|-------------| +| 0 | Success | +| 1 | General error | +| 2 | Invalid arguments | +| 3 | File not found | +| 4 | Authentication failed | +| 5 | Network error | +| 6 | Storage limit exceeded | \ No newline at end of file diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..2eeb9bd --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,121 @@ +# Configuration + +## Repository Configuration + +Each Stash repository stores its configuration in `.stash/config.json`: + +```json +{ + "version": 1, + "created_at": 1699999999.123, + "providers": { + "telegram": { + "type": "telegram", + "credentials": { + "token": "bot_token_here", + "chat_id": "-1001234567890" + }, + "settings": { + "max_concurrent": "3" + } + } + } +} +``` + +## Global Options + +| Option | Description | Default | +|--------|-------------|---------| +| `--repo, -r` | Repository path | Current directory | +| `--verbose, -v` | Verbose output | false | + +## Provider Configuration + +Each provider has specific credentials and settings: + +### Discord + +```bash +stash provider add discord \ + --token \ + --channel-id \ + --is-bot true \ + --max-concurrent 3 +``` + +| Setting | Description | Required | Default | +|---------|-------------|----------|---------| +| `token` | Bot token from Discord Developer Portal | Yes | - | +| `channel_id` | Channel ID to store files | Yes | - | +| `is_bot` | Use bot token (vs user token) | No | `true` | +| `max_concurrent` | Max concurrent uploads | No | `3` | + +### Telegram + +```bash +stash provider add telegram \ + --token \ + --chat-id \ + --max-concurrent 3 +``` + +| Setting | Description | Required | Default | +|---------|-------------|----------|---------| +| `token` | Bot token from @BotFather | Yes | - | +| `chat_id` | Chat/channel ID for storage | Yes | - | +| `max_concurrent` | Max concurrent uploads | No | `3` | + +## Global Settings + +Create `.stash/config.toml` for global defaults: + +```toml +[storage] +default_provider = "telegram" +default_chunk_size = 10485760 # 10MB +replication_factor = 1 + +[transfers] +upload_concurrency = 3 +download_concurrency = 3 +retry_count = 3 +retry_backoff = 1.0 + +[security] +auto_lock_timeout = 0 # 0 = never +key_derivation_iterations = 100000 + +[ui] +compact_mode = false +animations = true +progress_style = "bar" +``` + +## Environment Variables + +| Variable | Description | +|----------|-------------| +| `STASH_REPO` | Default repository path | +| `STASH_PASSWORD` | Default encryption password (not recommended) | +| `STASH_VERBOSE` | Enable verbose output | +| `DISCORD_TOKEN` | Default Discord bot token | +| `TELEGRAM_TOKEN` | Default Telegram bot token | + +## Provider Limits + +Each provider has built-in limits: + +| Provider | Max File Size | Max Chunk Size | Rate Limit | +|----------|---------------|----------------|------------| +| Discord | 25 MB | 10 MB | 5 req/s | +| Telegram | 20 MB | 10 MB | 30 req/s | + +## Encryption Settings + +| Setting | Default | Description | +|---------|---------|-------------| +| Algorithm | AES-256-GCM | Encryption algorithm | +| Key Size | 256 bits | Encryption key size | +| Chunk Key Derivation | HKDF-SHA256 | Per-chunk key derivation | +| Key Derivation | HKDF-SHA256 | Per-file key derivation | \ No newline at end of file diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..3c4407c --- /dev/null +++ b/docs/index.md @@ -0,0 +1,62 @@ +# Stash + +**Stash** is a privacy-focused CLI storage system that uses third-party platforms (Telegram, Discord) as encrypted storage backends. + +## Quick Start + +```bash +# Install +pip install stash + +# Initialize repository +stash init + +# Add storage provider +stash provider add telegram --token --chat-id + +# Store a file +stash put file.txt + +# Retrieve a file +stash get file.txt +``` + +## Documentation + +- [Installation](installation.md) +- [Configuration](configuration.md) +- [Providers](providers/README.md) +- [CLI Reference](cli-reference.md) +- [Architecture](architecture.md) +- [Security](security.md) +- [Troubleshooting](troubleshooting.md) + +## Features + +- **Client-side encryption**: AES-256-GCM with per-file keys +- **Multi-provider support**: Telegram, Discord (S3, B2, Google Drive planned) +- **Chunked storage**: Automatic chunking for large files +- **Multi-provider distribution**: Single, split, balanced, replicated strategies +- **Resumable uploads**: Resume interrupted transfers +- **Integrity verification**: SHA-256 checksums per chunk +- **Privacy-first**: Encryption happens locally, providers only see ciphertext + +## Providers + +| Provider | Max File Size | Chunk Size | Status | +|----------|---------------|------------|--------| +| Telegram | 20 MB | 10 MB | ✅ Stable | +| Discord | 25 MB | 10 MB | ✅ Stable | +| S3/MinIO | Unlimited | Configurable | 🚧 Planned | +| Google Drive | Unlimited | Configurable | 🚧 Planned | + +## Security + +- **AES-256-GCM** encryption per chunk +- **HKDF-SHA256** key derivation per chunk +- Per-file encryption keys with HKDF-SHA256 derivation +- Zero-knowledge: providers never see plaintext + +## License + +MIT License - see [LICENSE](../LICENSE) for details. \ No newline at end of file diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 0000000..dfe273d --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,70 @@ +# Installation + +## Prerequisites + +- Python 3.11+ +- pip (Python package manager) + +## Install from PyPI + +```bash +pip install stash +``` + +## Install from Source + +```bash +git clone https://github.com/Sparkleeop/Stashify +cd Stashify +pip install -e . +``` + +## Development Install + +```bash +git clone https://github.com/Sparkleeop/Stashify +cd Stashify +pip install -e ".[dev]" +``` + +## Verify Installation + +```bash +stash --help +stash --version +``` + +## Shell Completion + +```bash +# Bash +stash --install-completion bash + +# Zsh +stash --install-completion zsh + +# Fish +stash --install-completion fish +``` + +## Docker + +```bash +docker pull ghcr.io/sparkleeop/stash:latest +docker run --rm -v /path/to/repo:/repo ghcr.io/sparkleeop/stash:latest --repo /repo init +``` + +## Requirements + +- Python 3.11+ +- Dependencies are automatically installed via pip +- Optional: Docker for containerized deployment + +## Platform Support + +| OS | Status | +|------|--------| +| Linux | ✅ Fully supported | +| macOS | ✅ Fully supported | +| Windows | ✅ Fully supported | +| Docker | ✅ Supported | \ No newline at end of file diff --git a/docs/providers/README.md b/docs/providers/README.md new file mode 100644 index 0000000..b4ef25a --- /dev/null +++ b/docs/providers/README.md @@ -0,0 +1,102 @@ +# Storage Providers + +Stash uses a provider abstraction to support multiple storage backends. Each provider implements the `StorageProvider` interface. + +## Supported Providers + +| Provider | Status | Max File | Max Chunk | Rate Limit | +|----------|--------|----------|-----------|------------| +| [Telegram](telegram.md) | ✅ Stable | 20 MB | 10 MB | 30 req/s | +| [Discord](discord.md) | ✅ Stable | 25 MB | 10 MB | 5 req/s | +| S3/MinIO | 🚧 Planned | Unlimited | Configurable | AWS limits | +| Google Drive | 🚧 Planned | Unlimited | Configurable | API limits | +| S3/MinIO | 🚧 Planned | Unlimited | Configurable | AWS limits | +| Backblaze B2 | 🚧 Planned | Unlimited | Configurable | B2 limits | +| WebDAV | 🚧 Planned | Unlimited | Configurable | Server limits | +| Local FS | 🚧 Planned | Unlimited | Configurable | Disk I/O | + +## Adding a Provider + +```bash +stash provider add --type [options] +``` + +### List Providers +```bash +stash provider list +``` + +### Remove Provider +```bash +stash provider remove [--force] +``` + +## Provider Interface + +All providers implement the `StorageProvider` interface: + +```python +class StorageProvider(Protocol): + async def initialize(self, config: ProviderConfig) -> None: ... + async def upload_chunk(self, chunk: Chunk, remote_path: str) -> RemoteRef: ... + async def download_chunk(self, remote_ref: RemoteRef) -> bytes: ... + async def delete_chunk(self, remote_ref: RemoteRef) -> None: ... + async def list_chunks(self, prefix: str) -> list[RemoteRef]: ... + def get_limits(self) -> ProviderLimits: ... + async def close(self) -> None: ... +``` + +## Provider Configuration + +Each provider has specific credentials and settings stored in `.stash/config.json`: + +```json +{ + "providers": { + "telegram": { + "type": "telegram", + "credentials": { + "token": "bot_token", + "chat_id": "-1001234567890" + }, + "settings": { + "max_concurrent": "3" + } + } + } +} +``` + +## Adding a New Provider + +1. Create a new directory under `src/stash/providers//` +2. Implement `StorageProvider` interface +3. Add auth, limits, and provider modules +3. Register in `src/stash/providers/__init__.py` +4. Update CLI provider command +5. Add tests + +## Provider Limits + +Each provider defines limits in `ProviderLimits`: + +```python +@dataclass(frozen=True) +class ProviderLimits: + max_file_size: int # Maximum total file size + max_chunk_size: int # Maximum chunk size + max_concurrent_uploads: int # Concurrent uploads + rate_limit_requests: int # Requests per window + rate_limit_window: int # Rate limit window (seconds) + supports_resumable: bool = False + supports_multipart: bool = False +``` + +## Rate Limiting + +Each provider implements rate limiting: + +- **Discord**: 5 requests/second global, 10/minute for uploads +- **Telegram**: 30 requests/second per bot + +Rate limiting is handled automatically with exponential backoff. \ No newline at end of file diff --git a/docs/providers/discord.md b/docs/providers/discord.md new file mode 100644 index 0000000..ab084b4 --- /dev/null +++ b/docs/providers/discord.md @@ -0,0 +1,101 @@ +# Discord Provider + +Stash's Discord provider stores encrypted file chunks as message attachments in a Discord channel using a bot. + +## Setup + +### 1. Create Discord Bot + +1. Go to [Discord Developer Portal](https://discord.com/developers/applications) +2. Click "New Application" → Give it a name +3. Go to "Bot" tab → "Add Bot" +4. Copy the **Bot Token** (keep it secret!) +5. Enable **Message Content Intent** in Bot settings +6. Invite bot to your server with `Send Messages` and `Attach Files` permissions + +### 2. Get Channel ID + +1. Enable **Developer Mode** in Discord (User Settings → Advanced → Developer Mode) +2. Right-click the channel → "Copy Channel ID" +3. Channel ID format: `123456789012345678` + +### 3. Configure Stash + +```bash +stash provider add discord \ + --token \ + --channel-id 123456789012345678 \ + --is-bot true \ + --max-concurrent 3 +``` + +## How It Works + +1. **File Upload**: + - File encrypted and chunked (default 10MB chunks) + - Each chunk uploaded as Discord message attachment + - Message content: `stash-chunk::` + - Attachment filename: `/chunk-.bin` + +2. **Metadata Storage**: + - File manifest stored locally in `.stash/metadata/` + - Chunk metadata: `message_id`, `file_id`, `chunk_index`, `size` + - Encrypted filename stored in manifest + +3. **Retrieval**: + - Read manifest to get chunk list + - Fetch messages by ID from Discord + - Download attachments, decrypt, reconstruct + +## Limits + +| Limit | Value | +|-------|-------| +| Max file size | 25 MB (Discord limit) | +| Max chunk size | 10 MB (safe margin) | +| Rate limit | 5 req/s global, 10/min uploads | +| Max message size | 8 MB attachment + content | + +## Configuration + +```bash +stash provider add discord \ + --name my_discord \ + --token \ + --channel-id 123456789012345678 \ + --is-bot true \ + --max-concurrent 3 +``` + +### Settings + +| Setting | Description | Default | +|---------|-------------|---------| +| `max_concurrent` | Max concurrent uploads | 3 | +| `chunk_size` | Chunk size in bytes | 10MB | + +## Rate Limits + +| Endpoint | Limit | +|----------|-------| +| Global | 50 req/s | +| Send Message | 5/s per channel | +| Upload | 10/min per channel | + +## Troubleshooting + +| Error | Solution | +|-------|----------| +| "Invalid Discord token" | Check bot token is correct | +| "No permission to access channel" | Bot lacks permissions in channel | +| "Channel not found" | Check channel ID is correct | +| "Request entity too large" | File too large, reduce chunk size | +| "Rate limited" | Reduce `max_concurrent` | + +## Security Notes + +- Bot token is stored encrypted in local config +- Files encrypted before upload (AES-256-GCM) +- Bot only sees encrypted blobs +- Use dedicated channel for storage +- Revoke token if compromised \ No newline at end of file diff --git a/docs/providers/telegram.md b/docs/providers/telegram.md new file mode 100644 index 0000000..421da7c --- /dev/null +++ b/docs/providers/telegram.md @@ -0,0 +1,145 @@ +# Telegram Provider + +Stash's Telegram provider stores encrypted file chunks as documents in a Telegram chat/channel using a bot. + +## Setup + +### 1. Create Telegram Bot + +1. Go to [@BotFather](https://t.me/BotFather) on Telegram +2. Send `/newbot` and follow instructions +3. Copy the **Bot Token** (keep it secret!) + +### 2. Get Chat ID + +**For Private Chat:** +1. Message your bot +2. Forward message to [@userinfobot](https://t.me/userinfobot) +3. Copy your User ID (Chat ID) + +**For Channel/Group:** +1. Add bot to channel/group as admin +2. Forward message from channel to [@userinfobot](https://t.me/userinfobot) +3. Copy the Chat ID (negative number for channels: `-1001234567890`) + +### 3. Configure Stash + +```bash +stash provider add telegram \ + --token \ + --chat-id -1001234567890 \ + --max-concurrent 3 +``` + +## How It Works + +1. **File Upload**: + - File encrypted and chunked (default 10MB chunks) + - Each chunk uploaded as Telegram document + - Caption: `stash-chunk::` + - Document filename: `/chunk-.bin` + +2. **Metadata Storage**: + - File manifest stored locally in `.stash/metadata/` + - Chunk metadata: `message_id`, `file_id`, `chunk_index`, `size` + - Encrypted filename stored in manifest + +3. **Retrieval**: + - Read manifest to get chunk list + - Use `getFile` API to get file path + - Download file from `https://api.telegram.org/file/bot/` + - Decrypt and reconstruct + +## Limits + +| Limit | Value | +|-------|-------| +| Max file size | 20 MB (Bot API limit) | +| Max chunk size | 10 MB (safe margin) | +| Rate limit | 30 req/s global | + +## Configuration + +```bash +stash provider add telegram \ + --token \ + --chat-id -1001234567890 \ + --max-concurrent 3 +``` + +### Settings + +| Setting | Description | Default | +|---------|-------------|---------| +| `max_concurrent` | Max concurrent uploads | 3 | +| `chunk_size` | Chunk size in bytes | 10MB | + +## Rate Limits + +| Endpoint | Limit | +|----------|-------| +| Global | 30 req/s | +| sendDocument | ~20/s | +| getFile | 20/s | +| getChatHistory | 30/s | + +## File Size Limits + +| Tier | Limit | +|------|-------| +| Bot API | 20 MB | +| Telegram Premium | 4 GB (not supported by bot API) | + +## Chunking Strategy + +For files > 20 MB: +1. File is split into 10MB chunks +2. Each chunk uploaded as separate document +3. Manifest tracks chunk order and metadata +4. On retrieval: download all → decrypt → reconstruct + +## Configuration + +```bash +stash provider add telegram \ + --name my_telegram \ + --token \ + --chat-id -1001234567890 \ + --max-concurrent 3 +``` + +### Settings + +| Setting | Description | Default | +|---------|-------------|---------| +| `max_concurrent` | Max concurrent uploads | 3 | +| `chunk_size` | Chunk size in bytes | 10MB | + +## Rate Limits + +| Endpoint | Limit | +|----------|-------| +| Global | 30 req/s | +| sendDocument | ~20/s | +| getFile | 20/s | +| getChatHistory | 30/s | + +## Troubleshooting + +| Error | Solution | +|-------|----------| +| "Invalid Telegram bot token" | Check bot token from @BotFather | +| "Chat not found or bot not a member" | Add bot to chat/channel as admin | +| "Bot not a member of the chat" | Add bot to chat/channel | +| "File not found" | File may have been deleted from Telegram | +| "Request entity too large" | Reduce chunk size, file too big | +| "Rate limited" | Reduce `max_concurrent` | + +## Security Notes + +- Bot token is stored encrypted in local config +- Files encrypted before upload (AES-256-GCM) +- Bot only sees encrypted blobs +- Use private channel for storage +- Revoke token if compromised +- Telegram Premium doesn't increase bot API limits \ No newline at end of file diff --git a/docs/security.md b/docs/security.md new file mode 100644 index 0000000..e8f52ed --- /dev/null +++ b/docs/security.md @@ -0,0 +1,140 @@ +# Security + +## Overview + +Stash is designed with a **zero-knowledge** security model. The storage providers (Telegram, Discord, etc.) only ever receive encrypted, opaque data blobs. They cannot decrypt, inspect, or modify your data. + +## Encryption + +### Algorithm +- **AES-256-GCM** for all data encryption +- Authenticated encryption with associated data (AEAD) +- 12-byte nonce per chunk (randomly generated) +- 16-byte authentication tag per chunk + +### Key Hierarchy + +``` +User Password + ↓ PBKDF2 (100,000 iterations) / Argon2id (planned) +Master Key (32 bytes) + ↓ HKDF-SHA256 (salt: file_key_salt) +File Key (32 bytes per file) + ↓ HKDF-SHA256 (info: "stash-chunk-{index}") +Chunk Key (32 bytes per chunk) +``` + +### Chunk Encryption + +Each chunk is independently encrypted: +1. Derive chunk key from file key: `HKDF-SHA256(file_key, "stash-chunk-{index}")` +2. Generate random 12-byte nonce +3. Encrypt with AES-256-GCM: `ciphertext = AES-GCM(chunk_key, nonce, plaintext, aad=None)` +4. Store: `nonce (12 bytes) + ciphertext + tag (16 bytes)` + +### Key Wrapping + +File keys are wrapped with the user's password: +1. Derive wrapping key: `HKDF-SHA256(password, salt=16_bytes, info="stash-key-wrap")` +2. Encrypt file key: `AES-GCM(wrapping_key, nonce, file_key)` +3. Store: `salt (16) + nonce (12) + ciphertext + tag (16)` + +## Integrity + +### Chunk-Level +- AES-GCM authentication tag (16 bytes) per chunk +- SHA-256 checksum stored in manifest +- Verified on every download + +### File-Level +- Manifest includes SHA-256 of original file +- Verified after reconstruction + +### Manifest Integrity +- JSON serialization with deterministic ordering +- File IDs are SHA-256 hashes +- Manifest versioning for forward compatibility + +## Key Management + +### Password Handling +- Never stored, only used for key derivation +- Zeroized from memory after use +- Minimum 8 characters recommended + +### Key Rotation (Planned) +- Periodic master key rotation +- Re-encryption of file keys +- Automatic re-encryption on access + +## Provider Security + +### Data at Rest (Provider Side) +- Providers only receive encrypted blobs +- No metadata about file contents +- Chunk filenames are opaque (`/chunk-.bin`) +- Message content only contains chunk index + +### Provider Compromise +- Provider compromise = encrypted blobs only +- No plaintext, keys, or metadata leaked +- Re-encryption possible with new providers +- Forward secrecy: compromise doesn't affect past/future files + +## Network Security + +- All provider communication over HTTPS/TLS +- Certificate validation enforced +- No plaintext credentials in transit +- Token storage: encrypted in local config + +## Threat Model + +### Trusted +- Local machine (user's device) +- User password/credentials +- Stash binary (if verified) + +### Untrusted +- Storage providers (Telegram, Discord, etc.) +- Network infrastructure +- Compromised provider infrastructure + +### Out of Scope +- Local malware/keyloggers +- Physical device access +- Side-channel attacks +- Password brute force (mitigated by strong passwords) + +## Provider Compromise Scenarios + +| Scenario | Impact | Mitigation | +|----------|--------|------------| +| Provider reads files | Sees encrypted blobs only | Encryption | +| Provider modifies files | Checksum fails on download | Integrity checks | +| Provider deletes files | Local manifest has references | Replication strategy | +| Provider analyzes metadata | Only sees chunk sizes/timing | Fixed chunk sizes, padding (planned) | + +## Key Rotation (Planned) + +1. Generate new master key +2. Re-wrap all file keys +3. Re-encrypt filenames +4. Atomic manifest update +5. Old keys zeroized + +## Compliance Considerations + +- **GDPR**: Data minimization, right to deletion +- **HIPAA**: Encryption at rest/in transit +- **SOC 2**: Encryption, access controls, audit logging + +## Reporting Security Issues + +Report security vulnerabilities to: security@stashify.io + +Include: +- Description of vulnerability +- Steps to reproduce +- Impact assessment +- Suggested fix (if any) \ No newline at end of file diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 0000000..363742a --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,180 @@ +# Troubleshooting + +## Installation + +### `pip install` fails on Windows +Ensure you have the latest `pip` and Python 3.9+: + +```bash +python -m pip install --upgrade pip +python -m pip install stashify +``` + +If cryptography fails to build, install the precompiled wheel: + +```bash +python -m pip install cryptography --only-binary :all: +``` + +### `stash` command not found +Ensure the Python Scripts directory is on your PATH: + +```bash +python -m pip show stashify # shows install location +where stash # Windows +``` + +Add the Scripts directory to your PATH if needed. + +## Provider Configuration + +### "No providers configured" +You haven't added any providers. Add at least one: + +```bash +stash provider add --type ... +``` + +### "Provider not found" +The provider type is misspelled. Check the name with: + +```bash +stash provider list +``` + +Supported types: `telegram`, `discord`. + +## Telegram + +### "Invalid Telegram bot token" +1. Token format: `1234567890:AA...` (numbers colon letters) +2. Check with @BotFather → `/mybots` → your bot → API Token +3. Token must not contain spaces or quotes + +### "Chat not found or bot is not a member" +1. Add bot to the chat/channel as admin +2. For channels, Chat ID is negative: `-1001234567890` +3. Message the bot at least once before using it +4. Verify Chat ID with @userinfobot + +### "Request entity too large" +File exceeds the Telegram Bot API limit (20 MB). Stash handles this automatically by chunking, but if you set a custom chunk size above the limit: + +```bash +stash put --chunk-size 10485760 # 10 MB chunks +``` + +## Discord + +### "Invalid Discord token" +1. Token format: `MTE2...` (base64-like) +2. Check Discord Developer Portal → Bot → Token +3. Token must not include the "Bot " prefix when using `--token` + +### "No permission to access channel" +1. Bot must have `Send Messages` and `Attach Files` permissions +2. If using a channel in a server, invite the bot to that server +3. Bot can only access channels it has been granted access to + +### "Request entity too large" +Discord limit is 25 MB. Stash chunks files by default, but reduce the chunk size if you get this: + +```bash +stash put --chunk-size 10485760 # 10 MB chunks +``` + +### "Rate limited" +Discord rate limits are strict. Reduce concurrency: + +```bash +stash provider add discord --max-concurrent 2 +``` + +## Authentication + +### Password prompts fail +- Stash uses `getpass`, which requires an interactive terminal +- On Windows, use the built-in console (not some SSH/CI shells) +- If automation is needed, use `--password` flag (less secure) + +### "Authentication failed" on get +1. Provider credentials may have changed (token revoked/rotated) +2. Re-add the provider: + ```bash + stash provider remove --force + stash provider add --type ... + ``` + +## Storage + +### "Chunk not found" on get +Chunks may have been deleted from the provider. Check: +1. The chat/channel hasn't been cleared +2. Message IDs in the manifest still exist +3. No cleanup bot has deleted messages + +### Manifest file corrupted +Stash stores manifests in `.stash/metadata/`. If corrupted: +1. Check the JSON is valid +2. Restore from backup if you have one +3. Re-upload the file if all backups are gone + +## Encryption + +### "Decryption failed" +1. Wrong password: verify the password you used with `put` +2. Corrupted data: check provider messages are intact +3. Wrong provider credentials: chunks may be from a different account + +### Lost password +**There is no recovery.** Encryption keys are derived from your password and never stored. Re-upload files with a new password you can remember. + +## Networking + +### "Connection timeout" +1. Check your internet connection +2. Stash uses HTTPS to Telegram/Discord APIs +3. Firewall/proxy may be blocking connections + +### "SSL certificate verification failed" +1. Check your system clock is correct +2. Update CA certificates: + ```bash + python -m pip install --upgrade certifi + ``` + +## Docker + +### `stash` not found in container +Make sure you're using the image correctly: + +```bash +docker run --rm -v ${PWD}:/data stashify:latest --help +``` + +The `stash` binary is on the container PATH. + +### Docker build fails +The image requires `rich` (dependency). Run: + +```bash +docker build --no-cache -t stashify . +``` + +## Performance + +### Uploads are slow +1. Reduce `--max-concurrent` (provider rate limiting) +2. Increase chunk size for large files (fewer requests) +3. Check network bandwidth + +### Downloads are slow +1. Reduce concurrency to avoid rate limits +2. Larger chunk sizes mean fewer requests + +## Getting Help + +- Check the [GitHub Issues](https://github.com/Sparkleeop/Stashify/issues) +- Include the output of `stash status` +- Include the full error message +- Describe the exact commands you ran \ No newline at end of file From 4f68d211e2bda7479a698a61d5d66a98a53f718f Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 03:06:14 +0530 Subject: [PATCH 09/14] chore: stop tracking coverage.xml, add to gitignore --- .gitignore | 1 + coverage.xml | 1801 -------------------------------------------------- 2 files changed, 1 insertion(+), 1801 deletions(-) delete mode 100644 coverage.xml diff --git a/.gitignore b/.gitignore index 73bd2e9..a623892 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ htmlcov/ .nox/ .coverage .coverage.* +coverage.xml .cache .pytest_cache/ .hypothesis/ diff --git a/coverage.xml b/coverage.xml deleted file mode 100644 index 2af7b68..0000000 --- a/coverage.xml +++ /dev/null @@ -1,1801 +0,0 @@ - - - - - - C:\Users\Administrator\Desktop\Stashify - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From cdf42eea2007782ccf6d4fad68499221bb32511b Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 23:53:54 +0530 Subject: [PATCH 10/14] feat: add ChunkStatus and UploadStatus enums, update ChunkInfo and FileManifest with upload tracking fields --- src/stash/core/manifest.py | 59 +++++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/src/stash/core/manifest.py b/src/stash/core/manifest.py index 78fa92e..3fba401 100644 --- a/src/stash/core/manifest.py +++ b/src/stash/core/manifest.py @@ -18,6 +18,23 @@ class DistributionStrategy(Enum): REPLICATED = "replicated" +class ChunkStatus(Enum): + """Upload status of a chunk.""" + PENDING = "pending" + UPLOADING = "uploading" + UPLOADED = "uploaded" + FAILED = "failed" + + +class UploadStatus(Enum): + """Overall upload status of a file.""" + NOT_STARTED = "not_started" + IN_PROGRESS = "in_progress" + COMPLETED = "completed" + FAILED = "failed" + PAUSED = "paused" + + @dataclass(frozen=True, slots=True) class ChunkInfo: """Information about a single chunk.""" @@ -29,6 +46,9 @@ class ChunkInfo: remote_id: str nonce: bytes metadata: dict[str, str] = field(default_factory=dict) + status: ChunkStatus = ChunkStatus.PENDING + uploaded_at: float | None = None + error: str | None = None @dataclass(frozen=True, slots=True) @@ -54,12 +74,17 @@ class FileManifest: original_size: int chunk_size: int chunk_count: int - encryption: EncryptionInfo - chunks: tuple[ChunkInfo, ...] + encryption: "EncryptionInfo" + chunks: tuple["ChunkInfo", ...] strategy: DistributionStrategy created_at: float modified_at: float version: int = 1 + upload_status: UploadStatus = UploadStatus.NOT_STARTED + total_chunks: int = 0 + uploaded_chunks: int = 0 + started_at: float | None = None + completed_at: float | None = None def to_json(self) -> str: """Serialize manifest to JSON.""" @@ -68,8 +93,10 @@ def to_json(self) -> str: data["encryption"] = asdict(self.encryption) data["strategy"] = self.strategy.value data["encrypted_name_nonce"] = self.encrypted_name_nonce.hex() + data["upload_status"] = self.upload_status.value for chunk in data["chunks"]: chunk["nonce"] = chunk["nonce"].hex() + chunk["status"] = chunk["status"].value data["encryption"]["file_key_salt"] = data["encryption"]["file_key_salt"].hex() if data["encryption"]["file_key_wrapped"]: data["encryption"]["file_key_wrapped"] = data["encryption"]["file_key_wrapped"].hex() @@ -80,6 +107,7 @@ def from_json(cls, json_str: str) -> "FileManifest": """Deserialize manifest from JSON.""" data = json.loads(json_str) data["strategy"] = DistributionStrategy(data["strategy"]) + data["upload_status"] = UploadStatus(data.get("upload_status", "not_started")) enc_data = data["encryption"] enc_data["file_key_salt"] = bytes.fromhex(enc_data["file_key_salt"]) if enc_data["file_key_wrapped"]: @@ -89,11 +117,12 @@ def from_json(cls, json_str: str) -> "FileManifest": chunks = [] for c in data["chunks"]: c["nonce"] = bytes.fromhex(c["nonce"]) + c["status"] = ChunkStatus(c.get("status", "pending")) chunks.append(ChunkInfo(**c)) data["chunks"] = tuple(chunks) return cls(**data) - def get_chunk(self, index: int) -> ChunkInfo: + def get_chunk(self, index: int) -> "ChunkInfo": """Get chunk info by index.""" for chunk in self.chunks: if chunk.index == index: @@ -130,6 +159,9 @@ def add_chunk( remote_id: str, nonce: bytes, metadata: dict[str, str] | None = None, + status: ChunkStatus = ChunkStatus.PENDING, + uploaded_at: float | None = None, + error: str | None = None, ) -> None: """Add a chunk to the manifest.""" self.chunks.append(ChunkInfo( @@ -141,12 +173,26 @@ def add_chunk( remote_id=remote_id, nonce=nonce, metadata=metadata or {}, + status=status, + uploaded_at=uploaded_at, + error=error, )) - def build(self) -> FileManifest: + def build(self) -> "FileManifest": """Build the final manifest.""" if len(self.chunks) == 0: raise ManifestError("Cannot build manifest with no chunks") + + total = len(self.chunks) + uploaded = sum(1 for c in self.chunks if c.status == ChunkStatus.UPLOADED) + status = UploadStatus.NOT_STARTED + if uploaded == len(self.chunks) and total > 0: + status = UploadStatus.COMPLETED + elif uploaded > 0: + status = UploadStatus.IN_PROGRESS + elif any(c.status == ChunkStatus.FAILED for c in self.chunks): + status = UploadStatus.FAILED + return FileManifest( file_id=self.file_id, original_name=self.original_name, @@ -160,6 +206,11 @@ def build(self) -> FileManifest: strategy=self.strategy, created_at=self.created_at, modified_at=time.time(), + upload_status=status, + total_chunks=total, + uploaded_chunks=uploaded, + started_at=self.chunks[0].uploaded_at if self.chunks else None, + completed_at=time.time() if status == UploadStatus.COMPLETED else None, ) From 3b497aa2028e52c831f5082fe89fd00004b28893 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Fri, 21 Aug 2026 00:04:56 +0530 Subject: [PATCH 11/14] feat: implement incremental manifest saving during upload for resumable uploads --- src/stash/cli/commands/put.py | 265 +++++++++++++++++++++++++++++----- 1 file changed, 229 insertions(+), 36 deletions(-) diff --git a/src/stash/cli/commands/put.py b/src/stash/cli/commands/put.py index 48d738a..f5f82b2 100644 --- a/src/stash/cli/commands/put.py +++ b/src/stash/cli/commands/put.py @@ -1,6 +1,7 @@ """CLI command: put - Store a file.""" import asyncio +import time from pathlib import Path import click @@ -12,9 +13,12 @@ from stash.core.jobs import JobConfig from stash.core.keymanager import KeyManager from stash.core.manifest import ( + ChunkInfo, + ChunkStatus, DistributionStrategy, EncryptionInfo, ManifestBuilder, + UploadStatus, compute_checksum, generate_file_id, ) @@ -28,10 +32,13 @@ @click.option("--chunk-size", type=int, help="Chunk size in bytes (default: provider limit)") @click.option("--strategy", type=click.Choice(["single", "split", "balanced", "replicated"]), default="single", help="Distribution strategy") @click.option("--confirm/--no-confirm", default=True, help="Confirm before upload") +@click.option("--resume", is_flag=True, help="Resume an incomplete upload") +@click.option("--file-id", help="File ID to resume (required with --resume if multiple incomplete uploads exist)") +@click.option("--confirm/--no-confirm", default=True, help="Confirm before upload") @click.pass_context -def put_cmd(ctx: click.Context, file_path: Path, provider: str | None, chunk_size: int | None, strategy: str, confirm: bool) -> None: +def put_cmd(ctx: click.Context, file_path: Path, provider: str | None, chunk_size: int | None, strategy: str, confirm: bool, resume: bool, file_id: str | None) -> None: """Store a file in Stash.""" - asyncio.run(_put_async(file_path, ctx.obj["repo"], provider, chunk_size, strategy, confirm)) + asyncio.run(_put_async(file_path, ctx.obj["repo"], provider, chunk_size, strategy, confirm, resume, file_id)) async def _put_async( @@ -41,6 +48,8 @@ async def _put_async( chunk_size: int | None, strategy: str, do_confirm: bool, + resume: bool, + file_id: str | None, ) -> None: repo = repo_path.resolve() store = MetadataStore(repo) @@ -81,14 +90,80 @@ async def _put_async( return crypto = CryptoEngine() - file_id = generate_file_id() - file_key = crypto.generate_file_key(rmk) + + # Handle resume logic + existing_manifest = None + file_id = None + file_key = None + encrypted_name = None + encrypted_name_nonce = None + file_size = file_path.stat().st_size + + if resume: + # Find existing incomplete manifest + if file_id: + # Explicit file ID provided + if not store.file_exists(file_id): + print_error(f"File with ID '{file_id}' not found") + return + existing_manifest = store.load_manifest(file_id) + else: + # Auto-detect by filename + for fid in store.list_files(): + manifest = store.load_manifest(fid) + if manifest.original_name == file_path.name and manifest.upload_status != UploadStatus.COMPLETED: + existing_manifest = manifest + break + + if existing_manifest is None: + print_error("No incomplete upload found to resume") + print_info("Run 'stash put' without --resume to start a new upload") + return + + # Verify file matches + if existing_manifest.original_size != file_path.stat().st_size: + print_error("File size does not match the incomplete upload") + return + + # Verify file content matches (check first chunk checksum if available) + if existing_manifest.chunks: + chunker = Chunker(ChunkConfig(chunk_size=existing_manifest.chunk_size)) + first_chunk_data = next(chunker.chunk_file(file_path)).data + first_checksum = compute_checksum(first_chunk_data) + if existing_manifest.chunks[0].checksum != first_checksum: + print_error("File content does not match the incomplete upload") + return + + file_id = existing_manifest.file_id + file_key = crypto.derive_file_key_from_rmk(rmk, file_id.encode()) + + # Restore encrypted filename info + encrypted_name = existing_manifest.encrypted_name + encrypted_name_nonce = existing_manifest.encrypted_name_nonce + + print_info(f"Resuming upload of '{existing_manifest.original_name}' ({existing_manifest.file_id})") + print_info(f"Progress: {existing_manifest.uploaded_chunks}/{existing_manifest.total_chunks} chunks uploaded") + else: + # New upload - generate new file_id and file_key + file_id = generate_file_id() + file_key = crypto.generate_file_key(rmk) + + # Encrypt the filename using the file key + encrypted_name_ciphertext, encrypted_name_nonce = crypto.encrypt_filename( + file_path.name.encode(), file_key + ) + encrypted_name = encrypted_name_ciphertext.hex() + + crypto = CryptoEngine() + if not file_key: + file_key = crypto.generate_file_key(rmk) # Encrypt the filename using the file key - encrypted_name_ciphertext, encrypted_name_nonce = crypto.encrypt_filename( - file_path.name.encode(), file_key - ) - encrypted_name = encrypted_name_ciphertext.hex() + if encrypted_name is None: + encrypted_name_ciphertext, encrypted_name_nonce = crypto.encrypt_filename( + file_path.name.encode(), file_key + ) + encrypted_name = encrypted_name_ciphertext.hex() provider_configs = {} for name in provider_names: @@ -112,27 +187,107 @@ async def _put_async( dist_strategy = DistributionStrategy(strategy) - encryption_info = EncryptionInfo( - algorithm="AES-256-GCM", - key_size=32, - nonce_size=12, - chunk_key_derivation="HKDF-SHA256", - file_key_salt=file_key.salt, - file_key_wrapped=None, - ) - - builder = ManifestBuilder( - file_id=file_id, - original_name=file_path.name, - encrypted_name=encrypted_name, - encrypted_name_nonce=encrypted_name_nonce, - original_size=file_path.stat().st_size, - chunk_size=effective_chunk_size, - encryption=encryption_info, - strategy=dist_strategy, - ) - - print_info(f"Processing {num_chunks} chunks...") + # Initialize manifest builder + if resume and existing_manifest: + # Resume existing manifest - keep existing chunks, update status + file_id = existing_manifest.file_id + encryption_info = existing_manifest.encryption + builder = ManifestBuilder( + file_id=existing_manifest.file_id, + original_name=existing_manifest.original_name, + encrypted_name=existing_manifest.encrypted_name, + encrypted_name_nonce=existing_manifest.encrypted_name_nonce, + original_size=existing_manifest.original_size, + chunk_size=existing_manifest.chunk_size, + encryption=existing_manifest.encryption, + strategy=DistributionStrategy(existing_manifest.strategy), + ) + # Pre-populate with existing chunks + for chunk in existing_manifest.chunks: + builder.add_chunk( + index=chunk.index, + size=chunk.size, + encrypted_size=chunk.encrypted_size, + checksum=chunk.checksum, + provider=chunk.provider, + remote_id=chunk.remote_id, + nonce=chunk.nonce, + metadata=chunk.metadata, + status=chunk.status, + uploaded_at=chunk.uploaded_at, + error=chunk.error, + ) + else: + # New upload + file_id = generate_file_id() + file_key = crypto.generate_file_key(rmk) + + # Encrypt the filename using the file key + encrypted_name_ciphertext, encrypted_name_nonce = crypto.encrypt_filename( + file_path.name.encode(), file_key + ) + encrypted_name = encrypted_name_ciphertext.hex() + + encryption_info = EncryptionInfo( + algorithm="AES-256-GCM", + key_size=32, + nonce_size=12, + chunk_key_derivation="HKDF-SHA256", + file_key_salt=file_key.salt, + file_key_wrapped=None, + ) + + builder = ManifestBuilder( + file_id=file_id, + original_name=file_path.name, + encrypted_name=encrypted_name, + encrypted_name_nonce=encrypted_name_nonce, + original_size=file_size, + chunk_size=effective_chunk_size, + encryption=encryption_info, + strategy=dist_strategy, + ) + # Pre-populate all chunks as PENDING + for i in range(num_chunks): + builder.add_chunk( + index=i, + size=0, # Will be updated when chunk is uploaded + encrypted_size=0, + checksum="", + provider="", + remote_id="", + nonce=b"", + status=ChunkStatus.PENDING, + ) + + provider_configs = {} + for name in provider_names: + config = store.get_provider_config(name) + if not config: + print_error(f"Provider config not found: {name}") + return + provider_configs[name] = config + + provider_instances = {} + for name, config in provider_configs.items(): + instance = await ProviderRegistry.create(config.type, config) + provider_instances[name] = instance + + limits = {name: p.get_limits() for name, p in provider_instances.items()} + max_chunk = min(l.max_chunk_size for l in limits.values()) + effective_chunk_size = min(chunk_size or max_chunk, max_chunk) + + chunker = Chunker(ChunkConfig(chunk_size=effective_chunk_size)) + num_chunks = chunker.get_num_chunks(file_size) + + dist_strategy = DistributionStrategy(strategy) + + # Save initial manifest (PENDING state) + manifest = builder.build() + store.save_manifest(manifest) + + if not resume: + print_info(f"Processing {num_chunks} chunks...") JobConfig(max_workers=min(4, num_chunks)) @@ -141,6 +296,13 @@ async def _put_async( progress = create_progress() task = progress.add_task("Uploading", total=num_chunks) + # Track which chunks are already uploaded (for resume) + uploaded_indices = set() + if resume and existing_manifest: + for chunk in existing_manifest.chunks: + if chunk.status == ChunkStatus.UPLOADED: + uploaded_indices.add(chunk.index) + async def upload_chunk(chunk_data: bytes, chunk_index: int, provider_name: str) -> tuple[bytes, dict[str, str]]: """Encrypt and upload a single chunk.""" async with semaphores[provider_name]: @@ -160,30 +322,61 @@ async def upload_chunk(chunk_data: bytes, chunk_index: int, provider_name: str) progress = create_progress() task = progress.add_task("Uploading", total=num_chunks) - for chunk in chunker.chunk_file(file_path): - checksum = compute_checksum(chunk.data) + # Update progress for already uploaded chunks + for _ in uploaded_indices: + progress.advance(task) + + for c in chunker.chunk_file(file_path): + if c.index in uploaded_indices: + print_info(f"Chunk {c.index} already uploaded, skipping") + progress.advance(task) + continue + + checksum = compute_checksum(c.data) if dist_strategy == DistributionStrategy.SINGLE: target = provider_names[0] elif dist_strategy == DistributionStrategy.SPLIT: - target = provider_names[chunk.index % len(provider_names)] + target = provider_names[c.index % len(provider_names)] else: target = provider_names[0] - nonce, metadata = await upload_chunk(chunk.data, chunk.index, target) + # Mark chunk as uploading + builder.chunks[c.index] = ChunkInfo( + index=c.index, + size=c.size, + encrypted_size=0, + checksum=checksum, + provider=target, + remote_id="", + nonce=b"", + metadata={}, + status=ChunkStatus.UPLOADING, + ) + store.save_manifest(builder.build()) + + nonce, metadata = await upload_chunk(c.data, c.index, target) - builder.add_chunk( - index=chunk.index, - size=chunk.size, + # Update chunk as uploaded + builder.chunks[c.index] = ChunkInfo( + index=c.index, + size=c.size, encrypted_size=len(metadata.get("size", "0")), checksum=checksum, provider=target, remote_id=metadata.get("remote_id", ""), nonce=nonce, metadata=metadata, + status=ChunkStatus.UPLOADED, + uploaded_at=time.time(), + error=None, ) + manifest = builder.build() + store.save_manifest(manifest) + progress.advance(task) + # Final manifest build and save manifest = builder.build() store.save_manifest(manifest) From 14dab9ec3de0aabefd4f1f11667d6b5c76a2a820 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Sat, 22 Aug 2026 15:34:43 +0530 Subject: [PATCH 12/14] feat: enhance resumable upload functionality with integration tests --- src/stash/cli/commands/put.py | 16 +- tests/integration/test_resumable_uploads.py | 566 ++++++++++++++++++++ 2 files changed, 575 insertions(+), 7 deletions(-) create mode 100644 tests/integration/test_resumable_uploads.py diff --git a/src/stash/cli/commands/put.py b/src/stash/cli/commands/put.py index f5f82b2..b4a3710 100644 --- a/src/stash/cli/commands/put.py +++ b/src/stash/cli/commands/put.py @@ -31,12 +31,11 @@ @click.option("--provider", help="Specific provider to use (default: first available)") @click.option("--chunk-size", type=int, help="Chunk size in bytes (default: provider limit)") @click.option("--strategy", type=click.Choice(["single", "split", "balanced", "replicated"]), default="single", help="Distribution strategy") -@click.option("--confirm/--no-confirm", default=True, help="Confirm before upload") @click.option("--resume", is_flag=True, help="Resume an incomplete upload") @click.option("--file-id", help="File ID to resume (required with --resume if multiple incomplete uploads exist)") @click.option("--confirm/--no-confirm", default=True, help="Confirm before upload") @click.pass_context -def put_cmd(ctx: click.Context, file_path: Path, provider: str | None, chunk_size: int | None, strategy: str, confirm: bool, resume: bool, file_id: str | None) -> None: +def put_cmd(ctx: click.Context, file_path: Path, provider: str | None, chunk_size: int | None, strategy: str, resume: bool, file_id: str | None, confirm: bool) -> None: """Store a file in Stash.""" asyncio.run(_put_async(file_path, ctx.obj["repo"], provider, chunk_size, strategy, confirm, resume, file_id)) @@ -123,7 +122,7 @@ async def _put_async( # Verify file matches if existing_manifest.original_size != file_path.stat().st_size: print_error("File size does not match the incomplete upload") - return + raise SystemExit(1) # Verify file content matches (check first chunk checksum if available) if existing_manifest.chunks: @@ -132,7 +131,7 @@ async def _put_async( first_checksum = compute_checksum(first_chunk_data) if existing_manifest.chunks[0].checksum != first_checksum: print_error("File content does not match the incomplete upload") - return + raise SystemExit(1) file_id = existing_manifest.file_id file_key = crypto.derive_file_key_from_rmk(rmk, file_id.encode()) @@ -317,7 +316,8 @@ async def upload_chunk(chunk_data: bytes, chunk_index: int, provider_name: str) is_last=False, ) remote_ref = await provider_instances[provider_name].upload_chunk(encrypted_chunk, remote_path) - return encrypted.nonce, remote_ref.metadata + # Return nonce and a dict with remote_id and metadata + return encrypted.nonce, {"remote_id": remote_ref.remote_id, "metadata": remote_ref.metadata} # type: ignore[dict-item] progress = create_progress() task = progress.add_task("Uploading", total=num_chunks) @@ -355,7 +355,9 @@ async def upload_chunk(chunk_data: bytes, chunk_index: int, provider_name: str) ) store.save_manifest(builder.build()) - nonce, metadata = await upload_chunk(c.data, c.index, target) + nonce, upload_result = await upload_chunk(c.data, c.index, target) + remote_id: str = upload_result["remote_id"] + metadata: dict[str, str] = upload_result["metadata"] # type: ignore[assignment] # Update chunk as uploaded builder.chunks[c.index] = ChunkInfo( @@ -364,7 +366,7 @@ async def upload_chunk(chunk_data: bytes, chunk_index: int, provider_name: str) encrypted_size=len(metadata.get("size", "0")), checksum=checksum, provider=target, - remote_id=metadata.get("remote_id", ""), + remote_id=remote_id, nonce=nonce, metadata=metadata, status=ChunkStatus.UPLOADED, diff --git a/tests/integration/test_resumable_uploads.py b/tests/integration/test_resumable_uploads.py new file mode 100644 index 0000000..bd83d5a --- /dev/null +++ b/tests/integration/test_resumable_uploads.py @@ -0,0 +1,566 @@ +"""Integration tests for resumable uploads.""" + +import pytest +import tempfile +import time +import sys +from io import StringIO +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +from stash.core.chunking import ChunkConfig, Chunker +from stash.core.crypto import CryptoEngine +from stash.core.keymanager import KeyManager +from stash.core.manifest import ( + ChunkStatus, + ChunkInfo, + DistributionStrategy, + EncryptionInfo, + FileManifest, + ManifestBuilder, + UploadStatus, + compute_checksum, + generate_file_id, +) +from stash.core.metadata import MetadataStore +from stash.core.storage import RemoteRef + + +class MockProvider: + """Mock provider for testing.""" + + def __init__(self, name: str): + self.name = name + self.chunks = {} + self.closed = False + self.max_chunk_size = 10 * 1024 * 1024 + self.max_concurrent_uploads = 3 + self.config = type('Config', (), { + 'settings': {'max_concurrent': '3'}, + 'type': name, + 'credentials': {}, + 'settings': {'max_concurrent': '3'}, + })() + + async def initialize(self, config): + pass + + async def upload_chunk(self, chunk, remote_path): + self.chunks[remote_path] = chunk.data + return RemoteRef( + provider=self.name, + remote_id=remote_path, + metadata={'size': str(len(chunk.data))} + ) + + async def download_chunk(self, remote_ref): + return self.chunks.get(remote_ref.remote_id, b"") + + async def delete_chunk(self, remote_ref): + self.chunks.pop(remote_ref.remote_id, None) + + async def list_chunks(self, prefix): + return [] + + def get_limits(self): + from stash.core.storage import ProviderLimits + return ProviderLimits( + max_file_size=100 * 1024 * 1024, + max_chunk_size=10 * 1024 * 1024, + max_concurrent_uploads=3, + rate_limit_requests=30, + rate_limit_window=1, + ) + + async def close(self): + self.closed = True + + +@pytest.fixture +def temp_repo(): + """Create a temporary repository for testing.""" + with tempfile.TemporaryDirectory() as tmpdir: + repo_path = Path(tmpdir) + store = MetadataStore(repo_path) + + # Initialize repository with a dummy RMK + keymanager = KeyManager(repo_path) + identity = keymanager.initialize_repository() + + # Configure providers in the metadata store + from stash.core.storage import ProviderConfig + store.set_provider_config("telegram", ProviderConfig( + name="telegram", + type="telegram", + credentials={"token": "test_token", "chat_id": "-1001234567890"}, + settings={"max_concurrent": "3"}, + )) + store.set_provider_config("discord", ProviderConfig( + name="discord", + type="discord", + credentials={"token": "test_token", "channel_id": "123456789012345678"}, + settings={"max_concurrent": "3"}, + )) + + yield repo_path, store, keymanager, identity + + +@pytest.fixture +def test_file(temp_repo): + """Create a test file.""" + repo_path, _, _, _ = temp_repo + file_path = repo_path / "test_file.txt" + content = b"x" * (25 * 1024 * 1024) # 25MB file - 3 chunks at 10MB + file_path.write_bytes(content) + return file_path + + +@pytest.fixture +def mock_providers(): + """Create mock providers.""" + return { + "telegram": MockProvider("telegram"), + "discord": MockProvider("discord"), + } + + +class TestResumableUploads: + """Integration tests for resumable uploads.""" + + @pytest.mark.asyncio + async def test_new_upload_creates_partial_manifest(self, temp_repo, test_file, mock_providers): + """Test that a new upload creates a partial manifest with PENDING chunks.""" + repo_path, store, keymanager, identity = temp_repo + + # Import here to avoid circular imports + from stash.cli.commands.put import _put_async + + with patch('stash.cli.commands.put.ProviderRegistry') as mock_registry: + mock_registry.create = AsyncMock(side_effect=lambda name, config: mock_providers[name]) + + await _put_async( + file_path=test_file, + repo_path=repo_path, + provider_name="telegram", + chunk_size=None, + strategy="single", + do_confirm=False, + resume=False, + file_id=None, + ) + + # Verify manifest was created + files = store.list_files() + assert len(files) == 1 + + manifest = store.load_manifest(files[0]) + assert manifest.upload_status == UploadStatus.COMPLETED + assert manifest.total_chunks == 3 + assert manifest.uploaded_chunks == 3 + + # All chunks should be uploaded + for chunk in manifest.chunks: + assert chunk.status == ChunkStatus.UPLOADED + assert chunk.remote_id != "" + + @pytest.mark.asyncio + async def test_resume_upload_after_interruption(self, temp_repo, test_file, mock_providers): + """Test resuming an interrupted upload.""" + repo_path, store, keymanager, identity = temp_repo + + from stash.cli.commands.put import _put_async + + with patch('stash.cli.commands.put.ProviderRegistry') as mock_registry: + mock_registry.create = AsyncMock(side_effect=lambda name, config: mock_providers[name]) + + # First, start an upload and interrupt it after 1 chunk + await _put_async( + file_path=test_file, + repo_path=repo_path, + provider_name="telegram", + chunk_size=None, + strategy="single", + do_confirm=False, + resume=False, + file_id=None, + ) + + # Verify first upload completed + files = store.list_files() + manifest = store.load_manifest(files[0]) + assert manifest.upload_status == UploadStatus.COMPLETED + + # Now simulate interruption: manually set one chunk back to pending + # and remove it from the provider + file_id = files[0] + manifest = store.load_manifest(file_id) + + # Simulate interruption: mark last chunk as pending, remove from provider + with patch('stash.cli.commands.put.ProviderRegistry') as mock_registry: + mock_registry.create = AsyncMock(side_effect=lambda name, config: mock_providers[name]) + + # Remove last chunk from provider + last_chunk = manifest.chunks[-1] + if last_chunk.provider in mock_providers: + mock_providers[last_chunk.provider].chunks.pop(last_chunk.remote_id, None) + + # Update chunk status to pending + chunks = list(manifest.chunks) + old_chunk = chunks[-1] + chunks[-1] = ChunkInfo( + index=old_chunk.index, + size=old_chunk.size, + encrypted_size=old_chunk.encrypted_size, + checksum=old_chunk.checksum, + provider=old_chunk.provider, + remote_id="", + nonce=b"", + metadata={}, + status=ChunkStatus.PENDING, + uploaded_at=None, + error=None, + ) + # Rebuild manifest with updated chunks + from stash.core.manifest import FileManifest, EncryptionInfo, DistributionStrategy + new_manifest = FileManifest( + file_id=manifest.file_id, + original_name=manifest.original_name, + encrypted_name=manifest.encrypted_name, + encrypted_name_nonce=manifest.encrypted_name_nonce, + original_size=manifest.original_size, + chunk_size=manifest.chunk_size, + chunk_count=manifest.chunk_count, + encryption=manifest.encryption, + chunks=tuple(chunks), + strategy=manifest.strategy, + created_at=manifest.created_at, + modified_at=time.time(), + upload_status=UploadStatus.IN_PROGRESS, + total_chunks=manifest.total_chunks, + uploaded_chunks=manifest.uploaded_chunks - 1, + started_at=manifest.started_at, + completed_at=None, + ) + store.save_manifest(new_manifest) + + # Now resume the upload + await _put_async( + file_path=test_file, + repo_path=repo_path, + provider_name="telegram", + chunk_size=None, + strategy="single", + do_confirm=False, + resume=True, + file_id=None, + ) + + # Verify upload completed + manifest = store.load_manifest(file_id) + assert manifest.upload_status == UploadStatus.COMPLETED + assert manifest.uploaded_chunks == 3 + + for chunk in manifest.chunks: + assert chunk.status == ChunkStatus.UPLOADED + assert chunk.remote_id != "" + + @pytest.mark.asyncio + async def test_resume_with_explicit_file_id(self, temp_repo, test_file, mock_providers): + """Test resuming with explicit --file-id.""" + repo_path, store, keymanager, identity = temp_repo + + from stash.cli.commands.put import _put_async + + # Create an incomplete upload first + with patch('stash.cli.commands.put.ProviderRegistry') as mock_registry: + mock_registry.create = AsyncMock(side_effect=lambda name, config: mock_providers[name]) + + await _put_async( + file_path=test_file, + repo_path=repo_path, + provider_name="telegram", + chunk_size=None, + strategy="single", + do_confirm=False, + resume=False, + file_id=None, + ) + + files = store.list_files() + file_id = files[0] + + # Corrupt the upload - mark one chunk as pending + manifest = store.load_manifest(file_id) + chunks = list(manifest.chunks) + chunks[1] = ChunkInfo( + index=chunks[1].index, + size=chunks[1].size, + encrypted_size=chunks[1].encrypted_size, + checksum=chunks[1].checksum, + provider=chunks[1].provider, + remote_id="", + nonce=b"", + metadata={}, + status=ChunkStatus.PENDING, + uploaded_at=None, + error=None, + ) + + new_manifest = FileManifest( + file_id=manifest.file_id, + original_name=manifest.original_name, + encrypted_name=manifest.encrypted_name, + encrypted_name_nonce=manifest.encrypted_name_nonce, + original_size=manifest.original_size, + chunk_size=manifest.chunk_size, + chunk_count=manifest.chunk_count, + encryption=manifest.encryption, + chunks=tuple(chunks), + strategy=manifest.strategy, + created_at=manifest.created_at, + modified_at=time.time(), + upload_status=UploadStatus.IN_PROGRESS, + total_chunks=3, + uploaded_chunks=2, + started_at=manifest.started_at, + completed_at=None, + ) + store.save_manifest(new_manifest) + + # Remove chunk from provider + mock_providers["telegram"].chunks.pop(chunks[1].remote_id, None) + + # Resume with explicit file-id + with patch('stash.cli.commands.put.ProviderRegistry') as mock_registry: + mock_registry.create = AsyncMock(side_effect=lambda name, config: mock_providers[name]) + + await _put_async( + file_path=test_file, + repo_path=repo_path, + provider_name="telegram", + chunk_size=None, + strategy="single", + do_confirm=False, + resume=True, + file_id=file_id, + ) + + # Verify completed + manifest = store.load_manifest(file_id) + assert manifest.upload_status == UploadStatus.COMPLETED + assert manifest.uploaded_chunks == 3 + + @pytest.mark.asyncio + async def test_resume_fails_when_file_changed(self, temp_repo, test_file, mock_providers): + """Test that resume fails when file content has changed.""" + repo_path, store, keymanager, identity = temp_repo + + from stash.cli.commands.put import _put_async + + # Create incomplete upload + with patch('stash.cli.commands.put.ProviderRegistry') as mock_registry: + mock_registry.create = AsyncMock(side_effect=lambda name, config: mock_providers[name]) + + await _put_async( + file_path=test_file, + repo_path=repo_path, + provider_name="telegram", + chunk_size=None, + strategy="single", + do_confirm=False, + resume=False, + file_id=None, + ) + + files = store.list_files() + manifest = store.load_manifest(files[0]) + + # Corrupt the manifest to simulate incomplete upload + chunks = list(manifest.chunks) + chunks[0] = ChunkInfo( + index=chunks[0].index, + size=chunks[0].size, + encrypted_size=chunks[0].encrypted_size, + checksum=chunks[0].checksum, + provider=chunks[0].provider, + remote_id="", + nonce=b"", + metadata={}, + status=ChunkStatus.PENDING, + uploaded_at=None, + error=None, + ) + + from stash.core.manifest import FileManifest, UploadStatus + new_manifest = FileManifest( + file_id=manifest.file_id, + original_name=manifest.original_name, + encrypted_name=manifest.encrypted_name, + encrypted_name_nonce=manifest.encrypted_name_nonce, + original_size=manifest.original_size, + chunk_size=manifest.chunk_size, + chunk_count=manifest.chunk_count, + encryption=manifest.encryption, + chunks=tuple(chunks), + strategy=manifest.strategy, + created_at=manifest.created_at, + modified_at=time.time(), + upload_status=UploadStatus.IN_PROGRESS, + total_chunks=manifest.total_chunks, + uploaded_chunks=2, + started_at=manifest.started_at, + completed_at=None, + ) + store.save_manifest(new_manifest) + + # Modify the file content but keep the same size + new_content = b"y" * (25 * 1024 * 1024) # 25MB of different content + test_file.write_bytes(new_content) + + # Try to resume - should fail + with patch('stash.cli.commands.put.ProviderRegistry') as mock_registry: + mock_registry.create = AsyncMock(side_effect=lambda name, config: mock_providers[name]) + + import sys + from io import StringIO + old_stdout = sys.stdout + sys.stdout = StringIO() + try: + await _put_async( + file_path=test_file, + repo_path=repo_path, + provider_name="telegram", + chunk_size=None, + strategy="single", + do_confirm=False, + resume=True, + file_id=None, + ) + except SystemExit: + pass + finally: + captured_stdout = sys.stdout.getvalue() + sys.stdout = old_stdout + + # Should fail with file content mismatch error + assert "does not match" in captured_stdout + + @pytest.mark.asyncio + async def test_multiple_resume_cycles(self, temp_repo, test_file, mock_providers): + """Test multiple resume cycles work correctly.""" + repo_path, store, keymanager, identity = temp_repo + + from stash.cli.commands.put import _put_async + + # Initial upload + with patch('stash.cli.commands.put.ProviderRegistry') as mock_registry: + mock_registry.create = AsyncMock(side_effect=lambda name, config: mock_providers[name]) + + await _put_async( + file_path=test_file, + repo_path=repo_path, + provider_name="telegram", + chunk_size=None, + strategy="single", + do_confirm=False, + resume=False, + file_id=None, + ) + + files = store.list_files() + file_id = files[0] + + # Simulate 3 interruption/resume cycles + for cycle in range(3): + manifest = store.load_manifest(file_id) + + # Mark last completed chunk as pending + chunks = list(manifest.chunks) + last_uploaded = max((i for i, c in enumerate(chunks) if c.status == ChunkStatus.UPLOADED), default=0) + if last_uploaded >= 0: + old_chunk = chunks[last_uploaded] + chunks[last_uploaded] = ChunkInfo( + index=old_chunk.index, + size=old_chunk.size, + encrypted_size=old_chunk.encrypted_size, + checksum=old_chunk.checksum, + provider=old_chunk.provider, + remote_id="", + nonce=b"", + metadata={}, + status=ChunkStatus.PENDING, + uploaded_at=None, + error=None, + ) + + # Remove from provider + mock_providers["telegram"].chunks.pop(manifest.chunks[last_uploaded].remote_id, None) + + # Rebuild manifest + from stash.core.manifest import FileManifest + new_manifest = FileManifest( + file_id=manifest.file_id, + original_name=manifest.original_name, + encrypted_name=manifest.encrypted_name, + encrypted_name_nonce=manifest.encrypted_name_nonce, + original_size=manifest.original_size, + chunk_size=manifest.chunk_size, + chunk_count=manifest.chunk_count, + encryption=manifest.encryption, + chunks=tuple(chunks), + strategy=manifest.strategy, + created_at=manifest.created_at, + modified_at=time.time(), + upload_status=UploadStatus.IN_PROGRESS, + total_chunks=manifest.total_chunks, + uploaded_chunks=manifest.uploaded_chunks - 1, + started_at=manifest.started_at, + completed_at=None, + ) + store.save_manifest(new_manifest) + + # Resume + with patch('stash.cli.commands.put.ProviderRegistry') as mock_registry: + mock_registry.create = AsyncMock(side_effect=lambda name, config: mock_providers[name]) + + await _put_async( + file_path=test_file, + repo_path=repo_path, + provider_name="telegram", + chunk_size=None, + strategy="single", + do_confirm=False, + resume=True, + file_id=file_id, + ) + + # Verify completed + manifest = store.load_manifest(file_id) + assert manifest.upload_status == UploadStatus.COMPLETED + assert manifest.uploaded_chunks == 3 + + @pytest.mark.asyncio + async def test_corrupted_manifest_handling(self, temp_repo, test_file, mock_providers): + """Test handling of corrupted manifest.""" + repo_path, store, keymanager, identity = temp_repo + + # Create a corrupted manifest file + manifest_dir = repo_path / ".stash" / "metadata" / "files" + manifest_dir.mkdir(parents=True, exist_ok=True) + + # Write invalid JSON + corrupted_file = manifest_dir / "corrupted.json" + corrupted_file.write_text("{ invalid json") + + # Should not crash when listing files + files = store.list_files() + assert isinstance(files, list) + + # Try to load corrupted manifest - should raise MetadataError + from stash.core.exceptions import MetadataError + with pytest.raises(MetadataError): + store.load_manifest("corrupted") + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) \ No newline at end of file From 3fc3bd4e1ee29e671cef30c787f2dbad4bb05b6b Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Sat, 22 Aug 2026 15:42:10 +0530 Subject: [PATCH 13/14] test: add keyrings.alt for CI keyring backend --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 9e52e80..3d36819 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ test = [ "pytest>=9.1.1", "pytest-asyncio>=0.23", "hypothesis>=6.100", + "keyrings.alt>=4.0", ] [project.scripts] From 0121161ea0563966d7b232ae6072c28ff9ae2384 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Sat, 22 Aug 2026 15:46:42 +0530 Subject: [PATCH 14/14] ci: add keyrings.alt to dev dependencies for CI keyring backend --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 3d36819..e7d8c9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dev = [ "ruff>=0.6", "mypy>=2.3.1", "pre-commit>=3.7", + "keyrings.alt>=4.0", ] test = [ "pytest>=9.1.1",