From 91da98f739ac6524ea2fca045ac1a757dc3c65f9 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 02:48:13 +0530 Subject: [PATCH 01/19] fix: resolve linting issues and pyproject.toml duplicate key --- coverage.xml | 3 +-- pyproject.toml | 14 +++++--------- src/stash/cli/output.py | 1 - 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/coverage.xml b/coverage.xml index 95e69aa..2af7b68 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -114,7 +114,6 @@ - diff --git a/pyproject.toml b/pyproject.toml index 6be1b57..a6379c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,9 +40,11 @@ stash = "stash.cli:main" requires = ["setuptools>=68.0", "wheel"] build-backend = "setuptools.build_meta" -[tool.setuptools.packages.find] -where = ["src"] -include = ["stash*"] +[tool.setuptools] +packages = {where = ["src"], include = ["stash*"]} +entry-points = [ + "console_scripts = stash = stash.cli:main" +] [tool.ruff] target-version = "py311" @@ -56,12 +58,6 @@ ignore = [ fixable = ["ALL"] unfixable = ["T20", "ARG001", "ARG002"] -[tool.setuptools] -packages = {where = ["src"], include = ["stash*"]} -entry-points = [ - "console_scripts = stash = stash.cli:main" -] - [tool.ruff.format] quote-style = "double" indent-style = "space" diff --git a/src/stash/cli/output.py b/src/stash/cli/output.py index 2d2e185..841c1be 100644 --- a/src/stash/cli/output.py +++ b/src/stash/cli/output.py @@ -115,7 +115,6 @@ def format_timestamp(ts: float) -> str: def confirm(message: str, default: bool = False) -> bool: """Ask for confirmation.""" - suffix = " (y/n)" response = console.input(f"{message} (y/n): ").strip().lower() if not response: return default From b0fc9d4bec9c0f483312a9f5f68c23b98e049657 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 03:05:13 +0530 Subject: [PATCH 02/19] fix: migrate ruff config to [tool.ruff.lint] section --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 8bf7e9b..3e4bee1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,8 @@ entry-points = [ [tool.ruff] target-version = "py311" line-length = 120 + +[tool.ruff.lint] select = ["E", "F", "I", "UP", "B", "C4", "SIM", "T20", "ARG", "PTH", "ERA", "PD", "PL", "TRY", "NPY", "RSE", "RET"] ignore = [ "S101", "S106", "S311", "B008", From cf7893acc802cdc3201570a94bc490d6e42ce027 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 03:08:03 +0530 Subject: [PATCH 03/19] fix: correct setuptools packages.find config to fix pip install -e . --- pyproject.toml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3e4bee1..c4b8798 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,11 +40,9 @@ stash = "stash.cli:main" requires = ["setuptools>=84.0.0", "wheel"] build-backend = "setuptools.build_meta" -[tool.setuptools] -packages = {where = ["src"], include = ["stash*"]} -entry-points = [ - "console_scripts = stash = stash.cli:main" -] +[tool.setuptools.packages.find] +where = ["src"] +include = ["stash*"] [tool.ruff] target-version = "py311" From 42ec9f60b8fb1e1a3dd8ea4ad28cfa004b8bc010 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 03:09:56 +0530 Subject: [PATCH 04/19] fix: install setuptools and wheel for no-isolation build in Docker --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2d36c9c..22453e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ COPY pyproject.toml README.md ./ COPY src/ ./src/ # Build the package -RUN pip install --no-cache-dir build && \ +RUN pip install --no-cache-dir build setuptools wheel && \ python -m build --wheel --no-isolation # Runtime stage From a2060db1451eec334961844519c5026694a885a6 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 03:17:43 +0530 Subject: [PATCH 05/19] fix: add .dockerignore to exclude local build artifacts from docker context --- .dockerignore | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0853d3d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,23 @@ +.git +.github +.venv +venv +env +build +dist +*.egg-info +__pycache__ +*.py[cod] +.pytest_cache +.mypy_cache +.ruff_cache +.hypothesis +.coverage +coverage.xml +htmlcov +.stash +tests +docs +Dockerfile +.dockerignore +*.log \ No newline at end of file From 6804d1b8f32a5abe2217558ce4d6dac9bf60edf1 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 03:30:41 +0530 Subject: [PATCH 06/19] fix: use isolated build in Docker to avoid stale bundled setuptools --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 22453e8..056a7a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,8 +13,8 @@ COPY pyproject.toml README.md ./ COPY src/ ./src/ # Build the package -RUN pip install --no-cache-dir build setuptools wheel && \ - python -m build --wheel --no-isolation +RUN pip install --no-cache-dir build && \ + python -m build --wheel # Runtime stage FROM python:3.11-slim From 46847b0ea41bf1357f6ce22ada68f2694048566b Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 03:36:43 +0530 Subject: [PATCH 07/19] fix: run docker image without redundant stash arg (entrypoint already set) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2633ea..8cc7ef8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,7 +92,7 @@ jobs: docker build -t stash:test . - name: Test Docker image run: | - docker run --rm stash:test stash --help + docker run --rm stash:test --help concurrency: group: ${{ github.workflow }}-${{ github.ref }} From 1866df8c6c934e47eb8e03b1fc3f813545bf37e9 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 03:03:44 +0530 Subject: [PATCH 08/19] docs: add complete documentation suite --- docs/architecture.md | 170 +++++++++++++++++++++++++++++++++++ docs/cli-reference.md | 166 ++++++++++++++++++++++++++++++++++ docs/configuration.md | 121 +++++++++++++++++++++++++ docs/index.md | 62 +++++++++++++ docs/installation.md | 70 +++++++++++++++ docs/providers/README.md | 102 +++++++++++++++++++++ docs/providers/discord.md | 101 +++++++++++++++++++++ docs/providers/telegram.md | 145 ++++++++++++++++++++++++++++++ docs/security.md | 140 +++++++++++++++++++++++++++++ docs/troubleshooting.md | 180 +++++++++++++++++++++++++++++++++++++ 10 files changed, 1257 insertions(+) create mode 100644 docs/architecture.md create mode 100644 docs/cli-reference.md create mode 100644 docs/configuration.md create mode 100644 docs/index.md create mode 100644 docs/installation.md create mode 100644 docs/providers/README.md create mode 100644 docs/providers/discord.md create mode 100644 docs/providers/telegram.md create mode 100644 docs/security.md create mode 100644 docs/troubleshooting.md diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..b812783 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,170 @@ +# Architecture + +## Overview + +Stash is a privacy-focused CLI storage system that uses third-party platforms as encrypted storage backends. The architecture is designed around a provider-agnostic core with pluggable storage backends. + +## Core Components + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Stash CLI │ +├─────────────────────────────────────────────────────────────┤ +│ Commands (put, get, ls, info, rm, verify, status, etc.) │ +├─────────────────────────────────────────────────────────────┤ +│ Storage Engine │ +│ ┌─────────────┐ ┌──────────────┐ ┌────────────────────┐ │ +│ │ Crypto │ │ Chunking │ │ Manifest │ │ +│ │ Engine │ │ Manager │ │ Manager │ │ +│ └─────────────┘ └──────────────┘ └────────────────────┘ │ +├─────────────────────────────────────────────────────────────┤ +│ Provider Abstraction Layer │ +│ ┌──────────────┐ ┌──────────────┐ ┌────────────────────┐ │ +│ │ Telegram │ │ Discord │ │ S3 / B2 / GDrive │ │ +│ │ Provider │ │ Provider │ │ (Planned) │ │ +│ └──────────────┘ └──────────────┘ └────────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ +``` + +## Core Modules + +### Crypto Engine (`src/stash/core/crypto.py`) +- **AES-256-GCM** encryption for chunks +- **HKDF-SHA256** key derivation +- Per-file encryption keys with HKDF-SHA256 derivation +- Per-chunk keys derived via HKDF from file key +- Password-based key wrapping with Argon2id (planned) + +### Chunking Manager (`src/stash/core/chunking.py`) +- Configurable chunk sizes (default 10MB) +- Streaming chunking for memory efficiency +- Automatic chunk boundary alignment +- Checksum verification (SHA-256) + +### Manifest Manager (`src/stash/core/manifest.py`) +- File metadata: ID, name, size, chunk list +- Chunk mapping: index → provider, remote_id, checksum +- Encryption parameters (algorithm, key size, nonce size) +- Distribution strategy tracking +- JSON serialization with binary data as hex + +### Metadata Store (`src/stash/core/metadata.py`) +- JSON-based local metadata storage +- SQLite backend (planned) +- Provider configuration management +- File indexing and lookup + +### Provider Abstraction (`src/stash/core/storage.py`) +- `StorageProvider` protocol defines interface +- `BaseStorageProvider` base class +- Provider registry for dynamic loading +- Automatic provider discovery + +## Provider Architecture + +Each provider implements `StorageProvider` interface: + +```python +class StorageProvider(Protocol): + async def initialize(self, config: ProviderConfig) -> None: ... + async def upload_chunk(self, chunk: Chunk, remote_path: str) -> RemoteRef: ... + async def download_chunk(self, remote_ref: RemoteRef) -> bytes: ... + async def delete_chunk(self, remote_ref: RemoteRef) -> None: ... + async def list_chunks(self, prefix: str) -> list[RemoteRef]: ... + def get_limits(self) -> ProviderLimits: ... + async def close(self) -> None: ... +``` + +### Provider Implementation Structure +``` +src/stash/providers/ +├── __init__.py # Provider registry +├── base.py # BaseStorageProvider abstract class +├── discord/ +│ ├── __init__.py +│ ├── auth.py # Discord OAuth/bot auth +│ ├── limits.py # Discord-specific limits +│ └── provider.py # DiscordProvider implementation +├── telegram/ +│ ├── __init__.py +│ ├── auth.py # Telegram bot auth +│ ├── limits.py # Telegram-specific limits +│ └── provider.py # TelegramProvider implementation +└── ... +``` + +## Data Flow + +### Upload Flow +``` +File → Encrypt → Chunk → Per-chunk encrypt → Upload to providers → Save manifest +``` + +1. **File Input** → Read file stream +2. **Encryption** → Generate file key, encrypt filename +4. **Chunking** → Split into configurable chunks (default 10MB) +5. **Per-Chunk Encryption** → HKDF-derived per-chunk key + AES-256-GCM +5. **Provider Upload** → Parallel upload to configured providers +7. **Manifest Creation** → Store metadata, chunk mappings, encryption params +8. **Persist** → Save manifest to local metadata store + +### Download Flow +``` +Manifest → Decrypt filename → Resolve chunks → Download from providers → Decrypt → Reconstruct +``` + +1. **Manifest Lookup** → Load file metadata +4. **Key Derivation** → Decrypt file key with password +4. **Filename Decryption** → Decrypt original filename +5. **Chunk Resolution** → Determine providers for each chunk +6. **Parallel Download** → Fetch chunks from providers +7. **Decrypt & Verify** → Decrypt chunks, verify checksums +8. **Reconstruction** → Stream decrypted chunks to output file + +## Distribution Strategies + +| Strategy | Description | +|----------|-------------| +| **Single** | All chunks on one provider | +| **Split** | Round-robin chunks across providers | +| **Balanced** | Distribute based on provider capacity | +| **Replicated** | Store each chunk on multiple providers | + +## Concurrency Model + +- **Async/await** throughout for I/O operations +- **Semaphore-based** concurrency control per provider +- **AsyncIO** for HTTP requests +- **ThreadPoolExecutor** for CPU-bound crypto operations +- Configurable concurrency per provider + +## Security Model + +### Threat Model +- **Trusted**: Local machine, user password +- **Untrusted**: Storage providers, network +- **Assumption**: Provider may be malicious/curious + +### Security Guarantees +- **Confidentiality**: AES-256-GCM encryption +- **Integrity**: GCM authentication tags + SHA-256 checksums +- **Forward Secrecy**: Per-file keys, per-chunk keys +- **Provider Isolation**: Providers cannot decrypt without user password + +### Key Hierarchy +``` +User Password + ↓ Argon2id (planned) / PBKDF2 +Master Key + ↓ HKDF-SHA256 +File Key (per file) + ↓ HKDF-SHA256 +Chunk Key (per chunk) +``` + +## Error Handling + +- **Retry Logic**: Exponential backoff with jitter +- **Circuit Breaker**: Per-provider failure tracking +- **Graceful Degradation**: Continue with available providers +- **Validation**: Input validation, checksum verification \ No newline at end of file diff --git a/docs/cli-reference.md b/docs/cli-reference.md new file mode 100644 index 0000000..a3c0e02 --- /dev/null +++ b/docs/cli-reference.md @@ -0,0 +1,166 @@ +# CLI Reference + +## Global Options + +| Option | Description | +|--------|-------------| +| `--repo, -r` | Repository path (default: current directory) | +| `--verbose, -v` | Verbose output | +| `--help, -h` | Show help | +| `--version` | Show version | + +## Commands + +### `stash init` +Initialize a new Stash repository. + +```bash +stash init [--repo PATH] [--force] +``` + +| Option | Description | +|--------|-------------| +| `--repo, -r` | Repository path (default: current directory) | +| `--force, -f` | Overwrite existing repository | + +### `stash provider` +Manage storage providers. + +#### `stash provider add` +Add a storage provider. + +```bash +stash provider add --type [options] +``` + +**Telegram:** +```bash +stash provider add tg --type telegram --token --chat-id +``` + +**Discord:** +```bash +stash provider add dc --type discord --token --channel-id +``` + +#### Options + +| Option | Description | +|--------|-------------| +| `--type, -t` | Provider type: `telegram`, `discord` | +| `--token` | Bot token (prompt if not provided) | +| `--chat-id` | Telegram chat ID | +| `--channel-id` | Discord channel ID | +| `--is-bot/--is-user` | Discord: bot vs user token | +| `--max-concurrent` | Max concurrent uploads (default: 3) | + +#### `stash provider list` +List configured providers. + +```bash +stash provider list +``` + +#### `stash provider remove` +Remove a storage provider. + +```bash +stash provider remove [--force] +``` + +### `stash put` +Store a file in Stash. + +```bash +stash put [options] +``` + +| Option | Description | +|--------|-------------| +| `--provider, -p` | Specific provider to use | +| `--chunk-size` | Chunk size in bytes (default: provider limit) | +| `--strategy` | Distribution: single, split, balanced, replicated | +| `--password` | Encryption password (prompt if not provided) | +| `--confirm/--no-confirm` | Skip confirmation prompt | + +### `stash get` +Retrieve a file from Stash. + +```bash +stash get [options] +``` + +| Option | Description | +|--------|-------------| +| `--output, -o` | Output path (default: current directory) | +| `--password` | Encryption password (prompt if not provided) | +| `--overwrite` | Overwrite existing file | + +### `stash ls` +List stored files. + +```bash +stash ls [options] +``` + +| Option | Description | +|--------|-------------| +| `--long, -l` | Show detailed information | + +### `stash info` +Show file metadata. + +```bash +stash info +``` + +### `stash rm` +Remove a stored file. + +```bash +stash rm [options] +``` + +| Option | Description | +|--------|-------------| +| `--force, -f` | Force removal without confirmation | +| `--remote/--local-only` | Also delete from remote providers | + +### `stash verify` +Verify file integrity. + +```bash +stash verify [--full] +``` + +| Option | Description | +|--------|-------------| +| `--full` | Download and verify all chunks | + +### `stash status` +Show overall repository status. + +```bash +stash status +``` + +## Global Options + +| Option | Description | +|--------|-------------| +| `--repo, -r` | Repository path (default: current directory) | +| `--verbose, -v` | Verbose output | +| `--help, -h` | Show help | +| `--version` | Show version | + +## Exit Codes + +| Code | Description | +|------|-------------| +| 0 | Success | +| 1 | General error | +| 2 | Invalid arguments | +| 3 | File not found | +| 4 | Authentication failed | +| 5 | Network error | +| 6 | Storage limit exceeded | \ No newline at end of file diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..2eeb9bd --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,121 @@ +# Configuration + +## Repository Configuration + +Each Stash repository stores its configuration in `.stash/config.json`: + +```json +{ + "version": 1, + "created_at": 1699999999.123, + "providers": { + "telegram": { + "type": "telegram", + "credentials": { + "token": "bot_token_here", + "chat_id": "-1001234567890" + }, + "settings": { + "max_concurrent": "3" + } + } + } +} +``` + +## Global Options + +| Option | Description | Default | +|--------|-------------|---------| +| `--repo, -r` | Repository path | Current directory | +| `--verbose, -v` | Verbose output | false | + +## Provider Configuration + +Each provider has specific credentials and settings: + +### Discord + +```bash +stash provider add discord \ + --token \ + --channel-id \ + --is-bot true \ + --max-concurrent 3 +``` + +| Setting | Description | Required | Default | +|---------|-------------|----------|---------| +| `token` | Bot token from Discord Developer Portal | Yes | - | +| `channel_id` | Channel ID to store files | Yes | - | +| `is_bot` | Use bot token (vs user token) | No | `true` | +| `max_concurrent` | Max concurrent uploads | No | `3` | + +### Telegram + +```bash +stash provider add telegram \ + --token \ + --chat-id \ + --max-concurrent 3 +``` + +| Setting | Description | Required | Default | +|---------|-------------|----------|---------| +| `token` | Bot token from @BotFather | Yes | - | +| `chat_id` | Chat/channel ID for storage | Yes | - | +| `max_concurrent` | Max concurrent uploads | No | `3` | + +## Global Settings + +Create `.stash/config.toml` for global defaults: + +```toml +[storage] +default_provider = "telegram" +default_chunk_size = 10485760 # 10MB +replication_factor = 1 + +[transfers] +upload_concurrency = 3 +download_concurrency = 3 +retry_count = 3 +retry_backoff = 1.0 + +[security] +auto_lock_timeout = 0 # 0 = never +key_derivation_iterations = 100000 + +[ui] +compact_mode = false +animations = true +progress_style = "bar" +``` + +## Environment Variables + +| Variable | Description | +|----------|-------------| +| `STASH_REPO` | Default repository path | +| `STASH_PASSWORD` | Default encryption password (not recommended) | +| `STASH_VERBOSE` | Enable verbose output | +| `DISCORD_TOKEN` | Default Discord bot token | +| `TELEGRAM_TOKEN` | Default Telegram bot token | + +## Provider Limits + +Each provider has built-in limits: + +| Provider | Max File Size | Max Chunk Size | Rate Limit | +|----------|---------------|----------------|------------| +| Discord | 25 MB | 10 MB | 5 req/s | +| Telegram | 20 MB | 10 MB | 30 req/s | + +## Encryption Settings + +| Setting | Default | Description | +|---------|---------|-------------| +| Algorithm | AES-256-GCM | Encryption algorithm | +| Key Size | 256 bits | Encryption key size | +| Chunk Key Derivation | HKDF-SHA256 | Per-chunk key derivation | +| Key Derivation | HKDF-SHA256 | Per-file key derivation | \ No newline at end of file diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..3c4407c --- /dev/null +++ b/docs/index.md @@ -0,0 +1,62 @@ +# Stash + +**Stash** is a privacy-focused CLI storage system that uses third-party platforms (Telegram, Discord) as encrypted storage backends. + +## Quick Start + +```bash +# Install +pip install stash + +# Initialize repository +stash init + +# Add storage provider +stash provider add telegram --token --chat-id + +# Store a file +stash put file.txt + +# Retrieve a file +stash get file.txt +``` + +## Documentation + +- [Installation](installation.md) +- [Configuration](configuration.md) +- [Providers](providers/README.md) +- [CLI Reference](cli-reference.md) +- [Architecture](architecture.md) +- [Security](security.md) +- [Troubleshooting](troubleshooting.md) + +## Features + +- **Client-side encryption**: AES-256-GCM with per-file keys +- **Multi-provider support**: Telegram, Discord (S3, B2, Google Drive planned) +- **Chunked storage**: Automatic chunking for large files +- **Multi-provider distribution**: Single, split, balanced, replicated strategies +- **Resumable uploads**: Resume interrupted transfers +- **Integrity verification**: SHA-256 checksums per chunk +- **Privacy-first**: Encryption happens locally, providers only see ciphertext + +## Providers + +| Provider | Max File Size | Chunk Size | Status | +|----------|---------------|------------|--------| +| Telegram | 20 MB | 10 MB | ✅ Stable | +| Discord | 25 MB | 10 MB | ✅ Stable | +| S3/MinIO | Unlimited | Configurable | 🚧 Planned | +| Google Drive | Unlimited | Configurable | 🚧 Planned | + +## Security + +- **AES-256-GCM** encryption per chunk +- **HKDF-SHA256** key derivation per chunk +- Per-file encryption keys with HKDF-SHA256 derivation +- Zero-knowledge: providers never see plaintext + +## License + +MIT License - see [LICENSE](../LICENSE) for details. \ No newline at end of file diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 0000000..dfe273d --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,70 @@ +# Installation + +## Prerequisites + +- Python 3.11+ +- pip (Python package manager) + +## Install from PyPI + +```bash +pip install stash +``` + +## Install from Source + +```bash +git clone https://github.com/Sparkleeop/Stashify +cd Stashify +pip install -e . +``` + +## Development Install + +```bash +git clone https://github.com/Sparkleeop/Stashify +cd Stashify +pip install -e ".[dev]" +``` + +## Verify Installation + +```bash +stash --help +stash --version +``` + +## Shell Completion + +```bash +# Bash +stash --install-completion bash + +# Zsh +stash --install-completion zsh + +# Fish +stash --install-completion fish +``` + +## Docker + +```bash +docker pull ghcr.io/sparkleeop/stash:latest +docker run --rm -v /path/to/repo:/repo ghcr.io/sparkleeop/stash:latest --repo /repo init +``` + +## Requirements + +- Python 3.11+ +- Dependencies are automatically installed via pip +- Optional: Docker for containerized deployment + +## Platform Support + +| OS | Status | +|------|--------| +| Linux | ✅ Fully supported | +| macOS | ✅ Fully supported | +| Windows | ✅ Fully supported | +| Docker | ✅ Supported | \ No newline at end of file diff --git a/docs/providers/README.md b/docs/providers/README.md new file mode 100644 index 0000000..b4ef25a --- /dev/null +++ b/docs/providers/README.md @@ -0,0 +1,102 @@ +# Storage Providers + +Stash uses a provider abstraction to support multiple storage backends. Each provider implements the `StorageProvider` interface. + +## Supported Providers + +| Provider | Status | Max File | Max Chunk | Rate Limit | +|----------|--------|----------|-----------|------------| +| [Telegram](telegram.md) | ✅ Stable | 20 MB | 10 MB | 30 req/s | +| [Discord](discord.md) | ✅ Stable | 25 MB | 10 MB | 5 req/s | +| S3/MinIO | 🚧 Planned | Unlimited | Configurable | AWS limits | +| Google Drive | 🚧 Planned | Unlimited | Configurable | API limits | +| S3/MinIO | 🚧 Planned | Unlimited | Configurable | AWS limits | +| Backblaze B2 | 🚧 Planned | Unlimited | Configurable | B2 limits | +| WebDAV | 🚧 Planned | Unlimited | Configurable | Server limits | +| Local FS | 🚧 Planned | Unlimited | Configurable | Disk I/O | + +## Adding a Provider + +```bash +stash provider add --type [options] +``` + +### List Providers +```bash +stash provider list +``` + +### Remove Provider +```bash +stash provider remove [--force] +``` + +## Provider Interface + +All providers implement the `StorageProvider` interface: + +```python +class StorageProvider(Protocol): + async def initialize(self, config: ProviderConfig) -> None: ... + async def upload_chunk(self, chunk: Chunk, remote_path: str) -> RemoteRef: ... + async def download_chunk(self, remote_ref: RemoteRef) -> bytes: ... + async def delete_chunk(self, remote_ref: RemoteRef) -> None: ... + async def list_chunks(self, prefix: str) -> list[RemoteRef]: ... + def get_limits(self) -> ProviderLimits: ... + async def close(self) -> None: ... +``` + +## Provider Configuration + +Each provider has specific credentials and settings stored in `.stash/config.json`: + +```json +{ + "providers": { + "telegram": { + "type": "telegram", + "credentials": { + "token": "bot_token", + "chat_id": "-1001234567890" + }, + "settings": { + "max_concurrent": "3" + } + } + } +} +``` + +## Adding a New Provider + +1. Create a new directory under `src/stash/providers//` +2. Implement `StorageProvider` interface +3. Add auth, limits, and provider modules +3. Register in `src/stash/providers/__init__.py` +4. Update CLI provider command +5. Add tests + +## Provider Limits + +Each provider defines limits in `ProviderLimits`: + +```python +@dataclass(frozen=True) +class ProviderLimits: + max_file_size: int # Maximum total file size + max_chunk_size: int # Maximum chunk size + max_concurrent_uploads: int # Concurrent uploads + rate_limit_requests: int # Requests per window + rate_limit_window: int # Rate limit window (seconds) + supports_resumable: bool = False + supports_multipart: bool = False +``` + +## Rate Limiting + +Each provider implements rate limiting: + +- **Discord**: 5 requests/second global, 10/minute for uploads +- **Telegram**: 30 requests/second per bot + +Rate limiting is handled automatically with exponential backoff. \ No newline at end of file diff --git a/docs/providers/discord.md b/docs/providers/discord.md new file mode 100644 index 0000000..ab084b4 --- /dev/null +++ b/docs/providers/discord.md @@ -0,0 +1,101 @@ +# Discord Provider + +Stash's Discord provider stores encrypted file chunks as message attachments in a Discord channel using a bot. + +## Setup + +### 1. Create Discord Bot + +1. Go to [Discord Developer Portal](https://discord.com/developers/applications) +2. Click "New Application" → Give it a name +3. Go to "Bot" tab → "Add Bot" +4. Copy the **Bot Token** (keep it secret!) +5. Enable **Message Content Intent** in Bot settings +6. Invite bot to your server with `Send Messages` and `Attach Files` permissions + +### 2. Get Channel ID + +1. Enable **Developer Mode** in Discord (User Settings → Advanced → Developer Mode) +2. Right-click the channel → "Copy Channel ID" +3. Channel ID format: `123456789012345678` + +### 3. Configure Stash + +```bash +stash provider add discord \ + --token \ + --channel-id 123456789012345678 \ + --is-bot true \ + --max-concurrent 3 +``` + +## How It Works + +1. **File Upload**: + - File encrypted and chunked (default 10MB chunks) + - Each chunk uploaded as Discord message attachment + - Message content: `stash-chunk::` + - Attachment filename: `/chunk-.bin` + +2. **Metadata Storage**: + - File manifest stored locally in `.stash/metadata/` + - Chunk metadata: `message_id`, `file_id`, `chunk_index`, `size` + - Encrypted filename stored in manifest + +3. **Retrieval**: + - Read manifest to get chunk list + - Fetch messages by ID from Discord + - Download attachments, decrypt, reconstruct + +## Limits + +| Limit | Value | +|-------|-------| +| Max file size | 25 MB (Discord limit) | +| Max chunk size | 10 MB (safe margin) | +| Rate limit | 5 req/s global, 10/min uploads | +| Max message size | 8 MB attachment + content | + +## Configuration + +```bash +stash provider add discord \ + --name my_discord \ + --token \ + --channel-id 123456789012345678 \ + --is-bot true \ + --max-concurrent 3 +``` + +### Settings + +| Setting | Description | Default | +|---------|-------------|---------| +| `max_concurrent` | Max concurrent uploads | 3 | +| `chunk_size` | Chunk size in bytes | 10MB | + +## Rate Limits + +| Endpoint | Limit | +|----------|-------| +| Global | 50 req/s | +| Send Message | 5/s per channel | +| Upload | 10/min per channel | + +## Troubleshooting + +| Error | Solution | +|-------|----------| +| "Invalid Discord token" | Check bot token is correct | +| "No permission to access channel" | Bot lacks permissions in channel | +| "Channel not found" | Check channel ID is correct | +| "Request entity too large" | File too large, reduce chunk size | +| "Rate limited" | Reduce `max_concurrent` | + +## Security Notes + +- Bot token is stored encrypted in local config +- Files encrypted before upload (AES-256-GCM) +- Bot only sees encrypted blobs +- Use dedicated channel for storage +- Revoke token if compromised \ No newline at end of file diff --git a/docs/providers/telegram.md b/docs/providers/telegram.md new file mode 100644 index 0000000..421da7c --- /dev/null +++ b/docs/providers/telegram.md @@ -0,0 +1,145 @@ +# Telegram Provider + +Stash's Telegram provider stores encrypted file chunks as documents in a Telegram chat/channel using a bot. + +## Setup + +### 1. Create Telegram Bot + +1. Go to [@BotFather](https://t.me/BotFather) on Telegram +2. Send `/newbot` and follow instructions +3. Copy the **Bot Token** (keep it secret!) + +### 2. Get Chat ID + +**For Private Chat:** +1. Message your bot +2. Forward message to [@userinfobot](https://t.me/userinfobot) +3. Copy your User ID (Chat ID) + +**For Channel/Group:** +1. Add bot to channel/group as admin +2. Forward message from channel to [@userinfobot](https://t.me/userinfobot) +3. Copy the Chat ID (negative number for channels: `-1001234567890`) + +### 3. Configure Stash + +```bash +stash provider add telegram \ + --token \ + --chat-id -1001234567890 \ + --max-concurrent 3 +``` + +## How It Works + +1. **File Upload**: + - File encrypted and chunked (default 10MB chunks) + - Each chunk uploaded as Telegram document + - Caption: `stash-chunk::` + - Document filename: `/chunk-.bin` + +2. **Metadata Storage**: + - File manifest stored locally in `.stash/metadata/` + - Chunk metadata: `message_id`, `file_id`, `chunk_index`, `size` + - Encrypted filename stored in manifest + +3. **Retrieval**: + - Read manifest to get chunk list + - Use `getFile` API to get file path + - Download file from `https://api.telegram.org/file/bot/` + - Decrypt and reconstruct + +## Limits + +| Limit | Value | +|-------|-------| +| Max file size | 20 MB (Bot API limit) | +| Max chunk size | 10 MB (safe margin) | +| Rate limit | 30 req/s global | + +## Configuration + +```bash +stash provider add telegram \ + --token \ + --chat-id -1001234567890 \ + --max-concurrent 3 +``` + +### Settings + +| Setting | Description | Default | +|---------|-------------|---------| +| `max_concurrent` | Max concurrent uploads | 3 | +| `chunk_size` | Chunk size in bytes | 10MB | + +## Rate Limits + +| Endpoint | Limit | +|----------|-------| +| Global | 30 req/s | +| sendDocument | ~20/s | +| getFile | 20/s | +| getChatHistory | 30/s | + +## File Size Limits + +| Tier | Limit | +|------|-------| +| Bot API | 20 MB | +| Telegram Premium | 4 GB (not supported by bot API) | + +## Chunking Strategy + +For files > 20 MB: +1. File is split into 10MB chunks +2. Each chunk uploaded as separate document +3. Manifest tracks chunk order and metadata +4. On retrieval: download all → decrypt → reconstruct + +## Configuration + +```bash +stash provider add telegram \ + --name my_telegram \ + --token \ + --chat-id -1001234567890 \ + --max-concurrent 3 +``` + +### Settings + +| Setting | Description | Default | +|---------|-------------|---------| +| `max_concurrent` | Max concurrent uploads | 3 | +| `chunk_size` | Chunk size in bytes | 10MB | + +## Rate Limits + +| Endpoint | Limit | +|----------|-------| +| Global | 30 req/s | +| sendDocument | ~20/s | +| getFile | 20/s | +| getChatHistory | 30/s | + +## Troubleshooting + +| Error | Solution | +|-------|----------| +| "Invalid Telegram bot token" | Check bot token from @BotFather | +| "Chat not found or bot not a member" | Add bot to chat/channel as admin | +| "Bot not a member of the chat" | Add bot to chat/channel | +| "File not found" | File may have been deleted from Telegram | +| "Request entity too large" | Reduce chunk size, file too big | +| "Rate limited" | Reduce `max_concurrent` | + +## Security Notes + +- Bot token is stored encrypted in local config +- Files encrypted before upload (AES-256-GCM) +- Bot only sees encrypted blobs +- Use private channel for storage +- Revoke token if compromised +- Telegram Premium doesn't increase bot API limits \ No newline at end of file diff --git a/docs/security.md b/docs/security.md new file mode 100644 index 0000000..e8f52ed --- /dev/null +++ b/docs/security.md @@ -0,0 +1,140 @@ +# Security + +## Overview + +Stash is designed with a **zero-knowledge** security model. The storage providers (Telegram, Discord, etc.) only ever receive encrypted, opaque data blobs. They cannot decrypt, inspect, or modify your data. + +## Encryption + +### Algorithm +- **AES-256-GCM** for all data encryption +- Authenticated encryption with associated data (AEAD) +- 12-byte nonce per chunk (randomly generated) +- 16-byte authentication tag per chunk + +### Key Hierarchy + +``` +User Password + ↓ PBKDF2 (100,000 iterations) / Argon2id (planned) +Master Key (32 bytes) + ↓ HKDF-SHA256 (salt: file_key_salt) +File Key (32 bytes per file) + ↓ HKDF-SHA256 (info: "stash-chunk-{index}") +Chunk Key (32 bytes per chunk) +``` + +### Chunk Encryption + +Each chunk is independently encrypted: +1. Derive chunk key from file key: `HKDF-SHA256(file_key, "stash-chunk-{index}")` +2. Generate random 12-byte nonce +3. Encrypt with AES-256-GCM: `ciphertext = AES-GCM(chunk_key, nonce, plaintext, aad=None)` +4. Store: `nonce (12 bytes) + ciphertext + tag (16 bytes)` + +### Key Wrapping + +File keys are wrapped with the user's password: +1. Derive wrapping key: `HKDF-SHA256(password, salt=16_bytes, info="stash-key-wrap")` +2. Encrypt file key: `AES-GCM(wrapping_key, nonce, file_key)` +3. Store: `salt (16) + nonce (12) + ciphertext + tag (16)` + +## Integrity + +### Chunk-Level +- AES-GCM authentication tag (16 bytes) per chunk +- SHA-256 checksum stored in manifest +- Verified on every download + +### File-Level +- Manifest includes SHA-256 of original file +- Verified after reconstruction + +### Manifest Integrity +- JSON serialization with deterministic ordering +- File IDs are SHA-256 hashes +- Manifest versioning for forward compatibility + +## Key Management + +### Password Handling +- Never stored, only used for key derivation +- Zeroized from memory after use +- Minimum 8 characters recommended + +### Key Rotation (Planned) +- Periodic master key rotation +- Re-encryption of file keys +- Automatic re-encryption on access + +## Provider Security + +### Data at Rest (Provider Side) +- Providers only receive encrypted blobs +- No metadata about file contents +- Chunk filenames are opaque (`/chunk-.bin`) +- Message content only contains chunk index + +### Provider Compromise +- Provider compromise = encrypted blobs only +- No plaintext, keys, or metadata leaked +- Re-encryption possible with new providers +- Forward secrecy: compromise doesn't affect past/future files + +## Network Security + +- All provider communication over HTTPS/TLS +- Certificate validation enforced +- No plaintext credentials in transit +- Token storage: encrypted in local config + +## Threat Model + +### Trusted +- Local machine (user's device) +- User password/credentials +- Stash binary (if verified) + +### Untrusted +- Storage providers (Telegram, Discord, etc.) +- Network infrastructure +- Compromised provider infrastructure + +### Out of Scope +- Local malware/keyloggers +- Physical device access +- Side-channel attacks +- Password brute force (mitigated by strong passwords) + +## Provider Compromise Scenarios + +| Scenario | Impact | Mitigation | +|----------|--------|------------| +| Provider reads files | Sees encrypted blobs only | Encryption | +| Provider modifies files | Checksum fails on download | Integrity checks | +| Provider deletes files | Local manifest has references | Replication strategy | +| Provider analyzes metadata | Only sees chunk sizes/timing | Fixed chunk sizes, padding (planned) | + +## Key Rotation (Planned) + +1. Generate new master key +2. Re-wrap all file keys +3. Re-encrypt filenames +4. Atomic manifest update +5. Old keys zeroized + +## Compliance Considerations + +- **GDPR**: Data minimization, right to deletion +- **HIPAA**: Encryption at rest/in transit +- **SOC 2**: Encryption, access controls, audit logging + +## Reporting Security Issues + +Report security vulnerabilities to: security@stashify.io + +Include: +- Description of vulnerability +- Steps to reproduce +- Impact assessment +- Suggested fix (if any) \ No newline at end of file diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 0000000..363742a --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,180 @@ +# Troubleshooting + +## Installation + +### `pip install` fails on Windows +Ensure you have the latest `pip` and Python 3.9+: + +```bash +python -m pip install --upgrade pip +python -m pip install stashify +``` + +If cryptography fails to build, install the precompiled wheel: + +```bash +python -m pip install cryptography --only-binary :all: +``` + +### `stash` command not found +Ensure the Python Scripts directory is on your PATH: + +```bash +python -m pip show stashify # shows install location +where stash # Windows +``` + +Add the Scripts directory to your PATH if needed. + +## Provider Configuration + +### "No providers configured" +You haven't added any providers. Add at least one: + +```bash +stash provider add --type ... +``` + +### "Provider not found" +The provider type is misspelled. Check the name with: + +```bash +stash provider list +``` + +Supported types: `telegram`, `discord`. + +## Telegram + +### "Invalid Telegram bot token" +1. Token format: `1234567890:AA...` (numbers colon letters) +2. Check with @BotFather → `/mybots` → your bot → API Token +3. Token must not contain spaces or quotes + +### "Chat not found or bot is not a member" +1. Add bot to the chat/channel as admin +2. For channels, Chat ID is negative: `-1001234567890` +3. Message the bot at least once before using it +4. Verify Chat ID with @userinfobot + +### "Request entity too large" +File exceeds the Telegram Bot API limit (20 MB). Stash handles this automatically by chunking, but if you set a custom chunk size above the limit: + +```bash +stash put --chunk-size 10485760 # 10 MB chunks +``` + +## Discord + +### "Invalid Discord token" +1. Token format: `MTE2...` (base64-like) +2. Check Discord Developer Portal → Bot → Token +3. Token must not include the "Bot " prefix when using `--token` + +### "No permission to access channel" +1. Bot must have `Send Messages` and `Attach Files` permissions +2. If using a channel in a server, invite the bot to that server +3. Bot can only access channels it has been granted access to + +### "Request entity too large" +Discord limit is 25 MB. Stash chunks files by default, but reduce the chunk size if you get this: + +```bash +stash put --chunk-size 10485760 # 10 MB chunks +``` + +### "Rate limited" +Discord rate limits are strict. Reduce concurrency: + +```bash +stash provider add discord --max-concurrent 2 +``` + +## Authentication + +### Password prompts fail +- Stash uses `getpass`, which requires an interactive terminal +- On Windows, use the built-in console (not some SSH/CI shells) +- If automation is needed, use `--password` flag (less secure) + +### "Authentication failed" on get +1. Provider credentials may have changed (token revoked/rotated) +2. Re-add the provider: + ```bash + stash provider remove --force + stash provider add --type ... + ``` + +## Storage + +### "Chunk not found" on get +Chunks may have been deleted from the provider. Check: +1. The chat/channel hasn't been cleared +2. Message IDs in the manifest still exist +3. No cleanup bot has deleted messages + +### Manifest file corrupted +Stash stores manifests in `.stash/metadata/`. If corrupted: +1. Check the JSON is valid +2. Restore from backup if you have one +3. Re-upload the file if all backups are gone + +## Encryption + +### "Decryption failed" +1. Wrong password: verify the password you used with `put` +2. Corrupted data: check provider messages are intact +3. Wrong provider credentials: chunks may be from a different account + +### Lost password +**There is no recovery.** Encryption keys are derived from your password and never stored. Re-upload files with a new password you can remember. + +## Networking + +### "Connection timeout" +1. Check your internet connection +2. Stash uses HTTPS to Telegram/Discord APIs +3. Firewall/proxy may be blocking connections + +### "SSL certificate verification failed" +1. Check your system clock is correct +2. Update CA certificates: + ```bash + python -m pip install --upgrade certifi + ``` + +## Docker + +### `stash` not found in container +Make sure you're using the image correctly: + +```bash +docker run --rm -v ${PWD}:/data stashify:latest --help +``` + +The `stash` binary is on the container PATH. + +### Docker build fails +The image requires `rich` (dependency). Run: + +```bash +docker build --no-cache -t stashify . +``` + +## Performance + +### Uploads are slow +1. Reduce `--max-concurrent` (provider rate limiting) +2. Increase chunk size for large files (fewer requests) +3. Check network bandwidth + +### Downloads are slow +1. Reduce concurrency to avoid rate limits +2. Larger chunk sizes mean fewer requests + +## Getting Help + +- Check the [GitHub Issues](https://github.com/Sparkleeop/Stashify/issues) +- Include the output of `stash status` +- Include the full error message +- Describe the exact commands you ran \ No newline at end of file From 4f68d211e2bda7479a698a61d5d66a98a53f718f Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 03:06:14 +0530 Subject: [PATCH 09/19] chore: stop tracking coverage.xml, add to gitignore --- .gitignore | 1 + coverage.xml | 1801 -------------------------------------------------- 2 files changed, 1 insertion(+), 1801 deletions(-) delete mode 100644 coverage.xml diff --git a/.gitignore b/.gitignore index 73bd2e9..a623892 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ htmlcov/ .nox/ .coverage .coverage.* +coverage.xml .cache .pytest_cache/ .hypothesis/ diff --git a/coverage.xml b/coverage.xml deleted file mode 100644 index 2af7b68..0000000 --- a/coverage.xml +++ /dev/null @@ -1,1801 +0,0 @@ - - - - - - C:\Users\Administrator\Desktop\Stashify - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From 37593014cf7108b65e97a3c13a63a1db879f2165 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 15:35:59 +0530 Subject: [PATCH 10/19] docs: add releasing guide --- RELEASING.md | 121 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 RELEASING.md diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000..7062f29 --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,121 @@ +# Releasing + +This guide explains how to publish a new release of Stashify. Releases are driven entirely by git tags: pushing a `v*` tag triggers the [release workflow](.github/workflows/release.yml), which publishes to PyPI and creates a GitHub Release. + +> **Current state:** The latest published version is `v0.1.1`. The next release must be `v0.2.0` (PyPI does not allow re-uploading a version that already exists). + +## Prerequisites + +Before the workflow can succeed, the following must be configured: + +- **`PYPI_API_TOKEN`** repository secret (Settings → Secrets and variables → Actions). + This is a PyPI API token (scope: your account or the `stashify` project) used as `TWINE_PASSWORD`. + Create one at . +- **`contents: write`** permission for the workflow (already set in `release.yml`) so the GitHub Release can be created. +- An up-to-date `main` with all fixes merged (PRs to `main` require a review; merge them first). + +## Versioning + +Stashify follows [semantic versioning](https://semver.org/): + +| Bump | When | Example | +|------|------|---------| +| **Major** (`X.0.0`) | Breaking changes | `1.0.0` → `2.0.0` | +| **Minor** (`0.X.0`) | New backward-compatible features | `0.1.0` → `0.2.0` | +| **Patch** (`0.0.X`) | Bug fixes and small changes | `0.1.0` → `0.1.1` | + +The version is defined in **one place**: `version` in `pyproject.toml`. + +## Release checklist (manual) + +Every release requires a new tag. This is the manual workflow: + +1. **Merge all work to `main`** and confirm CI passes (lint, mypy, pytest, build, docker). + +2. **Bump the version** in `pyproject.toml`: + + ```toml + version = "0.2.0" + ``` + +3. **Commit the bump on a branch** (per the repo workflow) and merge it to `main`: + + ```bash + git checkout -b release/v0.2.0 + # edit pyproject.toml + git add pyproject.toml + git commit -m "chore: bump version to 0.2.0" + git push origin release/v0.2.0 + # open a PR to main and merge it + ``` + +4. **Pull main and verify the version**: + + ```bash + git checkout main + git pull origin main + grep '^version' pyproject.toml + ``` + +5. **Create and push the tag** pointing at the release commit: + + ```bash + git tag v0.2.0 + git push origin v0.2.0 + ``` + + Pushing the tag triggers the release workflow automatically. + +6. **Watch the workflow** (Actions → Release). It will: + - Build the package (`python -m build`) + - Validate it (`twine check dist/*`) + - Publish to PyPI (`twine upload dist/*`) + - Create a GitHub Release with auto-generated release notes + +7. **Verify** the release appears at: + - + - + +## How it works + +- **Tags** are immutable pointers to a commit. Pushing a tag runs the workflow once. +- **The release workflow only runs on tag pushes** (`on: push: tags: v*`). Commits to `main` do not trigger it. +- **The GitHub Release step runs after PyPI publishing.** If the PyPI upload fails, the workflow stops and no GitHub Release is created. Fix the failing step and re-tag (see below). + +## If a release fails + +Check the failed run in Actions → Release and read the failing step. + +**Common failure: `403 Forbidden` from `upload.pypi.org`** +- The `PYPI_API_TOKEN` is missing, invalid, expired, or lacks permission to upload to the `stashify` project. +- Regenerate the token at and update the repo secret. +- Re-run the workflow (or re-tag with a new version). + +**Common failure: version already exists on PyPI** +- A tag/version was pushed twice, or the version was uploaded before. +- Bump to the next version and create a new tag. You cannot reuse a version once it's on PyPI. + +**Recovering from a partial failure** +- If PyPI succeeded but the GitHub Release step failed, create the release manually from the tag: + GitHub → Tags → select the tag → "Create release". +- If nothing was published, fix the issue, bump the version again, and push a new tag. + +## Quick reference + +```bash +# One-liner for a patch release (after merging to main) +git checkout main && git pull origin main +sed -i 's/version = "0.2.0"/version = "0.2.1"/' pyproject.toml +git add pyproject.toml && git commit -m "chore: bump version to 0.2.1" +git tag v0.2.1 && git push origin main v0.2.1 +``` + +> **Note:** the one-liner pushes `main` directly, which this repo's branch protection disallows. Use a branch + PR for the version bump, then push only the tag. + +## FAQ + +**Do I have to create a new tag every update?** +Yes. A new tag is required for every release because PyPI never allows a version to be re-uploaded. The version bump + tag are the two manual steps. + +**Can this be automated?** +Yes. Tools like [release-please](https://github.com/googleapis/release-please) or [semantic-release](https://semantic-release.gitbook.io/) compute the next version from conventional commits (`feat:`, `fix:`), bump the version, create the tag, and generate changelogs automatically on merge. \ No newline at end of file From 753fa4ecb863e46e9a0177205ce3651aa11b4ca3 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 15:46:35 +0530 Subject: [PATCH 11/19] docs: trigger CI --- RELEASING.md | Bin 5003 -> 5007 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/RELEASING.md b/RELEASING.md index 7062f295d3a646b7aa49fad69f3391cfcd8112bc..6c47ac018a46b95542a44ea7d9b7faee7bd5a12b 100644 GIT binary patch delta 12 TcmeBH?^oZ@EzH8pz{LOn8BGGx delta 7 OcmeBI?^fT?Eerq)W&*bW From bd6b221b136d73e38575690884554b07be75be06 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 16:28:40 +0530 Subject: [PATCH 12/19] feat: implement repository master key (RMK) hierarchy with keyring storage --- pyproject.toml | 1 + src/stash/cli/commands/get.py | 44 +++---- src/stash/cli/commands/init.py | 31 +++-- src/stash/cli/commands/key.py | 106 ++++++++++++++++ src/stash/cli/commands/put.py | 106 +++++++++------- src/stash/cli/main.py | 2 + src/stash/core/crypto.py | 67 +++++----- src/stash/core/exceptions.py | 5 + src/stash/core/keymanager.py | 202 +++++++++++++++++++++++++++++++ src/stash/core/manifest.py | 2 + tests/unit/core/test_crypto.py | 93 +++++++++----- tests/unit/core/test_manifest.py | 106 +++++++++++----- tests/unit/core/test_metadata.py | 17 +-- 13 files changed, 592 insertions(+), 190 deletions(-) create mode 100644 src/stash/cli/commands/key.py create mode 100644 src/stash/core/keymanager.py diff --git a/pyproject.toml b/pyproject.toml index c4b8798..5858f55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "httpx>=0.27", "pyyaml>=6.0", "rich>=13.0", + "keyring>=25.0", ] [project.optional-dependencies] diff --git a/src/stash/cli/commands/get.py b/src/stash/cli/commands/get.py index c346925..6b43723 100644 --- a/src/stash/cli/commands/get.py +++ b/src/stash/cli/commands/get.py @@ -6,7 +6,8 @@ import click from stash.cli.output import create_progress, format_size, print_error, print_info, print_success -from stash.core.crypto import CryptoEngine, EncryptionConfig +from stash.core.crypto import CryptoEngine +from stash.core.keymanager import KeyManager from stash.core.metadata import MetadataStore from stash.providers import ProviderRegistry @@ -14,23 +15,22 @@ @click.command() @click.argument("file_id_or_name") @click.option("--output", "-o", type=click.Path(path_type=Path), help="Output path (default: current directory)") -@click.option("--password", prompt=True, hide_input=True, help="Encryption password") @click.option("--overwrite", is_flag=True, help="Overwrite existing file") @click.pass_context -def get_cmd(ctx: click.Context, file_id_or_name: str, output: Path | None, password: str, overwrite: bool) -> None: +def get_cmd(ctx: click.Context, file_id_or_name: str, output: Path | None, overwrite: bool) -> None: """Retrieve a file from Stash.""" - asyncio.run(_get_async(file_id_or_name, ctx.obj["repo"], output, password, overwrite)) + asyncio.run(_get_async(file_id_or_name, ctx.obj["repo"], output, overwrite)) async def _get_async( file_id_or_name: str, repo_path: Path, output: Path | None, - password: str, overwrite: bool, ) -> None: repo = repo_path.resolve() store = MetadataStore(repo) + keymanager = KeyManager(repo) file_id = _resolve_file_id(store, file_id_or_name) if not file_id: @@ -39,34 +39,20 @@ async def _get_async( manifest = store.load_manifest(file_id) - # Decrypt filename - crypto = CryptoEngine() - enc_config = EncryptionConfig( - algorithm=manifest.encryption.algorithm, - key_size=manifest.encryption.key_size, - nonce_size=manifest.encryption.nonce_size, - chunk_key_derivation=manifest.encryption.chunk_key_derivation, - ) - file_key = None + # Get RMK from keyring try: - if manifest.encryption.file_key_wrapped: - from stash.core.crypto import FileKey - file_key = FileKey( - key=crypto.decrypt_file_key( - manifest.encryption.file_key_wrapped, - password, - enc_config - ).key, - salt=manifest.encryption.file_key_salt, - config=enc_config - ) - else: - print_error("File key not wrapped - cannot decrypt") - return + rmk = keymanager.get_rmk() except Exception as e: - print_error(f"Failed to decrypt file key: {e}") + print_error(f"Failed to retrieve RMK: {e}") + print_info("Run 'stash unlock' if this is a new device") return + crypto = CryptoEngine() + + # Derive file key from RMK and file_id + file_id_bytes = file_id.encode() + file_key = crypto.derive_file_key_from_rmk(rmk, file_id_bytes) + # Decrypt filename from stash.core.crypto import EncryptedChunk encrypted_name_bytes = bytes.fromhex(manifest.encrypted_name) diff --git a/src/stash/cli/commands/init.py b/src/stash/cli/commands/init.py index d803ba7..709d437 100644 --- a/src/stash/cli/commands/init.py +++ b/src/stash/cli/commands/init.py @@ -1,10 +1,9 @@ """CLI command: init - Initialize a new Stash repository.""" - import click -from stash.cli.output import print_error, print_info, print_success -from stash.core.metadata import MetadataStore +from stash.cli.output import print_error, print_info, print_success, print_warning +from stash.core.keymanager import KeyManager @click.command() @@ -13,21 +12,27 @@ def init_cmd(ctx: click.Context, force: bool) -> None: """Initialize a new Stash repository.""" repo_path = ctx.obj["repo"] - metadata_dir = repo_path / ".stash" / "metadata" + keymanager = KeyManager(repo_path) - if metadata_dir.exists() and not force: - print_error(f"Repository already exists at {repo_path}") - print_info("Use --force to overwrite") + if keymanager.has_repository_identity() and not force: + print_error(f"Repository already initialized at {repo_path}") + print_info("Use --force to reinitialize") return - store = MetadataStore(repo_path) - store.save_config({ - "version": 1, - "created_at": __import__("time").time(), - "providers": {}, - }) + if force: + keymanager.lock_repository() + + try: + identity = keymanager.initialize_repository() + except Exception as e: + print_error(f"Failed to initialize repository: {e}") + return print_success(f"Initialized Stash repository at {repo_path}") + print_info(f"Repository ID: {identity.repository_id}") + print_warning("Store this recovery key securely!") + print_info(f"Recovery key (RMK): {keymanager.get_rmk().hex()}") + print_warning("This key can unlock the repository on any device") print_info("Add a provider with: stash provider add discord") diff --git a/src/stash/cli/commands/key.py b/src/stash/cli/commands/key.py new file mode 100644 index 0000000..89b0f42 --- /dev/null +++ b/src/stash/cli/commands/key.py @@ -0,0 +1,106 @@ +"""CLI commands: key management - lock, unlock, status.""" + +import click + +from stash.cli.output import print_error, print_info, print_success, print_warning +from stash.core.keymanager import KeyManager + + +@click.group() +def key_commands() -> None: + """Repository key management.""" + pass + + +@key_commands.command("lock") +@click.pass_context +def lock_cmd(ctx: click.Context) -> None: + """Lock the repository by removing RMK from keyring.""" + repo = ctx.obj["repo"].resolve() + keymanager = KeyManager(repo) + + if not keymanager.get_repository_identity(): + print_info("Repository not initialized") + return + + keymanager.lock_repository() + print_success("Repository locked (RMK removed from keyring)") + print_info("Run 'stash unlock' to restore access") + + +@key_commands.command("unlock") +@click.option("--recovery-key", help="Recovery key (hex) to restore RMK") +@click.pass_context +def unlock_cmd(ctx: click.Context, recovery_key: str | None) -> None: + """Unlock the repository on a new device using a recovery key.""" + repo = ctx.obj["repo"].resolve() + keymanager = KeyManager(repo) + + if keymanager.has_repository_identity(): + print_info("Repository already unlocked") + return + + if not recovery_key: + print_error("Recovery key required. Provide --recovery-key ") + print_info("The recovery key is the RMK hex that was generated during 'stash init'") + return + + try: + rmk_bytes = bytes.fromhex(recovery_key) + except ValueError: + print_error("Invalid recovery key format (must be hex)") + return + + if len(rmk_bytes) != 32: + print_error("Recovery key must be 32 bytes (64 hex characters)") + return + + try: + identity = keymanager.unlock_repository(rmk_bytes) + print_success("Repository unlocked successfully") + print_info(f"Repository ID: {identity.repository_id}") + except Exception as e: + print_error(f"Failed to unlock repository: {e}") + + +@key_commands.command("status") +@click.pass_context +def key_status_cmd(ctx: click.Context) -> None: + """Show key management status.""" + repo = ctx.obj["repo"].resolve() + keymanager = KeyManager(repo) + + identity = keymanager.get_repository_identity() + if identity is None: + print_warning("Repository not initialized or locked") + return + + try: + keymanager.get_rmk() + print_success("Repository unlocked") + print_info(f"Repository ID: {identity.repository_id}") + print_info(f"Created: {identity.created_at:.0f}") + except Exception as e: + print_warning(f"Repository locked or key unavailable: {e}") + + +@key_commands.command("recovery") +@click.pass_context +def recovery_cmd(ctx: click.Context) -> None: + """Show the recovery key (RMK) for backup purposes.""" + repo = ctx.obj["repo"].resolve() + keymanager = KeyManager(repo) + + identity = keymanager.get_repository_identity() + if identity is None: + print_error("Repository not initialized") + return + + try: + rmk = keymanager.get_rmk() + print_warning("Store this recovery key securely!") + print_info(f"Recovery key (RMK): {rmk.hex()}") + print_warning("This key can unlock the repository on any device") + print_warning("Anyone with this key can access all files in this repository") + except Exception as e: + print_error(f"Failed to retrieve RMK: {e}") \ No newline at end of file diff --git a/src/stash/cli/commands/put.py b/src/stash/cli/commands/put.py index 2c652e3..48d738a 100644 --- a/src/stash/cli/commands/put.py +++ b/src/stash/cli/commands/put.py @@ -10,6 +10,7 @@ from stash.core.chunking import ChunkConfig, Chunker from stash.core.crypto import CryptoEngine from stash.core.jobs import JobConfig +from stash.core.keymanager import KeyManager from stash.core.manifest import ( DistributionStrategy, EncryptionInfo, @@ -26,12 +27,11 @@ @click.option("--provider", help="Specific provider to use (default: first available)") @click.option("--chunk-size", type=int, help="Chunk size in bytes (default: provider limit)") @click.option("--strategy", type=click.Choice(["single", "split", "balanced", "replicated"]), default="single", help="Distribution strategy") -@click.option("--password", prompt=True, hide_input=True, help="Encryption password") @click.option("--confirm/--no-confirm", default=True, help="Confirm before upload") @click.pass_context -def put_cmd(ctx: click.Context, file_path: Path, provider: str | None, chunk_size: int | None, strategy: str, password: str, confirm: bool) -> None: +def put_cmd(ctx: click.Context, file_path: Path, provider: str | None, chunk_size: int | None, strategy: str, confirm: bool) -> None: """Store a file in Stash.""" - asyncio.run(_put_async(file_path, ctx.obj["repo"], provider, chunk_size, strategy, password, confirm)) + asyncio.run(_put_async(file_path, ctx.obj["repo"], provider, chunk_size, strategy, confirm)) async def _put_async( @@ -40,11 +40,11 @@ async def _put_async( provider_name: str | None, chunk_size: int | None, strategy: str, - password: str, do_confirm: bool, ) -> None: repo = repo_path.resolve() store = MetadataStore(repo) + keymanager = KeyManager(repo) providers = store.list_providers() if not providers: @@ -72,14 +72,23 @@ async def _put_async( print_info("Cancelled") return + # Get RMK from keyring + try: + rmk = keymanager.get_rmk() + except Exception as e: + print_error(f"Failed to retrieve RMK: {e}") + print_info("Run 'stash unlock' if this is a new device") + return + crypto = CryptoEngine() - file_key = crypto.generate_file_key() - wrapped_key = crypto.encrypt_file_key(file_key, password) + file_id = generate_file_id() + file_key = crypto.generate_file_key(rmk) - # Encrypt the filename - encrypted_name_chunk = crypto.encrypt_chunk(file_path.name.encode(), file_key, -1) - encrypted_name = encrypted_name_chunk.ciphertext.hex() - encrypted_name_nonce = encrypted_name_chunk.nonce + # Encrypt the filename using the file key + encrypted_name_ciphertext, encrypted_name_nonce = crypto.encrypt_filename( + file_path.name.encode(), file_key + ) + encrypted_name = encrypted_name_ciphertext.hex() provider_configs = {} for name in provider_names: @@ -109,21 +118,21 @@ async def _put_async( nonce_size=12, chunk_key_derivation="HKDF-SHA256", file_key_salt=file_key.salt, - file_key_wrapped=wrapped_key, + file_key_wrapped=None, ) builder = ManifestBuilder( - file_id=generate_file_id(), + file_id=file_id, original_name=file_path.name, encrypted_name=encrypted_name, encrypted_name_nonce=encrypted_name_nonce, - original_size=file_size, + original_size=file_path.stat().st_size, chunk_size=effective_chunk_size, encryption=encryption_info, strategy=dist_strategy, ) - print_info(f"Processing {num_chunks} chunks ({format_size(effective_chunk_size)} each)...") + print_info(f"Processing {num_chunks} chunks...") JobConfig(max_workers=min(4, num_chunks)) @@ -132,45 +141,48 @@ async def _put_async( progress = create_progress() task = progress.add_task("Uploading", total=num_chunks) - from stash.core.chunking import Chunk - - async def upload_chunk(chunk: Chunk, provider_name: str) -> None: + async def upload_chunk(chunk_data: bytes, chunk_index: int, provider_name: str) -> tuple[bytes, dict[str, str]]: + """Encrypt and upload a single chunk.""" async with semaphores[provider_name]: - # Use opaque identifier: file_id + chunk index (no filename) - remote_path = f"{builder.file_id}/chunk-{chunk.index:06d}" - remote_ref = await provider_instances[provider_name].upload_chunk(chunk, remote_path) - checksum = compute_checksum(chunk.data) - builder.add_chunk( - index=chunk.index, - size=chunk.size, - encrypted_size=len(remote_ref.metadata.get("size", "0")), - checksum=checksum, - provider=provider_name, - remote_id=remote_ref.remote_id, - nonce=encrypted.nonce, - metadata=remote_ref.metadata, - ) - progress.advance(task) - - with progress: - for chunk in chunker.chunk_file(file_path): - encrypted = crypto.encrypt_chunk(chunk.data, file_key, chunk.index) - encrypted_chunk = type(chunk)( - index=chunk.index, + encrypted = crypto.encrypt_chunk(chunk_data, file_key, chunk_index) + remote_path = f"{file_id}/chunk-{chunk_index:06d}" + from stash.core.chunking import Chunk + encrypted_chunk = Chunk( + index=chunk_index, data=encrypted.ciphertext, - offset=chunk.offset, + offset=0, size=len(encrypted.ciphertext), - is_last=chunk.is_last, + is_last=False, ) + remote_ref = await provider_instances[provider_name].upload_chunk(encrypted_chunk, remote_path) + return encrypted.nonce, remote_ref.metadata - if dist_strategy == DistributionStrategy.SINGLE: - target = provider_names[0] - elif dist_strategy == DistributionStrategy.SPLIT: - target = provider_names[chunk.index % len(provider_names)] - else: - target = provider_names[0] + progress = create_progress() + task = progress.add_task("Uploading", total=num_chunks) - await upload_chunk(encrypted_chunk, target) + for chunk in chunker.chunk_file(file_path): + checksum = compute_checksum(chunk.data) + + if dist_strategy == DistributionStrategy.SINGLE: + target = provider_names[0] + elif dist_strategy == DistributionStrategy.SPLIT: + target = provider_names[chunk.index % len(provider_names)] + else: + target = provider_names[0] + + nonce, metadata = await upload_chunk(chunk.data, chunk.index, target) + + builder.add_chunk( + index=chunk.index, + size=chunk.size, + encrypted_size=len(metadata.get("size", "0")), + checksum=checksum, + provider=target, + remote_id=metadata.get("remote_id", ""), + nonce=nonce, + metadata=metadata, + ) + progress.advance(task) manifest = builder.build() store.save_manifest(manifest) diff --git a/src/stash/cli/main.py b/src/stash/cli/main.py index e136488..59be899 100644 --- a/src/stash/cli/main.py +++ b/src/stash/cli/main.py @@ -8,6 +8,7 @@ from stash.cli.commands.get import get_commands from stash.cli.commands.info import info_commands from stash.cli.commands.init import init_commands +from stash.cli.commands.key import key_commands from stash.cli.commands.ls import ls_commands from stash.cli.commands.provider import provider_commands from stash.cli.commands.put import put_commands @@ -37,6 +38,7 @@ def main(ctx: click.Context, repo: Path | None, verbose: bool) -> None: main.add_command(rm_commands) main.add_command(verify_commands) main.add_command(status_commands) +main.add_command(key_commands) if __name__ == "__main__": diff --git a/src/stash/core/crypto.py b/src/stash/core/crypto.py index 1653087..e6f0494 100644 --- a/src/stash/core/crypto.py +++ b/src/stash/core/crypto.py @@ -47,11 +47,26 @@ class CryptoEngine: def __init__(self, config: EncryptionConfig | None = None): self.config = config or EncryptionConfig() - def generate_file_key(self) -> FileKey: - """Generate a new per-file encryption key.""" + def generate_file_key(self, rmk: bytes) -> FileKey: + """Generate a new per-file encryption key derived from RMK.""" salt = secrets.token_bytes(SALT_SIZE) - master_key = secrets.token_bytes(self.config.key_size) - return FileKey(key=master_key, salt=salt, config=self.config) + file_key = self._derive_file_key(rmk, salt) + return FileKey(key=file_key, salt=salt, config=self.config) + + def _derive_file_key(self, rmk: bytes, file_id: bytes) -> bytes: + """Derive a file encryption key from RMK and file ID.""" + hkdf = HKDF( + algorithm=hashes.SHA256(), + length=self.config.key_size, + salt=file_id, + info=b"stash-file-key", + ) + return hkdf.derive(rmk) + + def derive_file_key_from_rmk(self, rmk: bytes, file_id: bytes) -> FileKey: + """Derive a FileKey from RMK and file ID.""" + file_key = self._derive_file_key(rmk, file_id) + return FileKey(key=file_key, salt=file_id, config=self.config) def derive_chunk_key(self, file_key: FileKey, chunk_index: int) -> bytes: """Derive a per-chunk key from the file key.""" @@ -83,39 +98,31 @@ def decrypt_chunk(self, encrypted: EncryptedChunk, file_key: FileKey) -> bytes: except Exception as e: raise CryptoError(f"Decryption failed for chunk {encrypted.chunk_index}: {e}") from e - def encrypt_file_key(self, file_key: FileKey, password: str) -> bytes: - """Encrypt a file key with a password (for key wrapping).""" - salt = secrets.token_bytes(SALT_SIZE) + def encrypt_filename(self, filename: bytes, file_key: FileKey) -> tuple[bytes, bytes]: + """Encrypt a filename using the file key.""" hkdf = HKDF( algorithm=hashes.SHA256(), length=self.config.key_size, - salt=salt, - info=b"stash-key-wrap", + salt=file_key.salt, + info=b"stash-filename", ) - wrapping_key = hkdf.derive(password.encode()) - aesgcm = AESGCM(wrapping_key) + filename_key = hkdf.derive(file_key.key) nonce = secrets.token_bytes(self.config.nonce_size) - ciphertext = aesgcm.encrypt(nonce, file_key.key, None) - return salt + nonce + ciphertext - - def decrypt_file_key(self, wrapped: bytes, password: str, config: EncryptionConfig) -> FileKey: - """Decrypt a file key with a password.""" - # wrapped format: salt (16) + nonce (12) + ciphertext - if len(wrapped) < SALT_SIZE + config.nonce_size + TAG_SIZE: - raise CryptoError("Invalid wrapped key format") - salt = wrapped[:SALT_SIZE] - nonce = wrapped[SALT_SIZE:SALT_SIZE + config.nonce_size] - ciphertext = wrapped[SALT_SIZE + config.nonce_size:] + aesgcm = AESGCM(filename_key) + ciphertext = aesgcm.encrypt(nonce, filename, None) + return ciphertext, nonce + + def decrypt_filename(self, ciphertext: bytes, nonce: bytes, file_key: FileKey) -> bytes: + """Decrypt a filename using the file key.""" hkdf = HKDF( algorithm=hashes.SHA256(), - length=config.key_size, - salt=salt, - info=b"stash-key-wrap", + length=self.config.key_size, + salt=file_key.salt, + info=b"stash-filename", ) - wrapping_key = hkdf.derive(password.encode()) - aesgcm = AESGCM(wrapping_key) + filename_key = hkdf.derive(file_key.key) + aesgcm = AESGCM(filename_key) try: - key = aesgcm.decrypt(nonce, ciphertext, None) - return FileKey(key=key, salt=salt, config=config) + return aesgcm.decrypt(nonce, ciphertext, None) except Exception as e: - raise CryptoError(f"Key unwrapping failed: {e}") from e \ No newline at end of file + raise CryptoError(f"Filename decryption failed: {e}") from e \ No newline at end of file diff --git a/src/stash/core/exceptions.py b/src/stash/core/exceptions.py index c38d6d1..1aca08d 100644 --- a/src/stash/core/exceptions.py +++ b/src/stash/core/exceptions.py @@ -63,4 +63,9 @@ class ConfigurationError(StashError): class ValidationError(StashError): """Input validation errors.""" + pass + + +class KeyManagementError(StashError): + """Key management errors (RMK, keyring, etc.).""" pass \ No newline at end of file diff --git a/src/stash/core/keymanager.py b/src/stash/core/keymanager.py new file mode 100644 index 0000000..a94ca14 --- /dev/null +++ b/src/stash/core/keymanager.py @@ -0,0 +1,202 @@ +"""Repository Master Key management using OS keyring.""" + +import contextlib +import json +import secrets +import time +from dataclasses import dataclass +from pathlib import Path + +import keyring +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.kdf.hkdf import HKDF + +from stash.core.exceptions import KeyManagementError + +SERVICE_NAME = "stash" +RMK_KEY_SIZE = 32 + + +@dataclass(frozen=True, slots=True) +class RepositoryIdentity: + """Non-secret repository identity information.""" + repository_id: str + created_at: float + version: int = 1 + + +class KeyManager: + """Manages the Repository Master Key (RMK) using OS keyring.""" + + def __init__(self, repo_path: Path): + self.repo_path = repo_path + self.metadata_dir = repo_path / ".stash" + self.identity_file = self.metadata_dir / "identity.json" + + def _get_keyring_username(self) -> str: + """Get a unique username for this repository in keyring.""" + repo_id = self._get_repo_id() + return f"stash:{repo_id}" + + def _get_repo_id(self) -> str: + """Get or generate a unique repository ID.""" + if self.identity_file.exists(): + try: + with self.identity_file.open("r") as f: + data: dict[str, str] = json.load(f) + return data.get("repository_id", "") + except (json.JSONDecodeError, OSError): + pass + return "" + + def has_repository_identity(self) -> bool: + """Check if this repository has an identity (and thus RMK).""" + return self.identity_file.exists() + + def get_repository_identity(self) -> RepositoryIdentity | None: + """Get the repository identity if it exists.""" + if not self.identity_file.exists(): + return None + try: + with self.identity_file.open("r") as f: + data = json.load(f) + return RepositoryIdentity( + repository_id=data["repository_id"], + created_at=data["created_at"], + version=data.get("version", 1), + ) + except (json.JSONDecodeError, OSError, KeyError): + return None + + def initialize_repository(self) -> RepositoryIdentity: + """Initialize a new repository with a new RMK.""" + if self.has_repository_identity(): + raise KeyManagementError("Repository already initialized") + + repo_id = secrets.token_hex(16) + + rmk = secrets.token_bytes(32) + username = f"stash:{repo_id}" + + try: + keyring.set_password(SERVICE_NAME, username, rmk.hex()) + except Exception as e: + raise KeyManagementError(f"Failed to store RMK in keyring: {e}") from e + + identity = RepositoryIdentity( + repository_id=repo_id, + created_at=time.time(), + version=1, + ) + + self._save_identity(identity) + return identity + + def _save_identity(self, identity: RepositoryIdentity) -> None: + """Save repository identity to disk.""" + self.metadata_dir.mkdir(parents=True, exist_ok=True) + data = { + "repository_id": identity.repository_id, + "created_at": identity.created_at, + "version": identity.version, + } + with self.identity_file.open("w") as f: + json.dump(data, f) + + def get_rmk(self) -> bytes: + """Retrieve the RMK from keyring.""" + identity = self.get_repository_identity() + if identity is None: + raise KeyManagementError( + "Repository not initialized. Run 'stash init' first." + ) + + username = f"stash:{identity.repository_id}" + rmk_hex = keyring.get_password(SERVICE_NAME, username) + + if rmk_hex is None: + raise KeyManagementError( + f"RMK not found in keyring for repository {identity.repository_id}. " + "Run 'stash unlock' to recover." + ) + + try: + return bytes.fromhex(rmk_hex) + except ValueError as e: + raise KeyManagementError(f"Invalid RMK format in keyring: {e}") from e + + def derive_file_key(self, rmk: bytes, file_id: bytes) -> bytes: + """Derive a file encryption key from RMK and file ID.""" + hkdf = HKDF( + algorithm=hashes.SHA256(), + length=32, + salt=file_id, + info=b"stash-file-key", + ) + return hkdf.derive(rmk) + + def derive_chunk_key(self, file_key: bytes, chunk_index: int) -> bytes: + """Derive a chunk encryption key from file key and chunk index.""" + hkdf = HKDF( + algorithm=hashes.SHA256(), + length=32, + salt=b"", + info=f"stash-chunk-{chunk_index}".encode(), + ) + return hkdf.derive(file_key) + + def lock_repository(self) -> None: + """Remove RMK from keyring (lock the repository).""" + identity = self.get_repository_identity() + if identity is None: + return + + username = f"stash:{identity.repository_id}" + with contextlib.suppress(keyring.errors.PasswordDeleteError): + keyring.delete_password(SERVICE_NAME, username) + + def unlock_repository(self, recovery_key: bytes) -> RepositoryIdentity: + """Unlock repository using a recovery key.""" + if self.has_repository_identity(): + raise KeyManagementError("Repository already initialized") + + repo_id = secrets.token_hex(16) + + username = f"stash:{repo_id}" + + try: + keyring.set_password(SERVICE_NAME, username, recovery_key.hex()) + except Exception as e: + raise KeyManagementError(f"Failed to store RMK in keyring: {e}") from e + + identity = RepositoryIdentity( + repository_id=repo_id, + created_at=time.time(), + version=1, + ) + self._save_identity(identity) + return identity + + def change_repository_id(self, new_repo_id: str) -> None: + """Change the repository ID (and thus the keyring entry).""" + old_identity = self.get_repository_identity() + if old_identity is None: + raise KeyManagementError("Repository not initialized") + + old_username = f"stash:{old_identity.repository_id}" + rmk_hex = keyring.get_password(SERVICE_NAME, old_username) + if rmk_hex is None: + raise KeyManagementError("RMK not found in keyring") + + new_username = f"stash:{new_repo_id}" + try: + keyring.set_password(SERVICE_NAME, new_username, rmk_hex) + keyring.delete_password(SERVICE_NAME, old_username) + except Exception as e: + raise KeyManagementError(f"Failed to change repository ID: {e}") from e + + self._save_identity(RepositoryIdentity( + repository_id=new_repo_id, + created_at=old_identity.created_at, + version=old_identity.version, + )) \ No newline at end of file diff --git a/src/stash/core/manifest.py b/src/stash/core/manifest.py index 7f21e88..78fa92e 100644 --- a/src/stash/core/manifest.py +++ b/src/stash/core/manifest.py @@ -39,6 +39,8 @@ class EncryptionInfo: nonce_size: int chunk_key_derivation: str file_key_salt: bytes + # file_key_wrapped is no longer used with RMK-based system + # kept for backwards compatibility file_key_wrapped: bytes | None = None diff --git a/tests/unit/core/test_crypto.py b/tests/unit/core/test_crypto.py index 423d2e5..e55e945 100644 --- a/tests/unit/core/test_crypto.py +++ b/tests/unit/core/test_crypto.py @@ -1,7 +1,7 @@ """Tests for crypto module.""" import pytest -from stash.core.crypto import CryptoEngine, EncryptionConfig +from stash.core.crypto import CryptoEngine, EncryptionConfig, FileKey def test_crypto_engine_initialization(): @@ -12,9 +12,10 @@ def test_crypto_engine_initialization(): def test_generate_file_key(): - """Test file key generation.""" + """Test file key generation from RMK.""" engine = CryptoEngine() - file_key = engine.generate_file_key() + rmk = b"0" * 32 # dummy RMK for testing + file_key = engine.generate_file_key(rmk) assert file_key.key is not None assert len(file_key.key) == 32 assert file_key.salt is not None @@ -24,7 +25,8 @@ def test_generate_file_key(): def test_derive_chunk_key(): """Test chunk key derivation.""" engine = CryptoEngine() - file_key = engine.generate_file_key() + rmk = b"0" * 32 + file_key = engine.generate_file_key(rmk) chunk_key_0 = engine.derive_chunk_key(file_key, 0) chunk_key_1 = engine.derive_chunk_key(file_key, 1) assert chunk_key_0 != chunk_key_1 @@ -34,14 +36,15 @@ def test_derive_chunk_key(): def test_encrypt_decrypt_chunk(): """Test chunk encryption and decryption round-trip.""" engine = CryptoEngine() - file_key = engine.generate_file_key() + rmk = b"0" * 32 + file_key = engine.generate_file_key(rmk) data = b"test data for encryption" - + encrypted = engine.encrypt_chunk(data, file_key, 0) assert encrypted.ciphertext != data assert encrypted.nonce is not None assert encrypted.chunk_index == 0 - + decrypted = engine.decrypt_chunk(encrypted, file_key) assert decrypted == data @@ -49,7 +52,8 @@ def test_encrypt_decrypt_chunk(): def test_encrypt_empty_chunk_raises(): """Test encrypting empty chunk raises error.""" engine = CryptoEngine() - file_key = engine.generate_file_key() + rmk = b"0" * 32 + file_key = engine.generate_file_key(rmk) with pytest.raises(Exception): engine.encrypt_chunk(b"", file_key, 0) @@ -57,37 +61,62 @@ def test_encrypt_empty_chunk_raises(): def test_decrypt_wrong_key_fails(): """Test decrypting with wrong key fails.""" engine = CryptoEngine() - file_key1 = engine.generate_file_key() - file_key2 = engine.generate_file_key() + rmk = b"0" * 32 + file_key1 = engine.generate_file_key(rmk) + file_key2 = engine.generate_file_key(rmk) data = b"test data" - + encrypted = engine.encrypt_chunk(data, file_key1, 0) with pytest.raises(Exception): engine.decrypt_chunk(encrypted, file_key2) -def test_encrypt_file_key(): - """Test file key encryption.""" +def test_filename_encryption(): + """Test filename encryption and decryption.""" engine = CryptoEngine() - file_key = engine.generate_file_key() - password = "test_password" - - wrapped = engine.encrypt_file_key(file_key, password) - assert wrapped is not None - assert len(wrapped) > 0 + rmk = b"0" * 32 + file_key = engine.generate_file_key(rmk) + filename = b"test_file.txt" + + ciphertext, nonce = engine.encrypt_filename(filename, file_key) + assert ciphertext is not None + assert nonce is not None + assert len(nonce) == 12 + + decrypted = engine.decrypt_filename(ciphertext, nonce, file_key) + assert decrypted == filename -def test_decrypt_file_key(): - """Test file key decryption.""" +def test_derive_file_key_from_rmk(): + """Test deriving file key directly from RMK and file_id.""" engine = CryptoEngine() - file_key = engine.generate_file_key() - password = "test_password" - - wrapped = engine.encrypt_file_key(file_key, password) - config = EncryptionConfig() - - decrypted = engine.decrypt_file_key(wrapped, password, config) - assert decrypted.key == file_key.key - # The salt in the decrypted FileKey is the wrapping salt, not the original file key salt - # The important thing is that the key decrypts correctly - assert decrypted.key == file_key.key \ No newline at end of file + rmk = b"0" * 32 + file_id = b"test-file-id-123" + + file_key = engine.derive_file_key_from_rmk(rmk, file_id) + assert isinstance(file_key, FileKey) + assert file_key.key is not None + assert len(file_key.key) == 32 + assert file_key.salt == file_id + + +def test_deterministic_file_key(): + """Test that same RMK + file_id always produces same file key.""" + engine = CryptoEngine() + rmk = b"0" * 32 + file_id = b"test-file-id-456" + + file_key1 = engine.derive_file_key_from_rmk(rmk, file_id) + file_key2 = engine.derive_file_key_from_rmk(rmk, file_id) + assert file_key1.key == file_key2.key + assert file_key1.salt == file_key2.salt + + +def test_different_rmk_different_key(): + """Test that different RMKs produce different file keys.""" + engine = CryptoEngine() + file_id = b"test-file-id-789" + + file_key1 = engine.derive_file_key_from_rmk(b"0" * 32, file_id) + file_key2 = engine.derive_file_key_from_rmk(b"1" * 32, file_id) + assert file_key1.key != file_key2.key \ No newline at end of file diff --git a/tests/unit/core/test_manifest.py b/tests/unit/core/test_manifest.py index 4ed549c..aba56f0 100644 --- a/tests/unit/core/test_manifest.py +++ b/tests/unit/core/test_manifest.py @@ -10,6 +10,7 @@ compute_checksum, generate_file_id, ) +from stash.core.crypto import CryptoEngine def test_compute_checksum(): @@ -55,10 +56,9 @@ def test_chunk_info(): def test_manifest_builder(): """Test ManifestBuilder.""" - from stash.core.crypto import CryptoEngine, FileKey - - engine = CryptoEngine() - file_key = engine.generate_file_key() + crypto = CryptoEngine() + rmk = b"0" * 32 + file_key = crypto.generate_file_key(rmk) enc = EncryptionInfo( algorithm="AES-256-GCM", key_size=32, @@ -66,27 +66,26 @@ def test_manifest_builder(): chunk_key_derivation="HKDF-SHA256", file_key_salt=file_key.salt, ) - - # Encrypt filename - crypto = CryptoEngine() - encrypted_name_chunk = crypto.encrypt_chunk(b"test.txt", file_key, -1) - encrypted_name = encrypted_name_chunk.ciphertext.hex() - encrypted_name_nonce = encrypted_name_chunk.nonce - + + # Encrypt filename using file key + ciphertext, nonce = crypto.encrypt_filename(b"test.txt", file_key) + encrypted_name = ciphertext.hex() + encrypted_name_nonce = nonce + builder = ManifestBuilder( file_id="test123", original_name="test.txt", encrypted_name=encrypted_name, - encrypted_name_nonce=encrypted_name_nonce, + encrypted_name_nonce=nonce, original_size=1000, chunk_size=1024, encryption=enc, strategy=DistributionStrategy.SINGLE, ) - + builder.add_chunk(0, 100, 120, "checksum", "discord", "msg1", b"nonce") manifest = builder.build() - + assert manifest.file_id == "test123" assert manifest.original_name == "test.txt" assert manifest.chunk_count == 1 @@ -95,11 +94,10 @@ def test_manifest_builder(): def test_manifest_serialization(): """Test manifest JSON serialization round-trip.""" - from stash.core.crypto import CryptoEngine, FileKey - - engine = CryptoEngine() - file_key = engine.generate_file_key() - + crypto = CryptoEngine() + rmk = b"0" * 32 + file_key = crypto.generate_file_key(rmk) + enc = EncryptionInfo( algorithm="AES-256-GCM", key_size=32, @@ -107,32 +105,31 @@ def test_manifest_serialization(): chunk_key_derivation="HKDF-SHA256", file_key_salt=file_key.salt, ) - - # Encrypt filename - crypto = CryptoEngine() - encrypted_name_chunk = crypto.encrypt_chunk(b"test.txt", file_key, -1) - encrypted_name = encrypted_name_chunk.ciphertext.hex() - encrypted_name_nonce = encrypted_name_chunk.nonce - + + # Encrypt filename using file key + ciphertext, nonce = crypto.encrypt_filename(b"test.txt", file_key) + encrypted_name = ciphertext.hex() + encrypted_name_nonce = nonce + builder = ManifestBuilder( file_id="test123", original_name="test.txt", encrypted_name=encrypted_name, - encrypted_name_nonce=encrypted_name_nonce, + encrypted_name_nonce=nonce, original_size=1000, chunk_size=1024, encryption=enc, strategy=DistributionStrategy.SINGLE, ) - + builder.add_chunk(0, 100, 120, "checksum", "discord", "msg1", b"nonce") manifest = builder.build() - + # Serialize json_str = manifest.to_json() assert "test123" in json_str assert "discord" in json_str - + # Deserialize manifest2 = FileManifest.from_json(json_str) assert manifest2.file_id == manifest.file_id @@ -144,4 +141,51 @@ def test_manifest_serialization(): def test_distribution_strategies(): """Test all distribution strategies.""" for strategy in DistributionStrategy: - assert strategy.value in ["single", "split", "balanced", "replicated"] \ No newline at end of file + assert strategy.value in ["single", "split", "balanced", "replicated"] + + +def test_filename_encryption_in_manifest(): + """Test that filename encryption works correctly in manifest round-trip.""" + crypto = CryptoEngine() + rmk = b"0" * 32 + file_key = crypto.generate_file_key(rmk) + + enc = EncryptionInfo( + algorithm="AES-256-GCM", + key_size=32, + nonce_size=12, + chunk_key_derivation="HKDF-SHA256", + file_key_salt=file_key.salt, + ) + + original_name = "my_test_file.txt" + ciphertext, nonce = crypto.encrypt_filename(original_name.encode(), file_key) + encrypted_name = ciphertext.hex() + encrypted_name_nonce = nonce + + builder = ManifestBuilder( + file_id="test456", + original_name=original_name, + encrypted_name=encrypted_name, + encrypted_name_nonce=nonce, + original_size=2000, + chunk_size=1024, + encryption=enc, + strategy=DistributionStrategy.SINGLE, + ) + + builder.add_chunk(0, 500, 520, "checksum1", "discord", "msg1", b"nonce1") + builder.add_chunk(1, 500, 520, "checksum2", "telegram", "msg2", b"nonce2") + manifest = builder.build() + + # Serialize and deserialize + json_str = manifest.to_json() + manifest2 = FileManifest.from_json(json_str) + + # Decrypt filename using decrypt_filename method + decrypted = crypto.decrypt_filename( + bytes.fromhex(manifest2.encrypted_name), + manifest2.encrypted_name_nonce, + file_key + ) + assert decrypted.decode() == original_name \ No newline at end of file diff --git a/tests/unit/core/test_metadata.py b/tests/unit/core/test_metadata.py index 9936bb2..c60613a 100644 --- a/tests/unit/core/test_metadata.py +++ b/tests/unit/core/test_metadata.py @@ -12,17 +12,18 @@ DistributionStrategy, generate_file_id, ) -from stash.core.crypto import CryptoEngine, FileKey +from stash.core.crypto import CryptoEngine def _make_builder(file_id: str, original_name: str, size: int = 1000): - """Create a ManifestBuilder with encrypted filename.""" - engine = CryptoEngine() - file_key = engine.generate_file_key() + """Create a ManifestBuilder with encrypted filename using RMK.""" crypto = CryptoEngine() - encrypted_name_chunk = crypto.encrypt_chunk(original_name.encode(), file_key, -1) - encrypted_name = encrypted_name_chunk.ciphertext.hex() - encrypted_name_nonce = encrypted_name_chunk.nonce + rmk = b"0" * 32 # dummy RMK + file_key = crypto.generate_file_key(rmk) + + ciphertext, nonce = crypto.encrypt_filename(original_name.encode(), file_key) + encrypted_name = ciphertext.hex() + encrypted_name_nonce = nonce enc = EncryptionInfo( algorithm="AES-256-GCM", @@ -36,7 +37,7 @@ def _make_builder(file_id: str, original_name: str, size: int = 1000): file_id=file_id, original_name=original_name, encrypted_name=encrypted_name, - encrypted_name_nonce=encrypted_name_nonce, + encrypted_name_nonce=nonce, original_size=size, chunk_size=1024, encryption=enc, From b62f1309a323c2e82e305653e3f14e3f7d5df6b0 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 17:50:45 +0530 Subject: [PATCH 13/19] update version string --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5858f55..9e52e80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "stashify" -version = "0.1.0" +version = "0.2.0" description = "Privacy-focused CLI storage system using third-party platforms as encrypted backends" readme = "README.md" license = {text = "MIT"} From 309ac8f741acfeb438b9c74d36a1d60595bfdd75 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 17:55:36 +0530 Subject: [PATCH 14/19] fix: fix unlock flow for locked repositories --- src/stash/cli/commands/key.py | 16 +++++++++++++--- src/stash/core/keymanager.py | 21 ++++++++++----------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/stash/cli/commands/key.py b/src/stash/cli/commands/key.py index 89b0f42..c9fd117 100644 --- a/src/stash/cli/commands/key.py +++ b/src/stash/cli/commands/key.py @@ -3,6 +3,7 @@ import click from stash.cli.output import print_error, print_info, print_success, print_warning +from stash.core.exceptions import KeyManagementError from stash.core.keymanager import KeyManager @@ -36,8 +37,9 @@ def unlock_cmd(ctx: click.Context, recovery_key: str | None) -> None: repo = ctx.obj["repo"].resolve() keymanager = KeyManager(repo) - if keymanager.has_repository_identity(): - print_info("Repository already unlocked") + identity = keymanager.get_repository_identity() + if identity is None: + print_error("Repository not initialized. Run 'stash init' first.") return if not recovery_key: @@ -55,8 +57,16 @@ def unlock_cmd(ctx: click.Context, recovery_key: str | None) -> None: print_error("Recovery key must be 32 bytes (64 hex characters)") return + # Check if already unlocked + try: + keymanager.get_rmk() + print_info("Repository already unlocked") + return + except KeyManagementError: + pass # Not unlocked, proceed with unlock + try: - identity = keymanager.unlock_repository(rmk_bytes) + keymanager.unlock_repository(rmk_bytes) print_success("Repository unlocked successfully") print_info(f"Repository ID: {identity.repository_id}") except Exception as e: diff --git a/src/stash/core/keymanager.py b/src/stash/core/keymanager.py index a94ca14..6fee788 100644 --- a/src/stash/core/keymanager.py +++ b/src/stash/core/keymanager.py @@ -157,24 +157,23 @@ def lock_repository(self) -> None: def unlock_repository(self, recovery_key: bytes) -> RepositoryIdentity: """Unlock repository using a recovery key.""" - if self.has_repository_identity(): - raise KeyManagementError("Repository already initialized") - - repo_id = secrets.token_hex(16) + identity = self.get_repository_identity() + if identity is None: + raise KeyManagementError("Repository not initialized. Run 'stash init' first.") - username = f"stash:{repo_id}" + # Check if RMK is already available in keyring + username = f"stash:{identity.repository_id}" + existing = keyring.get_password(SERVICE_NAME, username) + if existing is not None: + raise KeyManagementError("Repository already unlocked") + # Store recovery key as RMK + username = f"stash:{identity.repository_id}" try: keyring.set_password(SERVICE_NAME, username, recovery_key.hex()) except Exception as e: raise KeyManagementError(f"Failed to store RMK in keyring: {e}") from e - identity = RepositoryIdentity( - repository_id=repo_id, - created_at=time.time(), - version=1, - ) - self._save_identity(identity) return identity def change_repository_id(self, new_repo_id: str) -> None: From 90af30b5dfd16e797aba75dadf861bcd6bfcf5fb Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 18:12:37 +0530 Subject: [PATCH 15/19] docs: update README, docs, requirements.txt for RMK-based key management --- README.md | 472 ++++++++++++++++++++++++---------------- docs/architecture.md | 57 +++-- docs/cli-reference.md | 62 +++++- docs/configuration.md | 36 ++- docs/index.md | 14 +- docs/installation.md | 41 +++- docs/security.md | 85 +++++--- docs/troubleshooting.md | 39 ++-- requirements.txt | 11 + 9 files changed, 545 insertions(+), 272 deletions(-) create mode 100644 requirements.txt diff --git a/README.md b/README.md index 19d445a..82e8877 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # Stashify +[![IMG-20260820-WA0006.jpg](https://i.postimg.cc/15hbDPcL/IMG-20260820-WA0006.jpg)](https://postimg.cc/7b9ByFRV) + +

Encrypted storage. Your providers. Your keys.

@@ -32,23 +35,23 @@ Stashify lets you use services such as **Telegram and Discord as storage backend Files are encrypted locally, split into chunks, and uploaded as ciphertext. Chunks can be stored on one provider or distributed across multiple providers depending on your configuration. ```text - STASHIFY - │ - Storage Engine - │ - ┌─────────────┴─────────────┐ - │ │ - Encryption Chunking - │ │ - └─────────────┬─────────────┘ - │ - Storage Router - │ - ┌─────────────┴─────────────┐ - │ │ - Telegram Discord - │ │ - encrypted chunks encrypted chunks + STASHIFY + │ + Storage Engine + │ + ┌─────────────┴─────────────┐ + │ │ + Encryption Chunking + │ │ + └─────────────┬─────────────┘ + │ + Storage Router + │ + ┌─────────────┴─────────────┐ + │ │ + Telegram Discord + │ │ + encrypted chunks encrypted chunks ``` Stashify does **not** provide the underlying storage. @@ -83,22 +86,22 @@ Stashify separates those concepts. Your files are encrypted **before they leave your device**. ```text - YOUR DEVICE - │ - Plaintext - │ - ▼ - Encryption - │ - ▼ - Chunking - │ - ▼ - Encrypted ciphertext - │ - ┌──────────┴──────────┐ - ▼ ▼ - Telegram Discord + YOUR DEVICE + │ + Plaintext + │ + ▼ + Encryption + │ + ▼ + Chunking + │ + ▼ + Encrypted ciphertext + │ + ┌──────────┴──────────┐ + ▼ ▼ + Telegram Discord ``` Storage providers should only receive ciphertext. @@ -120,7 +123,146 @@ Stashify uses established cryptographic libraries rather than implementing crypt --- -### Chunked storage +## Key Management + +Stashify uses a **Repository Master Key (RMK)** hierarchy for key management: + +```text +Repository Master Key (RMK) + │ + ├── File Encryption Key 1 + ├── File Encryption Key 2 + └── File Encryption Key N +``` + +### How it works + +1. **`stash init`** generates a cryptographically random **Repository Master Key (RMK)** +2. The RMK is **stored in your OS credential store** (Windows Credential Manager, macOS Keychain, Linux secret-service) via the `keyring` library +3. Each file gets a **unique File Encryption Key** derived from the RMK + file ID +4. File keys are used to encrypt chunks and filenames +5. No password prompts during normal operations + +### Key Storage + +* **RMK** → OS keyring (Windows Credential Manager / macOS Keychain / Linux secret-service) +* **File keys** → Derived on-the-fly from RMK + file ID (never stored) +* **Chunk keys** → Derived from file key + chunk index +* **Metadata** → Only non-secret identity info in `.stash/identity.json` + +### UX Flow + +**First device:** +```text +stash init + ↓ +Generate RMK + ↓ +Protect/store RMK in OS keyring + ↓ +Show recovery key (RMK hex) — SAVE THIS! + ↓ +Ready +``` + +**Normal operations:** +```text +stash put file.zip + ↓ +Retrieve RMK from keyring + ↓ +Generate/use File Encryption Key + ↓ +Encrypt + ↓ +Upload +``` +No password prompt. + +**New device / recovery:** +```text +stash unlock --recovery-key + ↓ +RMK restored in OS keyring + ↓ +Ready +``` + +### Key Commands + +| Command | Description | +|---------|-------------| +| `stash key-commands status` | Show key management status | +| `stash key-commands lock` | Remove RMK from keyring (lock repo) | +| `stash key-commands unlock --recovery-key ` | Restore RMK from recovery key | +| `stash key-commands recovery` | Show RMK for backup | + +--- + +## Why Stashify? + +Traditional cloud storage usually means trusting one provider with both your data and your storage. + +Stashify separates those concepts. + +| Problem | Stashify | +| -------------------------- | ------------------------------------- | +| Vendor lock-in | Provider-agnostic storage abstraction | +| Provider sees plaintext | Files are encrypted before upload | +| Large files | Automatic chunking | +| Provider-specific limits | Provider-aware chunking and routing | +| Single provider dependency | Multi-provider storage | +| Interrupted uploads | Resumable operations | +| Manual file management | Unified CLI | +| Provider-specific APIs | One consistent interface | + +--- + +## Features + +### Client-side encryption + +Your files are encrypted **before they leave your device**. + +```text + YOUR DEVICE + │ + Plaintext + │ + ▼ + Encryption + │ + ▼ + Chunking + │ + ▼ + Encrypted ciphertext + │ + ┌──────────┴──────────┐ + ▼ ▼ + Telegram Discord +``` + +Storage providers should only receive ciphertext. + +Stashify is designed around: + +* Client-side encryption +* Authenticated encryption (AEAD) +* Per-file cryptographic keys +* Proper key derivation +* Cryptographically secure randomness +* No custom cryptographic primitives +* Authenticated chunk integrity +* Local key management + +Stashify uses established cryptographic libraries rather than implementing cryptography from scratch. + +> **Security note:** Stashify does not claim that multi-provider storage makes encryption stronger. Confidentiality comes from the cryptographic design and key management. Multi-provider storage primarily provides distribution, redundancy, and provider independence. + +--- + +## Chunked storage Large files are automatically split into manageable chunks. @@ -145,7 +287,7 @@ The storage engine can account for provider-specific upload limitations without --- -### Multi-provider storage +## Multi-provider storage Stashify can distribute a file across multiple storage providers. @@ -174,20 +316,20 @@ This allows Stashify to build storage around the providers available to you inst --- -### Provider abstraction +## Provider abstraction Providers are implementations of the same storage interface. ```text - Storage Provider - │ - ┌────────────────┼────────────────┐ - │ │ │ - Telegram Discord S3 / B2 - │ │ │ - └────────────────┼────────────────┘ - │ - Storage Engine + Storage Provider + │ + ┌────────────────┼────────────────┐ + │ │ │ + Telegram Discord S3 / B2 + │ │ │ + └────────────────┼────────────────┘ + │ + Storage Engine ``` The core engine does not need to know how Telegram or Discord works. @@ -205,20 +347,20 @@ Planned providers include: --- -### Asynchronous transfers +## Asynchronous transfers Stashify is designed around asynchronous I/O. Large uploads can consist of hundreds or thousands of chunks, so operations should run concurrently with bounded workers. ```text - Upload Queue - │ - ┌───────────┼───────────┐ - ▼ ▼ ▼ - Worker 1 Worker 2 Worker 3 - │ │ │ - Telegram Discord Telegram + Upload Queue + │ + ┌───────────┼───────────┐ + ▼ ▼ ▼ + Worker 1 Worker 2 Worker 3 + │ │ │ + Telegram Discord Telegram ``` The transfer system is designed to support: @@ -236,7 +378,7 @@ The transfer system is designed to support: --- -### Resumable uploads +## Resumable uploads Interrupted transfers shouldn't mean starting from zero. @@ -257,7 +399,7 @@ Stashify keeps track of individual chunks so completed work can be preserved acr --- -### Integrity verification +## Integrity verification Encrypted chunks are authenticated and tracked using local metadata. @@ -275,7 +417,7 @@ The final reconstructed file can also be verified against file-level integrity i --- -### Manifest-based storage +## Manifest-based storage Every stored file has a manifest describing how it can be reconstructed. @@ -298,81 +440,56 @@ File └── ... ``` -Local metadata is stored in SQLite. - ---- - -# Terminal UI (planned) - -Stashify is designed to have a full interactive terminal interface rather than being limited to traditional commands. - -Run: - -```bash -stashify -``` - -to launch the TUI. - -The interface is designed around a keyboard-first workflow with: - -* Local file explorer -* Remote storage explorer -* Multi-file selection -* Upload/download controls -* Live transfer progress -* Provider status -* Transfer queue -* Provider management -* Configuration -* Search -* Command palette -* Keyboard shortcuts +Local metadata is stored in `.stash/metadata/`. --- -# CLI +## CLI -Stashify can also be used without the interactive interface. +Stashify can be used entirely from the command line. ```bash -# Initialize -stashify init +# Initialize (generates RMK, stores in OS keyring, shows recovery key) +stash init # Configure providers -stashify provider add telegram -stashify provider add discord +stash provider add telegram +stash provider add discord # List providers -stashify provider list +stash provider list -# Upload -stashify put ./movie.mkv +# Upload (no password prompt — uses RMK from keyring) +stash put ./movie.mkv # List stored files -stashify ls +stash ls # Inspect a file -stashify info movie.mkv +stash info movie.mkv -# Download -stashify get movie.mkv +# Download (no password prompt) +stash get movie.mkv # Delete -stashify rm movie.mkv +stash rm movie.mkv # Verify -stashify verify movie.mkv +stash verify movie.mkv # Check status -stashify status -``` +stash status -The exact command set may evolve during development. +# Key management +stash key-commands status +stash key-commands lock +stash key-commands unlock --recovery-key +stash key-commands recovery +``` --- -# Provider Support +## Provider Support | Provider | Status | | ---------------- | -------------- | @@ -398,25 +515,25 @@ Stashify's job is to abstract those providers and make the best use of the stora --- -# Security & Privacy +## Security & Privacy Stashify is designed around a simple trust model: ```text - Trusted - │ - ▼ - ┌───────────┐ - │ User Device│ - └─────┬─────┘ - │ - encrypted data - │ - ┌───────┴───────┐ - ▼ ▼ - Telegram Discord - untrusted untrusted - storage storage + Trusted + │ + ▼ + ┌───────────┐ + │ User Device│ + └─────┬─────┘ + │ + encrypted data + │ + ┌───────┴───────┐ + ▼ ▼ + Telegram Discord + untrusted untrusted + storage storage ``` ### Principles @@ -435,11 +552,11 @@ Stashify uses established cryptographic implementations. **Authenticated data** -Encrypted chunks should provide confidentiality and integrity. +Encrypted chunks provide confidentiality and integrity. **Minimal exposure** -Provider APIs should receive only the information required to store and retrieve encrypted data. +Provider APIs receive only the information required to store and retrieve encrypted data. **Open source** @@ -449,34 +566,34 @@ The codebase is intended to remain publicly auditable. --- -# Architecture +## Architecture At a high level: ```text - ┌───────▼───────┐ - │ CLI │ - └───────┬───────┘ - │ - ┌───────▼───────┐ - │ Core Engine │ - └───────┬───────┘ - │ - ┌──────────────────┼──────────────────┐ - │ │ │ - ▼ ▼ ▼ - Encryption Chunking Metadata - │ │ │ - └──────────────────┼──────────────────┘ - │ - ┌───────▼───────┐ - │ Storage Router│ - └───────┬───────┘ - │ - ┌────────────┼────────────┐ - ▼ ▼ ▼ - Telegram Discord Future - Providers + ┌───────▼───────┐ + │ CLI │ + └───────┬───────┘ + │ + ┌───────▼───────┐ + │ Core Engine │ + └───────┬───────┘ + │ + ┌──────────────────┼──────────────────┐ + │ │ │ + ▼ ▼ ▼ + Encryption Chunking Metadata + │ │ │ + └──────────────────┼──────────────────┘ + │ + ┌───────▼───────┐ + │ Storage Router│ + └───────┬───────┘ + │ + ┌────────────┼────────────┐ + ▼ ▼ ▼ + Telegram Discord Future + Providers ``` The architecture intentionally separates: @@ -493,7 +610,7 @@ This allows the system to evolve without coupling the entire codebase to a speci --- -# Project Status +## Project Status > **Stashify is currently in early development.** @@ -510,6 +627,7 @@ The architecture is being actively developed and APIs may change significantly. | Telegram provider | Implemented | | Discord provider | Implemented | | Async job engine | Implemented | +| **Key management (RMK)** | **Implemented** | | Resumable transfers | Planned | | Multi-provider routing | Planned | | Verification | Planned | @@ -520,7 +638,7 @@ Features marked **Planned** or **Future** should not be considered implemented. --- -# Development +## Development Stashify is built with Python and is designed around asynchronous I/O. @@ -530,49 +648,28 @@ A typical architecture is: ```text UI - │ - └── CLI - │ - ▼ + │ + └── CLI + │ + ▼ Core Storage Engine - │ - ├── Crypto - ├── Chunking - ├── Metadata - ├── Jobs - └── Storage Router - │ - ├── Telegram - ├── Discord - └── Future Providers + │ + ├── Crypto + ├── Chunking + ├── Metadata + ├── Jobs + └── Storage Router + │ + ├── Telegram + ├── Discord + └── Future Providers ``` See [CONTRIBUTING.md](CONTRIBUTING.md) for contribution guidelines. --- -# Contributing - -Contributions are welcome. - -Some areas where contributions will be especially useful: - -* Storage providers -* Encryption and security review -* Chunking and transfer reliability -* TUI/UX improvements -* Testing -* Documentation -* Performance -* Cross-platform support - -Please read [CONTRIBUTING.md](CONTRIBUTING.md) before opening a pull request. - -If you discover a potential security vulnerability, please follow the project's security reporting process rather than publicly disclosing the issue immediately. - ---- - -# Roadmap +## Roadmap The long-term goal is to turn Stashify into a flexible encrypted storage layer that can sit on top of almost any suitable storage provider. @@ -580,10 +677,11 @@ The long-term goal is to turn Stashify into a flexible encrypted storage layer t * [x] Core encryption pipeline * [x] Chunking -* [ ] SQLite metadata +* [x] SQLite metadata * [x] Telegram provider * [x] Discord provider * [x] Async transfer system +* [x] **Repository Master Key (RMK) hierarchy** * [ ] Resumable uploads ### Medium term @@ -608,7 +706,7 @@ The roadmap is intentionally flexible and will evolve as the project matures. --- -# License +## License Stashify is released under the **MIT License**. @@ -619,4 +717,4 @@ See [LICENSE](LICENSE) for the full license text.

Stashify
Your files. Your keys. Your storage. -

+

\ No newline at end of file diff --git a/docs/architecture.md b/docs/architecture.md index b812783..8a9c809 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2,13 +2,13 @@ ## Overview -Stash is a privacy-focused CLI storage system that uses third-party platforms as encrypted storage backends. The architecture is designed around a provider-agnostic core with pluggable storage backends. +Stashify is a privacy-focused CLI storage system that uses third-party platforms as encrypted storage backends. The architecture is designed around a provider-agnostic core with pluggable storage backends. ## Core Components ``` ┌─────────────────────────────────────────────────────────────┐ -│ Stash CLI │ +│ Stashify CLI │ ├─────────────────────────────────────────────────────────────┤ │ Commands (put, get, ls, info, rm, verify, status, etc.) │ ├─────────────────────────────────────────────────────────────┤ @@ -29,11 +29,20 @@ Stash is a privacy-focused CLI storage system that uses third-party platforms as ## Core Modules ### Crypto Engine (`src/stash/core/crypto.py`) -- **AES-256-GCM** encryption for chunks +- **AES-256-GCM** encryption for chunks and filenames - **HKDF-SHA256** key derivation -- Per-file encryption keys with HKDF-SHA256 derivation -- Per-chunk keys derived via HKDF from file key -- Password-based key wrapping with Argon2id (planned) +- **Repository Master Key (RMK)** based key hierarchy +- File keys derived from RMK + file_id via HKDF-SHA256 +- Chunk keys derived from file key + chunk_index via HKDF-SHA256 +- Filename encryption via dedicated filename key +- No password-based key wrapping (RMK replaces password-based system) + +### Key Manager (`src/stash/core/keymanager.py`) +- **Repository Master Key (RMK)** management using OS keyring +- RMK generation, storage, retrieval via `keyring` library +- OS credential store integration (CredMan/Keychain/secret-service) +- RMK locking/unlocking with recovery key support +- Repository identity management ### Chunking Manager (`src/stash/core/chunking.py`) - Configurable chunk sizes (default 10MB) @@ -101,20 +110,23 @@ File → Encrypt → Chunk → Per-chunk encrypt → Upload to providers → Sav ``` 1. **File Input** → Read file stream -2. **Encryption** → Generate file key, encrypt filename +2. **Key Retrieval** → Get RMK from OS keyring +3. **File Key Derivation** → HKDF-SHA256(RMK, file_id) +3. **Filename Encryption** → Encrypt with file key 4. **Chunking** → Split into configurable chunks (default 10MB) -5. **Per-Chunk Encryption** → HKDF-derived per-chunk key + AES-256-GCM +4. **Per-Chunk Encryption** → HKDF-derived per-chunk key + AES-256-GCM 5. **Provider Upload** → Parallel upload to configured providers 7. **Manifest Creation** → Store metadata, chunk mappings, encryption params 8. **Persist** → Save manifest to local metadata store ### Download Flow ``` -Manifest → Decrypt filename → Resolve chunks → Download from providers → Decrypt → Reconstruct +Manifest → Get RMK from keyring → Derive file key → Decrypt filename → Resolve chunks → Download from providers → Decrypt → Reconstruct ``` 1. **Manifest Lookup** → Load file metadata -4. **Key Derivation** → Decrypt file key with password +2. **RMK Retrieval** → Get RMK from OS keyring +4. **File Key Derivation** → HKDF-SHA256(RMK, file_id) 4. **Filename Decryption** → Decrypt original filename 5. **Chunk Resolution** → Determine providers for each chunk 6. **Parallel Download** → Fetch chunks from providers @@ -141,7 +153,7 @@ Manifest → Decrypt filename → Resolve chunks → Download from providers → ## Security Model ### Threat Model -- **Trusted**: Local machine, user password +- **Trusted**: Local machine, user recovery key - **Untrusted**: Storage providers, network - **Assumption**: Provider may be malicious/curious @@ -149,19 +161,26 @@ Manifest → Decrypt filename → Resolve chunks → Download from providers → - **Confidentiality**: AES-256-GCM encryption - **Integrity**: GCM authentication tags + SHA-256 checksums - **Forward Secrecy**: Per-file keys, per-chunk keys -- **Provider Isolation**: Providers cannot decrypt without user password +- **Provider Isolation**: Providers cannot decrypt without RMK ### Key Hierarchy ``` -User Password - ↓ Argon2id (planned) / PBKDF2 -Master Key - ↓ HKDF-SHA256 -File Key (per file) - ↓ HKDF-SHA256 -Chunk Key (per chunk) +Repository Master Key (RMK) + │ + ├── File Encryption Key 1 (HKDF-SHA256(RMK, file_id)) + ├── File Encryption Key 2 + └── File Encryption Key N + │ + ├── Chunk Key 0 (HKDF-SHA256(file_key, "stash-chunk-0")) + ├── Chunk Key 1 + └── Chunk Key N ``` +- RMK: 32 random bytes, generated at `stash init`, stored in OS keyring +- File keys: HKDF-SHA256(RMK, salt=file_id, info="stash-file-key") +- Chunk keys: HKDF-SHA256(file_key, info="stash-chunk-{index}") +- Filename keys: HKDF-SHA256(file_key, info="stash-filename") + ## Error Handling - **Retry Logic**: Exponential backoff with jitter diff --git a/docs/cli-reference.md b/docs/cli-reference.md index a3c0e02..1c7467a 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -12,7 +12,7 @@ ## Commands ### `stash init` -Initialize a new Stash repository. +Initialize a new Stash repository (generates RMK, stores in OS keyring). ```bash stash init [--repo PATH] [--force] @@ -23,6 +23,8 @@ stash init [--repo PATH] [--force] | `--repo, -r` | Repository path (default: current directory) | | `--force, -f` | Overwrite existing repository | +**Output:** Shows Repository ID and Recovery Key (RMK hex) — **save the recovery key!** + ### `stash provider` Manage storage providers. @@ -69,7 +71,7 @@ stash provider remove [--force] ``` ### `stash put` -Store a file in Stash. +Store a file in Stash (uses RMK from keyring — no password prompt). ```bash stash put [options] @@ -80,11 +82,10 @@ stash put [options] | `--provider, -p` | Specific provider to use | | `--chunk-size` | Chunk size in bytes (default: provider limit) | | `--strategy` | Distribution: single, split, balanced, replicated | -| `--password` | Encryption password (prompt if not provided) | | `--confirm/--no-confirm` | Skip confirmation prompt | ### `stash get` -Retrieve a file from Stash. +Retrieve a file from Stash (uses RMK from keyring — no password prompt). ```bash stash get [options] @@ -93,7 +94,6 @@ stash get [options] | Option | Description | |--------|-------------| | `--output, -o` | Output path (default: current directory) | -| `--password` | Encryption password (prompt if not provided) | | `--overwrite` | Overwrite existing file | ### `stash ls` @@ -144,6 +144,41 @@ Show overall repository status. stash status ``` +### `stash key-commands` +Repository key management. + +#### `stash key-commands lock` +Lock the repository by removing RMK from keyring. + +```bash +stash key-commands lock +``` + +#### `stash key-commands unlock` +Unlock the repository on a new device using a recovery key. + +```bash +stash key-commands unlock --recovery-key +``` + +| Option | Description | +|--------|-------------| +| `--recovery-key` | Recovery key (RMK hex) to restore RMK | + +#### `stash key-commands status` +Show key management status. + +```bash +stash key-commands status +``` + +#### `stash key-commands recovery` +Show the recovery key (RMK) for backup purposes. + +```bash +stash key-commands recovery +``` + ## Global Options | Option | Description | @@ -163,4 +198,19 @@ stash status | 3 | File not found | | 4 | Authentication failed | | 5 | Network error | -| 6 | Storage limit exceeded | \ No newline at end of file +| 6 | Storage limit exceeded | + +## Key Management Flow + +``` +First device: New device: +stash init stash unlock --recovery-key + ↓ ↓ +Generate RMK Restore RMK to keyring + ↓ ↓ +Store in OS keyring Ready to use + ↓ +Show recovery key (SAVE!) +``` + +**Important:** The recovery key (RMK hex) is shown **once** during `stash init`. Save it securely — it's the only way to unlock the repository on a new device. \ No newline at end of file diff --git a/docs/configuration.md b/docs/configuration.md index 2eeb9bd..4f64a16 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -2,7 +2,7 @@ ## Repository Configuration -Each Stash repository stores its configuration in `.stash/config.json`: +Each Stashify repository stores its configuration in `.stash/config.json`: ```json { @@ -23,6 +23,18 @@ Each Stash repository stores its configuration in `.stash/config.json`: } ``` +The repository identity (including RMK reference) is stored in `.stash/identity.json`: + +```json +{ + "repository_id": "abc123...", + "created_at": 1699999999.123, + "version": 1 +} +``` + +The **Repository Master Key (RMK)** is stored in the OS credential store, not in config files. + ## Global Options | Option | Description | Default | @@ -92,12 +104,29 @@ animations = true progress_style = "bar" ``` +## Key Management + +The **Repository Master Key (RMK)** is managed by the OS keyring: + +| Command | Description | +|---------|-------------| +| `stash key-commands status` | Show key management status | +| `stash key-commands lock` | Remove RMK from keyring (lock repo) | +| `stash key-commands unlock --recovery-key ` | Restore RMK from recovery key | +| `stash key-commands recovery` | Show RMK for backup | + +The RMK is stored in the OS credential store: +- **Windows**: Credential Manager +- **macOS**: Keychain +- **Linux**: secret-service (GNOME Keyring, KWallet, etc.) + +No passwords or raw keys are stored in configuration files. + ## Environment Variables | Variable | Description | |----------|-------------| | `STASH_REPO` | Default repository path | -| `STASH_PASSWORD` | Default encryption password (not recommended) | | `STASH_VERBOSE` | Enable verbose output | | `DISCORD_TOKEN` | Default Discord bot token | | `TELEGRAM_TOKEN` | Default Telegram bot token | @@ -118,4 +147,5 @@ Each provider has built-in limits: | Algorithm | AES-256-GCM | Encryption algorithm | | Key Size | 256 bits | Encryption key size | | Chunk Key Derivation | HKDF-SHA256 | Per-chunk key derivation | -| Key Derivation | HKDF-SHA256 | Per-file key derivation | \ No newline at end of file +| File Key Derivation | HKDF-SHA256 | Per-file key derivation from RMK | +| RMK Derivation | HKDF-SHA256 | File key derivation from RMK + file_id | \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 3c4407c..c00e706 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,20 +1,20 @@ -# Stash +# Stashify -**Stash** is a privacy-focused CLI storage system that uses third-party platforms (Telegram, Discord) as encrypted storage backends. +**Stashify** is a privacy-focused CLI storage system that uses third-party platforms (Telegram, Discord) as encrypted storage backends. ## Quick Start ```bash # Install -pip install stash +pip install stashify -# Initialize repository +# Initialize repository (generates RMK, stores in OS keyring) stash init # Add storage provider stash provider add telegram --token --chat-id -# Store a file +# Store a file (no password prompt — uses RMK from keyring) stash put file.txt # Retrieve a file @@ -34,6 +34,7 @@ stash get file.txt ## Features - **Client-side encryption**: AES-256-GCM with per-file keys +- **Repository Master Key (RMK) hierarchy**: Keys stored in OS keyring - **Multi-provider support**: Telegram, Discord (S3, B2, Google Drive planned) - **Chunked storage**: Automatic chunking for large files - **Multi-provider distribution**: Single, split, balanced, replicated strategies @@ -54,7 +55,8 @@ stash get file.txt - **AES-256-GCM** encryption per chunk - **HKDF-SHA256** key derivation per chunk -- Per-file encryption keys with HKDF-SHA256 derivation +- **Repository Master Key (RMK)** hierarchy stored in OS keyring +- Per-file encryption keys derived from RMK + file ID - Zero-knowledge: providers never see plaintext ## License diff --git a/docs/installation.md b/docs/installation.md index dfe273d..aa09e53 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -2,13 +2,13 @@ ## Prerequisites -- Python 3.11+ +- Python 3.12+ - pip (Python package manager) ## Install from PyPI ```bash -pip install stash +pip install stashify ``` ## Install from Source @@ -50,13 +50,13 @@ stash --install-completion fish ## Docker ```bash -docker pull ghcr.io/sparkleeop/stash:latest -docker run --rm -v /path/to/repo:/repo ghcr.io/sparkleeop/stash:latest --repo /repo init +docker pull ghcr.io/sparkleeop/stashify:latest +docker run --rm -v /path/to/repo:/repo ghcr.io/sparkleeop/stashify:latest --repo /repo init ``` ## Requirements -- Python 3.11+ +- Python 3.12+ - Dependencies are automatically installed via pip - Optional: Docker for containerized deployment @@ -67,4 +67,33 @@ docker run --rm -v /path/to/repo:/repo ghcr.io/sparkleeop/stash:latest --repo /r | Linux | ✅ Fully supported | | macOS | ✅ Fully supported | | Windows | ✅ Fully supported | -| Docker | ✅ Supported | \ No newline at end of file +| Docker | ✅ Supported | + +## First Run: Initialize Repository + +After installation, you need to initialize a repository: + +```bash +stash init +``` + +This will: +1. Generate a cryptographically random **Repository Master Key (RMK)** +2. Store the RMK securely in your OS credential store (Windows Credential Manager, macOS Keychain, Linux secret-service) +3. Display a **recovery key (RMK hex)** — **SAVE THIS SECURELY!** + +The recovery key is the only way to unlock the repository on a new device or if the keyring entry is lost. + +## Configure a Provider + +After initialization, add at least one storage provider: + +```bash +# Telegram +stash provider add telegram --token --chat-id + +# Discord +stash provider add discord --token --channel-id +``` + +See [Providers](providers/README.md) for detailed setup instructions. \ No newline at end of file diff --git a/docs/security.md b/docs/security.md index e8f52ed..4b6afd2 100644 --- a/docs/security.md +++ b/docs/security.md @@ -2,7 +2,7 @@ ## Overview -Stash is designed with a **zero-knowledge** security model. The storage providers (Telegram, Discord, etc.) only ever receive encrypted, opaque data blobs. They cannot decrypt, inspect, or modify your data. +Stashify is designed with a **zero-knowledge** security model. The storage providers (Telegram, Discord, etc.) only ever receive encrypted, opaque data blobs. They cannot decrypt, inspect, or modify your data. ## Encryption @@ -15,13 +15,15 @@ Stash is designed with a **zero-knowledge** security model. The storage provider ### Key Hierarchy ``` -User Password - ↓ PBKDF2 (100,000 iterations) / Argon2id (planned) -Master Key (32 bytes) - ↓ HKDF-SHA256 (salt: file_key_salt) -File Key (32 bytes per file) - ↓ HKDF-SHA256 (info: "stash-chunk-{index}") -Chunk Key (32 bytes per chunk) +Repository Master Key (RMK) + │ + ├── File Encryption Key 1 (derived from RMK + file_id) + ├── File Encryption Key 2 + └── File Encryption Key N + │ + ├── Chunk Key 0 (derived from file_key + chunk_index) + ├── Chunk Key 1 + └── Chunk Key N ``` ### Chunk Encryption @@ -32,12 +34,17 @@ Each chunk is independently encrypted: 3. Encrypt with AES-256-GCM: `ciphertext = AES-GCM(chunk_key, nonce, plaintext, aad=None)` 4. Store: `nonce (12 bytes) + ciphertext + tag (16 bytes)` -### Key Wrapping +### File Key Derivation -File keys are wrapped with the user's password: -1. Derive wrapping key: `HKDF-SHA256(password, salt=16_bytes, info="stash-key-wrap")` -2. Encrypt file key: `AES-GCM(wrapping_key, nonce, file_key)` -3. Store: `salt (16) + nonce (12) + ciphertext + tag (16)` +Per-file encryption keys are derived from the RMK: +1. Derive file key: `HKDF-SHA256(RMK, salt=file_id, info="stash-file-key")` +2. File key is 32 bytes (AES-256) + +### Filename Encryption + +Filenames are encrypted using the file key: +1. Derive filename key: `HKDF-SHA256(file_key, info="stash-filename")` +2. Encrypt with AES-256-GCM ## Integrity @@ -57,15 +64,36 @@ File keys are wrapped with the user's password: ## Key Management -### Password Handling -- Never stored, only used for key derivation -- Zeroized from memory after use -- Minimum 8 characters recommended +### Repository Master Key (RMK) + +The RMK is the root key for the repository: +- Generated once during `stash init` (32 random bytes) +- Stored in OS credential store (Windows Credential Manager, macOS Keychain, Linux secret-service) +- Never written to disk +- Accessed via `keyring` library + +### Recovery Key + +The RMK itself serves as the **recovery key**: +- Displayed once during `stash init` +- Must be saved securely by the user +- Used to unlock repository on new devices via `stash key-commands unlock --recovery-key ` + +### Key Storage + +| Component | Storage | Encryption | +|-----------|---------|------------| +| RMK | OS keyring (CredMan/Keychain/secret-service) | Encrypted by OS | +| File keys | Derived on-the-fly from RMK + file_id | Not stored | +| Chunk keys | Derived on-the-fly from file_key + chunk_index | Not stored | +| Provider credentials | `.stash/config.json` | Plaintext (OS file permissions) | +| Recovery key | User-managed (offline) | User responsibility | + +### Lock / Unlock -### Key Rotation (Planned) -- Periodic master key rotation -- Re-encryption of file keys -- Automatic re-encryption on access +- `stash key-commands lock` — removes RMK from keyring +- `stash key-commands unlock --recovery-key ` — restores RMK +- `stash key-commands status` — shows current lock state ## Provider Security @@ -86,14 +114,14 @@ File keys are wrapped with the user's password: - All provider communication over HTTPS/TLS - Certificate validation enforced - No plaintext credentials in transit -- Token storage: encrypted in local config +- Token storage: local config (OS file permissions) ## Threat Model ### Trusted - Local machine (user's device) -- User password/credentials -- Stash binary (if verified) +- User recovery key (if backed up) +- Stashify binary (if verified) ### Untrusted - Storage providers (Telegram, Discord, etc.) @@ -104,7 +132,6 @@ File keys are wrapped with the user's password: - Local malware/keyloggers - Physical device access - Side-channel attacks -- Password brute force (mitigated by strong passwords) ## Provider Compromise Scenarios @@ -117,11 +144,11 @@ File keys are wrapped with the user's password: ## Key Rotation (Planned) -1. Generate new master key -2. Re-wrap all file keys +1. Generate new RMK +2. Re-wrap all file keys with new RMK 3. Re-encrypt filenames -4. Atomic manifest update -5. Old keys zeroized +5. Atomic manifest update +6. Old keys zeroized ## Compliance Considerations diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 363742a..1c2ffe3 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -3,7 +3,7 @@ ## Installation ### `pip install` fails on Windows -Ensure you have the latest `pip` and Python 3.9+: +Ensure you have the latest `pip` and Python 3.12+: ```bash python -m pip install --upgrade pip @@ -90,12 +90,29 @@ Discord rate limits are strict. Reduce concurrency: stash provider add discord --max-concurrent 2 ``` -## Authentication +## Key Management (RMK) -### Password prompts fail -- Stash uses `getpass`, which requires an interactive terminal -- On Windows, use the built-in console (not some SSH/CI shells) -- If automation is needed, use `--password` flag (less secure) +### "RMK not found in keyring" / "Repository locked or key unavailable" +The Repository Master Key (RMK) is not in the OS keyring. This happens when: +- You're on a new device and haven't run `stash unlock` +- You ran `stash key-commands lock` and haven't unlocked +- The keyring entry was deleted + +**Fix:** Run `stash unlock --recovery-key ` with your recovery key. + +### "Recovery key required" / "Invalid recovery key format" +You must provide a valid 64-character hex recovery key (32 bytes = 64 hex chars): +```bash +stash key-commands unlock --recovery-key <64-char-hex> +``` + +### "Repository already unlocked" +The RMK is already in the keyring. No action needed. + +### Lost recovery key +**There is no recovery.** The RMK is the only way to unlock the repository. If you lose the recovery key and the RMK is not in the keyring, you cannot decrypt existing files. You must re-initialize the repository and re-upload files. + +## Provider Authentication ### "Authentication failed" on get 1. Provider credentials may have changed (token revoked/rotated) @@ -119,16 +136,6 @@ Stash stores manifests in `.stash/metadata/`. If corrupted: 2. Restore from backup if you have one 3. Re-upload the file if all backups are gone -## Encryption - -### "Decryption failed" -1. Wrong password: verify the password you used with `put` -2. Corrupted data: check provider messages are intact -3. Wrong provider credentials: chunks may be from a different account - -### Lost password -**There is no recovery.** Encryption keys are derived from your password and never stored. Re-upload files with a new password you can remember. - ## Networking ### "Connection timeout" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2c906a9 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +cryptography>=49.0 +click>=8.1 +pydantic>=2.7 +pydantic-settings>=2.3 +sqlite-utils>=3.37 +tqdm>=4.70.0 +structlog>=24.1 +httpx>=0.27 +pyyaml>=6.0 +rich>=13.0 +keyring>=25.0 \ No newline at end of file From c2a1d7fde4c426b6efdd7cc19107b942aede47c5 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 18:14:33 +0530 Subject: [PATCH 16/19] docs: update README to enhance image display and alignment --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 82e8877..18e1750 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,10 @@ # Stashify -[![IMG-20260820-WA0006.jpg](https://i.postimg.cc/15hbDPcL/IMG-20260820-WA0006.jpg)](https://postimg.cc/7b9ByFRV) +

+ + Stashify + +

From 6ea8d9f34ec4baf0d67c699e812835f9f84bacd2 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 18:26:12 +0530 Subject: [PATCH 17/19] docs: update README for improved layout and badge links --- README.md | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 18e1750..c6b90d8 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,8 @@ -# Stashify -

- - Stashify - + Stashify Logo

+# Stashify

Encrypted storage. Your providers. Your keys. @@ -16,13 +13,13 @@

- + GitHub Stars - + License - Python + Python Status

From beb0bca17ea309c286e26796b53dbe6573a31102 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 18:51:37 +0530 Subject: [PATCH 18/19] docs: remove fake claims from README and docs - be honest about what's implemented --- README.md | 314 +++++++----------------------------------- docs/cli-reference.md | 3 +- docs/configuration.md | 26 +--- src/stash/__init__.py | 2 +- 4 files changed, 51 insertions(+), 294 deletions(-) diff --git a/README.md b/README.md index c6b90d8..fd26220 100644 --- a/README.md +++ b/README.md @@ -40,19 +40,19 @@ Files are encrypted locally, split into chunks, and uploaded as ciphertext. Chun │ Storage Engine │ - ┌─────────────┴─────────────┐ - │ │ - Encryption Chunking - │ │ - └─────────────┬─────────────┘ - │ - Storage Router - │ - ┌─────────────┴─────────────┐ - │ │ - Telegram Discord - │ │ - encrypted chunks encrypted chunks + ┌─────────────┴─────────────┐ + │ │ + Encryption Chunking + │ │ + └─────────────┬─────────────┘ + │ + Storage Router + │ + ┌─────────────┴─────────────┐ + │ │ + Telegram Discord + │ │ + encrypted chunks encrypted chunks ``` Stashify does **not** provide the underlying storage. @@ -72,10 +72,8 @@ Stashify separates those concepts. | Vendor lock-in | Provider-agnostic storage abstraction | | Provider sees plaintext | Files are encrypted before upload | | Large files | Automatic chunking | -| Provider-specific limits | Provider-aware chunking and routing | +| Provider-specific limits | Provider-aware chunking | | Single provider dependency | Multi-provider storage | -| Interrupted uploads | Resumable operations | -| Manual file management | Unified CLI/TUI | | Provider-specific APIs | One consistent interface | --- @@ -98,7 +96,7 @@ Your files are encrypted **before they leave your device**. Chunking │ ▼ - Encrypted ciphertext + Encrypted ciphertext │ ┌──────────┴──────────┐ ▼ ▼ @@ -200,69 +198,6 @@ Ready --- -## Why Stashify? - -Traditional cloud storage usually means trusting one provider with both your data and your storage. - -Stashify separates those concepts. - -| Problem | Stashify | -| -------------------------- | ------------------------------------- | -| Vendor lock-in | Provider-agnostic storage abstraction | -| Provider sees plaintext | Files are encrypted before upload | -| Large files | Automatic chunking | -| Provider-specific limits | Provider-aware chunking and routing | -| Single provider dependency | Multi-provider storage | -| Interrupted uploads | Resumable operations | -| Manual file management | Unified CLI | -| Provider-specific APIs | One consistent interface | - ---- - -## Features - -### Client-side encryption - -Your files are encrypted **before they leave your device**. - -```text - YOUR DEVICE - │ - Plaintext - │ - ▼ - Encryption - │ - ▼ - Chunking - │ - ▼ - Encrypted ciphertext - │ - ┌──────────┴──────────┐ - ▼ ▼ - Telegram Discord -``` - -Storage providers should only receive ciphertext. - -Stashify is designed around: - -* Client-side encryption -* Authenticated encryption (AEAD) -* Per-file cryptographic keys -* Proper key derivation -* Cryptographically secure randomness -* No custom cryptographic primitives -* Authenticated chunk integrity -* Local key management - -Stashify uses established cryptographic libraries rather than implementing cryptography from scratch. - -> **Security note:** Stashify does not claim that multi-provider storage makes encryption stronger. Confidentiality comes from the cryptographic design and key management. Multi-provider storage primarily provides distribution, redundancy, and provider independence. - ---- - ## Chunked storage Large files are automatically split into manageable chunks. @@ -290,30 +225,16 @@ The storage engine can account for provider-specific upload limitations without ## Multi-provider storage -Stashify can distribute a file across multiple storage providers. +Stashify can store files across multiple storage providers. Currently only the **Single** strategy is implemented (all chunks on one provider). Additional strategies are planned. -For example: +Current strategies: -```text -100 encrypted chunks - -Telegram: - 0 2 4 6 8 10 12 ... - -Discord: - 1 3 5 7 9 11 13 ... -``` - -Multiple storage strategies are planned: - -| Strategy | Description | -| -------------- | ----------------------------------------------------- | -| **Single** | Store all chunks on one provider | -| **Split** | Distribute chunks across multiple providers | -| **Balanced** | Dynamically distribute chunks based on provider state | -| **Replicated** | Store copies across multiple providers | - -This allows Stashify to build storage around the providers available to you instead of forcing you into a single backend. +| Strategy | Status | Description | +| -------------- | ------------ | ----------------------------------------------------- | +| **Single** | ✅ Implemented | Store all chunks on one provider | +| **Split** | 🚧 Planned | Distribute chunks across multiple providers | +| **Balanced** | 🚧 Planned | Dynamically distribute chunks based on provider state | +| **Replicated** | 🚧 Planned | Store copies across multiple providers | --- @@ -348,159 +269,17 @@ Planned providers include: --- -## Asynchronous transfers - -Stashify is designed around asynchronous I/O. - -Large uploads can consist of hundreds or thousands of chunks, so operations should run concurrently with bounded workers. - -```text - Upload Queue - │ - ┌───────────┼───────────┐ - ▼ ▼ ▼ - Worker 1 Worker 2 Worker 3 - │ │ │ - Telegram Discord Telegram -``` - -The transfer system is designed to support: - -* Concurrent uploads -* Concurrent downloads -* Bounded concurrency -* Retries -* Exponential backoff -* Provider-aware rate limiting -* Cancellation -* Progress reporting -* Failed-job tracking -* Resumable operations - ---- - -## Resumable uploads - -Interrupted transfers shouldn't mean starting from zero. - -```text -200 chunks - -chunk 000 ✓ -chunk 001 ✓ -chunk 002 ✓ -... -chunk 147 ✓ -chunk 148 ✗ -chunk 149 ✗ -... -``` - -Stashify keeps track of individual chunks so completed work can be preserved across interruptions. - ---- - -## Integrity verification - -Encrypted chunks are authenticated and tracked using local metadata. - -Stashify is designed to detect: - -* Missing chunks -* Corrupted chunks -* Modified ciphertext -* Incomplete downloads -* Invalid manifests -* Incorrect chunk ordering -* Failed reconstruction - -The final reconstructed file can also be verified against file-level integrity information. - ---- - -## Manifest-based storage - -Every stored file has a manifest describing how it can be reconstructed. - -Conceptually: - -```text -File -├── ID -├── Original name -├── Original size -├── Chunk size -├── Chunk count -├── Encryption metadata -├── Integrity information -│ -└── Chunks - ├── 0 → Telegram → remote ID - ├── 1 → Discord → remote ID - ├── 2 → Telegram → remote ID - └── ... -``` - -Local metadata is stored in `.stash/metadata/`. - ---- - -## CLI - -Stashify can be used entirely from the command line. - -```bash -# Initialize (generates RMK, stores in OS keyring, shows recovery key) -stash init - -# Configure providers -stash provider add telegram -stash provider add discord - -# List providers -stash provider list - -# Upload (no password prompt — uses RMK from keyring) -stash put ./movie.mkv - -# List stored files -stash ls - -# Inspect a file -stash info movie.mkv - -# Download (no password prompt) -stash get movie.mkv - -# Delete -stash rm movie.mkv - -# Verify -stash verify movie.mkv - -# Check status -stash status - -# Key management -stash key-commands status -stash key-commands lock -stash key-commands unlock --recovery-key -stash key-commands recovery -``` - ---- - ## Provider Support | Provider | Status | | ---------------- | -------------- | -| Telegram | Implemented | -| Discord | Implemented | -| S3-compatible | Planned | -| Backblaze B2 | Planned | -| Google Drive | Planned | -| Local filesystem | Planned | -| WebDAV | Planned | +| Telegram | ✅ Implemented | +| Discord | ✅ Implemented | +| S3-compatible | 🚧 Planned | +| Backblaze B2 | 🚧 Planned | +| Google Drive | 🚧 Planned | +| Local filesystem | 🚧 Planned | +| WebDAV | 🚧 Planned | Provider availability and capabilities are subject to the APIs, limits, and policies of the respective services. @@ -594,7 +373,7 @@ At a high level: ┌────────────┼────────────┐ ▼ ▼ ▼ Telegram Discord Future - Providers + Providers ``` The architecture intentionally separates: @@ -620,20 +399,20 @@ The architecture is being actively developed and APIs may change significantly. | Component | Status | | ---------------------- | ----------- | | Project architecture | In progress | -| Python CLI | In progress | -| Encryption | Implemented | -| Chunking | Implemented | -| Manifest / metadata | Implemented | -| Provider abstraction | In progress | -| Telegram provider | Implemented | -| Discord provider | Implemented | -| Async job engine | Implemented | -| **Key management (RMK)** | **Implemented** | -| Resumable transfers | Planned | -| Multi-provider routing | Planned | -| Verification | Planned | -| Repair / recovery | Future | -| Additional providers | Future | +| Python CLI | ✅ Implemented | +| Encryption | ✅ Implemented | +| Chunking | ✅ Implemented | +| Manifest / metadata | ✅ Implemented (JSON) | +| Provider abstraction | ✅ Implemented | +| Telegram provider | ✅ Implemented | +| Discord provider | ✅ Implemented | +| Async job engine | ✅ Implemented | +| Key management (RMK) | ✅ Implemented | +| Multi-provider routing | 🚧 Planned | +| Resumable transfers | 🚧 Planned | +| Integrity verification | 🚧 Planned | +| Repair / recovery | 🚧 Planned | +| Additional providers | 🚧 Planned | Features marked **Planned** or **Future** should not be considered implemented. @@ -678,12 +457,13 @@ The long-term goal is to turn Stashify into a flexible encrypted storage layer t * [x] Core encryption pipeline * [x] Chunking -* [x] SQLite metadata +* [x] JSON metadata * [x] Telegram provider * [x] Discord provider * [x] Async transfer system * [x] **Repository Master Key (RMK) hierarchy** * [ ] Resumable uploads +* [ ] Multi-provider routing ### Medium term diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 1c7467a..277be1f 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -81,9 +81,10 @@ stash put [options] |--------|-------------| | `--provider, -p` | Specific provider to use | | `--chunk-size` | Chunk size in bytes (default: provider limit) | -| `--strategy` | Distribution: single, split, balanced, replicated | | `--confirm/--no-confirm` | Skip confirmation prompt | +> **Note:** Distribution strategy is currently always "single" (all chunks on one provider). Additional strategies (split, balanced, replicated) are planned. + ### `stash get` Retrieve a file from Stash (uses RMK from keyring — no password prompt). diff --git a/docs/configuration.md b/docs/configuration.md index 4f64a16..5632937 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -78,31 +78,7 @@ stash provider add telegram \ | `chat_id` | Chat/channel ID for storage | Yes | - | | `max_concurrent` | Max concurrent uploads | No | `3` | -## Global Settings - -Create `.stash/config.toml` for global defaults: - -```toml -[storage] -default_provider = "telegram" -default_chunk_size = 10485760 # 10MB -replication_factor = 1 - -[transfers] -upload_concurrency = 3 -download_concurrency = 3 -retry_count = 3 -retry_backoff = 1.0 - -[security] -auto_lock_timeout = 0 # 0 = never -key_derivation_iterations = 100000 - -[ui] -compact_mode = false -animations = true -progress_style = "bar" -``` + ## Key Management diff --git a/src/stash/__init__.py b/src/stash/__init__.py index bb06226..372049c 100644 --- a/src/stash/__init__.py +++ b/src/stash/__init__.py @@ -1,5 +1,5 @@ """Stash - Privacy-focused CLI storage system.""" -__version__ = "0.1.0" +__version__ = "0.2.0" __author__ = "Stash Contributors" __license__ = "MIT" \ No newline at end of file From c6a5cbbd1f8e1a3292f3868ad54ec921f0ecc9d6 Mon Sep 17 00:00:00 2001 From: sparklee_op Date: Thu, 20 Aug 2026 18:54:32 +0530 Subject: [PATCH 19/19] docs: remove test_restored.md to eliminate outdated content --- test_restored.md | 242 ----------------------------------------------- 1 file changed, 242 deletions(-) delete mode 100644 test_restored.md diff --git a/test_restored.md b/test_restored.md deleted file mode 100644 index 4bf08a6..0000000 --- a/test_restored.md +++ /dev/null @@ -1,242 +0,0 @@ -# Stash - -**An open-source, privacy-focused command-line storage system that uses third-party platforms as encrypted storage backends.** - ---- - -## What is Stash? - -Stash is a storage abstraction layer. It lets you store files on external services — Telegram, Discord, and eventually S3, Backblaze B2, Google Drive, local filesystem, WebDAV, and others — while keeping all encryption strictly client-side. - -Stash is **not** a cloud storage provider. It is a CLI tool that turns interchangeable storage backends into a unified, encrypted filesystem-like experience. - -``` - Stash CLI - | - Storage abstraction - | - +---------------+---------------+ - | | - Telegram Discord - backend backend - | | - encrypted chunks encrypted chunks -``` - ---- - -## Why Stash? - -| Problem | Stash Approach | -|---------|----------------| -| Vendor lock-in | Provider-agnostic abstraction | -| No client-side encryption in most platforms | Encrypt locally, upload ciphertext only | -| Single point of failure | Distribute chunks across multiple providers | -| Manual chunk/message management | Automatic chunking, upload, metadata, resumption | -| Platform-specific limits | Chunking adapts to each provider's constraints | - ---- - -## Core Architecture - -### Client-Side Encryption - -Files are encrypted **before** they leave your device. - -``` -plaintext → Stash (encrypt + chunk) → ciphertext chunks → providers -``` - -- Providers only ever receive authenticated ciphertext -- Established cryptographic libraries (no custom primitives) -- Per-file encryption keys with proper key derivation -- Authenticated encryption (AEAD) for every chunk -- Optional double-encryption mode (whole-file + per-chunk) - -**Security claims are conservative:** encryption and key management provide confidentiality. Multi-provider distribution provides redundancy and availability — not additional cryptographic security. - -### File Chunking - -Large files are split into chunks to accommodate provider-specific size limits. - -``` -original file - | - v -encrypt → chunk → chunk 0, chunk 1, chunk 2, ... -``` - -Each chunk is independently encrypted and tracked. Metadata (manifest) describes how to reconstruct the file. - -### Multi-Provider Storage - -A single file's chunks can be distributed across providers. - -| Strategy | Description | -|----------|-------------| -| **Single** | All chunks on one provider | -| **Split** | Chunks distributed across providers (0→Telegram, 1→Discord, …) | -| **Balanced** | Dynamic distribution based on availability/performance | -| **Replicated** | Each chunk stored on multiple providers for redundancy | - -``` -100 encrypted chunks - -Telegram: 0, 2, 4, 6, 8, ... -Discord: 1, 3, 5, 7, 9, ... -``` - -Even if an attacker obtains **all chunks from every provider**, the data remains protected by the encryption design. - -### Provider Abstraction - -The core engine knows nothing about Telegram or Discord specifics. - -``` -Storage Provider (interface) - | - +-- Telegram - +-- Discord - +-- S3 / B2 / GDrive / Local / WebDAV (planned) - +-- Future providers -``` - -New providers implement a clean interface. The storage engine remains unchanged. - -### Asynchronous & Resumable - -- Bounded concurrent workers for uploads/downloads -- Retries with exponential backoff -- Provider-aware rate limiting -- Progress reporting -- Cancellation support -- **Resumable operations**: if 75/100 chunks uploaded, interruption resumes at chunk 75 - -``` -Upload Queue - | -+----+----+----+ -| | | | -W1 W2 W3 W4 (bounded workers) -| | | | -TG DC TG S3 (providers) -``` - -### Manifest & Local Metadata - -Each stored file has a manifest containing: - -- File ID, original name, size -- Chunk size, count, encryption params -- Chunk indexes → provider assignments → remote identifiers -- Integrity verification data - -Local metadata stored in SQLite (initially). - ---- - -## Basic Usage - -```bash -# Initialize repository -stash init - -# Add providers -stash provider add telegram -stash provider add discord - -# List configured providers -stash provider list - -# Store a file -stash put ./movie.mkv - -# List stored files -stash ls - -# Show file metadata -stash info movie.mkv - -# Retrieve a file -stash get movie.mkv - -# Remove a file -stash rm movie.mkv - -# Verify integrity (local metadata + remote) -stash verify movie.mkv - -# Repair missing chunks from replicas (planned) -stash repair movie.mkv - -# Overall status -stash status -``` - -The user never manages individual messages, attachments, chunks, or encryption metadata. - ---- - -## Provider Support - -| Provider | Status | Notes | -|----------|--------|-------| -| Telegram | Initial | Bot API, channel/group storage | -| Discord | Initial | Bot/user token, channel attachments | -| S3-compatible | Planned | AWS S3, MinIO, R2, etc. | -| Backblaze B2 | Planned | Native B2 API | -| Google Drive | Planned | OAuth, Drive API | -| Local filesystem | Planned | Directory backend | -| WebDAV | Planned | Generic WebDAV servers | - -**Important:** Stash does not provide "unlimited storage." Actual limits, rate limits, file-size restrictions, and policies depend entirely on each provider. Stash adapts to those constraints via chunking and provider-specific logic. - ---- - -## Security & Privacy Philosophy - -- **Local-first encryption**: Plaintext never leaves your device -- **No custom crypto**: Established libraries (e.g., `cryptography`, `libsodium` bindings) -- **Minimal trust**: Providers are untrusted storage buckets -- **Transparency**: Open source, auditable code paths -- **Provider independence**: No single provider can compromise your data -- **Conservative claims**: We describe what the cryptography actually guarantees - ---- - -## Project Status - -**Early development.** Core architecture, provider abstraction, encryption model, and CLI structure are being designed and implemented. - -| Area | Status | -|------|--------| -| CLI framework | In progress | -| Provider abstraction | In progress | -| Telegram backend | In progress | -| Discord backend | Planned | -| Encryption/chunking | In progress | -| Manifest/metadata | In progress | -| Async job engine | In progress | -| Resumable uploads | Planned | -| Multi-provider strategies | Planned | -| Verification/repair | Future | - -Features described as "planned" or "future" are not yet implemented. This README will be updated as milestones land. - ---- - -## Contributing - -We welcome contributions — especially new storage providers, core improvements, testing, and documentation. - -See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines. - ---- - -## License - -MIT License. See [LICENSE](LICENSE) for details. - ---- - -*Stash: your files, your keys, your choice of storage.* \ No newline at end of file