diff --git a/.docker/Dockerfile b/.docker/Dockerfile index e4c187d..eafab56 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -21,6 +21,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ unixodbc-dev \ && rm -rf /var/lib/apt/lists/* +RUN pip install --no-cache-dir tox + RUN groupadd --gid ${GROUP_ID} dev \ && useradd --uid ${USER_ID} --gid ${GROUP_ID} --create-home --shell /bin/bash dev diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml index aea5667..2ca1c99 100644 --- a/.docker/docker-compose.yml +++ b/.docker/docker-compose.yml @@ -1,3 +1,5 @@ +name: python-db2sql + services: dev: build: @@ -12,6 +14,7 @@ services: - ..:/workspace environment: DEVCONTAINER: "true" + PIPUSER: "0" HOME: /home/dev MSSQL_HOST: mssql MSSQL_PORT: "1433" @@ -28,7 +31,16 @@ services: PG_USER: postgres PG_PASSWORD: ${POSTGRES_PASSWORD:-postgres} PG_DATABASE: db2sqltarget + PG_SOURCE_SCHEMA: apptest + MYSQL_HOST: mysql + MYSQL_PORT: "3306" + MYSQL_USER: root + MYSQL_PASSWORD: ${MYSQL_ROOT_PASSWORD:-mysqlpw} + MYSQL_DATABASE: db2sqltest init: true + tty: true + stdin_open: true + command: ["sleep", "infinity"] mssql: profiles: ["functional"] @@ -38,7 +50,7 @@ services: MSSQL_SA_PASSWORD: ${MSSQL_SA_PASSWORD:-Db2sqlTest!Strong} MSSQL_PID: Developer ports: - - "${MSSQL_PORT:-1433}:1433" + - "${MSSQL_PORT:-11433}:1433" volumes: - mssql-data:/var/opt/mssql healthcheck: @@ -82,7 +94,7 @@ services: APP_USER: ${ORACLE_APP_USER:-apptest} APP_USER_PASSWORD: ${ORACLE_APP_PASSWORD:-apptestpw} ports: - - "${ORACLE_PORT:-1521}:1521" + - "${ORACLE_PORT:-11521}:1521" volumes: - ./oracle/init:/container-entrypoint-initdb.d:ro - oracle-data:/opt/oracle/oradata @@ -93,6 +105,26 @@ services: retries: 60 start_period: 60s + mysql: + profiles: ["functional"] + image: mysql:8.4 + environment: + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-mysqlpw} + MYSQL_DATABASE: db2sqltest + ports: + - "${MYSQL_PORT:-13306}:3306" + volumes: + - ./mysql/init:/docker-entrypoint-initdb.d:ro + - mysql-data:/var/lib/mysql + healthcheck: + test: + - CMD-SHELL + - mysqladmin ping -h 127.0.0.1 -uroot -p"$$MYSQL_ROOT_PASSWORD" --silent + interval: 5s + timeout: 5s + retries: 30 + start_period: 20s + postgres: profiles: ["functional"] image: postgres:16-alpine @@ -101,7 +133,7 @@ services: POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} POSTGRES_DB: db2sqltarget ports: - - "${POSTGRES_PORT:-5432}:5432" + - "${PG_PORT:-15432}:5432" volumes: - ./postgres/init:/docker-entrypoint-initdb.d:ro - postgres-data:/var/lib/postgresql/data @@ -113,5 +145,6 @@ services: volumes: mssql-data: + mysql-data: oracle-data: postgres-data: diff --git a/.docker/mysql/init/01-schema.sql b/.docker/mysql/init/01-schema.sql new file mode 100644 index 0000000..1472caf --- /dev/null +++ b/.docker/mysql/init/01-schema.sql @@ -0,0 +1,90 @@ +-- Functional-test fixture for MySQL. +-- +-- The MySQLSourceReader uses the connection's database as the "schema" +-- (MySQL has no notion of schema separate from database), so all fixtures +-- live in the ``db2sqltest`` database created from MYSQL_DATABASE. +-- +-- Coverage: every MySQL source type referenced by +-- ``PostgresSqlEmitter.DEFAULT_TYPE_MAP`` that is native to MySQL, plus the +-- standard relational fixture (author/book + foreign key + secondary index). + +CREATE DATABASE IF NOT EXISTS db2sqltest; +USE db2sqltest; + +DROP TABLE IF EXISTS book; +DROP TABLE IF EXISTS author; +DROP TABLE IF EXISTS type_matrix; + +-- Type-coverage table ------------------------------------------------------- +CREATE TABLE type_matrix ( + id INT NOT NULL AUTO_INCREMENT, + c_bit BIT(1) NULL, + c_tinyint TINYINT NULL, + c_smallint SMALLINT NULL, + c_mediumint MEDIUMINT NULL, + c_int INT NULL, + c_bigint BIGINT NULL, + c_decimal DECIMAL(12,4) NULL, + c_numeric NUMERIC(18,6) NULL, + c_float FLOAT NULL, + c_double DOUBLE NULL, + c_char CHAR(8) NULL, + c_varchar VARCHAR(64) NULL, + c_text TEXT NULL, + c_mediumtext MEDIUMTEXT NULL, + c_longtext LONGTEXT NULL, + c_binary BINARY(8) NULL, + c_varbinary VARBINARY(64) NULL, + c_blob BLOB NULL, + c_date DATE NULL, + c_time TIME NULL, + c_datetime DATETIME NULL, + c_timestamp TIMESTAMP NULL DEFAULT NULL, + c_json JSON NULL, + PRIMARY KEY (id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +-- Relational mini-fixture (parallels mssql/oracle/postgres fixtures) +CREATE TABLE author ( + id INT NOT NULL AUTO_INCREMENT, + name VARCHAR(120) NOT NULL, + birth_year INT NULL, + PRIMARY KEY (id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE book ( + id INT NOT NULL AUTO_INCREMENT, + author_id INT NOT NULL, + title VARCHAR(200) NOT NULL, + PRIMARY KEY (id), + CONSTRAINT fk_book_author FOREIGN KEY (author_id) REFERENCES author (id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE INDEX idx_book_title ON book (title); + +-- Representative payload (one populated row + one all-NULL row) +INSERT INTO type_matrix + (c_bit, c_tinyint, c_smallint, c_mediumint, c_int, c_bigint, + c_decimal, c_numeric, c_float, c_double, + c_char, c_varchar, c_text, c_mediumtext, c_longtext, + c_binary, c_varbinary, c_blob, + c_date, c_time, c_datetime, c_timestamp, c_json) +VALUES + (b'1', 7, 32100, 8388600, 2147483640, 9000000000000000000, + 1234.5678, 1234567.890123, 1.5, 3.141592653589793, + 'fixed', 'with accent é', 'long text payload', 'medium text', 'long text', + UNHEX('DEADBEEFCAFEBABE'), UNHEX('01020304'), UNHEX('05060708'), + '2024-01-31', '14:30:00', '2024-01-31 14:30:00', + '2024-01-31 14:30:00', JSON_OBJECT('k', 'v')), + (NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL); + +INSERT INTO author (name, birth_year) VALUES ('Alice', 1980); +INSERT INTO author (name, birth_year) VALUES ('Bob', NULL); + +INSERT INTO book (author_id, title) VALUES (1, 'First'); +INSERT INTO book (author_id, title) VALUES (1, 'Second\'s ride'); +INSERT INTO book (author_id, title) VALUES (2, 'Bob book'); diff --git a/.docker/postgres/init/01-schema.sql b/.docker/postgres/init/01-schema.sql index 675b049..853dc2e 100644 --- a/.docker/postgres/init/01-schema.sql +++ b/.docker/postgres/init/01-schema.sql @@ -1,11 +1,93 @@ --- Target PostgreSQL database used by functional tests to validate a generated --- dump. The dump is supposed to be self-sufficient (CREATE SCHEMA + DDL + --- COPY/INSERT), so this init script only ensures the database exists and is --- otherwise empty. +-- Functional-test fixture for PostgreSQL. +-- +-- The "db2sqltarget" database (created from POSTGRES_DB by the entrypoint) is +-- used both as: +-- * a TARGET database for dumps produced by db2sql (apply step) — left +-- untouched outside the apptest schema below; +-- * a SOURCE database whose ``apptest`` schema mirrors the mssql/oracle +-- fixtures (type_matrix + author/book) so the PostgreSQL reader can be +-- exercised by the functional suite. +-- +-- The reader filters out pg_catalog/information_schema/pg_toast, so the +-- ``apptest`` schema is the only one surfaced by collect_metadata(). +-- Note: types covered are the PG-native ones referenced by +-- ``PostgresSqlEmitter.DEFAULT_TYPE_MAP`` (no MSSQL/Oracle-only types). --- The database "db2sqltarget" is created from POSTGRES_DB by the entrypoint, --- so nothing else is strictly required here. We keep the file so the volume --- mount /docker-entrypoint-initdb.d is non-empty and to host any future --- preconditioning (extensions, roles, etc.). +CREATE SCHEMA IF NOT EXISTS apptest; -SELECT 'postgres target ready' AS status; +DROP TABLE IF EXISTS apptest.book; +DROP TABLE IF EXISTS apptest.author; +DROP TABLE IF EXISTS apptest.type_matrix; + +-- Type-coverage table ------------------------------------------------------- +CREATE TABLE apptest.type_matrix ( + id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + c_boolean BOOLEAN, + c_smallint SMALLINT, + c_integer INTEGER, + c_bigint BIGINT, + c_real REAL, + c_double DOUBLE PRECISION, + c_numeric NUMERIC(18, 6), + c_decimal DECIMAL(12, 4), + c_char CHAR(8), + c_varchar VARCHAR(64), + c_text TEXT, + c_bytea BYTEA, + c_date DATE, + c_time TIME, + c_timestamp TIMESTAMP, + c_timestamptz TIMESTAMP WITH TIME ZONE, + c_uuid UUID, + c_json JSON, + c_jsonb JSONB, + c_xml XML +); + +-- Relational mini-fixture (parallels mssql/oracle/sqlite fixtures) +CREATE TABLE apptest.author ( + id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name VARCHAR(120) NOT NULL, + birth_year INTEGER +); + +CREATE TABLE apptest.book ( + id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + author_id INTEGER NOT NULL, + title VARCHAR(200) NOT NULL, + CONSTRAINT fk_book_author FOREIGN KEY (author_id) REFERENCES apptest.author (id) +); + +CREATE INDEX idx_book_title ON apptest.book (title); + +-- Representative payload (one populated row + one all-NULL row) +INSERT INTO apptest.type_matrix + (c_boolean, c_smallint, c_integer, c_bigint, + c_real, c_double, c_numeric, c_decimal, + c_char, c_varchar, c_text, c_bytea, + c_date, c_time, c_timestamp, c_timestamptz, + c_uuid, c_json, c_jsonb, c_xml) +VALUES + (TRUE, 32100, 2147483640, 9000000000000000000, + 1.5, 3.141592653589793, 1234567.890123, 1234.5678, + 'fixed', 'with accent é', 'long text payload', '\xDEADBEEFCAFEBABE', + DATE '2024-01-31', TIME '14:30:00', + TIMESTAMP '2024-01-31 14:30:00', + TIMESTAMP WITH TIME ZONE '2024-01-31 14:30:00+01', + '11111111-2222-3333-4444-555555555555', + '{"k": "v"}', '{"k": "v"}', + XMLPARSE(DOCUMENT 'v')), + (NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL); + +INSERT INTO apptest.author (name, birth_year) VALUES ('Alice', 1980); +INSERT INTO apptest.author (name, birth_year) VALUES ('Bob', NULL); + +INSERT INTO apptest.book (author_id, title) VALUES (1, 'First'); +INSERT INTO apptest.book (author_id, title) VALUES (1, 'Second''s ride'); +INSERT INTO apptest.book (author_id, title) VALUES (2, 'Bob book'); + +SELECT 'postgres apptest fixture loaded' AS status; diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd64281..7de7977 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,7 +75,7 @@ jobs: run: sphinx-build -b html -W --keep-going docs docs/_build/html functional: - name: Functional tests (MSSQL + Postgres, no Oracle) + name: Functional tests (MSSQL + MySQL + Postgres, no Oracle) runs-on: ubuntu-latest # Light stack (~2-3 min boot). Runs on every push to main, and on PRs # carrying the `run-functional` label. Oracle is on-demand only — see @@ -98,6 +98,12 @@ jobs: POSTGRES_PASSWORD: postgres PG_PASSWORD: postgres PG_DATABASE: db2sqltarget + MYSQL_HOST: localhost + MYSQL_PORT: "3306" + MYSQL_USER: root + MYSQL_ROOT_PASSWORD: mysqlpw + MYSQL_PASSWORD: mysqlpw + MYSQL_DATABASE: db2sqltest steps: - uses: actions/checkout@v4 @@ -112,8 +118,11 @@ jobs: - name: Install package and test dependencies run: pip install -e ".[all]" pytest - - name: Start docker stack (mssql + postgres) and wait until healthy - run: docker compose -f .docker/docker-compose.yml --profile functional up -d --wait + - name: Start docker stack (mssql + mysql + postgres) and wait until healthy + run: docker compose -f .docker/docker-compose.yml --profile functional up -d --wait mssql mysql postgres + + - name: Apply MSSQL init schema + run: docker compose -f .docker/docker-compose.yml --profile functional run --rm mssql-init - name: Run functional tests (excluding Oracle) run: pytest -m "functional and not oracle" tests/functional -v diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml index ec949be..318f30d 100644 --- a/.github/workflows/release-binaries.yml +++ b/.github/workflows/release-binaries.yml @@ -39,19 +39,16 @@ jobs: - label: windows-x86_64 os: windows-latest python: "3.12" - shell: pwsh archive-glob: "installer/dist/db2sql-*-windows-*.zip" container: "" - label: linux-x86_64 os: ubuntu-latest python: "3.12" - shell: bash archive-glob: "installer/dist/db2sql-*-linux-*.tar.gz" container: "python:3.12-bookworm" - label: macos-arm64 os: macos-14 # Apple Silicon runner python: "3.12" - shell: bash archive-glob: "installer/dist/db2sql-*-macos-*.tar.gz" container: "" @@ -79,7 +76,6 @@ jobs: rm -rf /var/lib/apt/lists/* - name: Show toolchain info - shell: ${{ matrix.shell }} run: | python --version python -c "import platform; print('platform:', platform.platform()); print('machine:', platform.machine())" @@ -89,14 +85,12 @@ jobs: run: ldd --version | head -1 - name: Install build dependencies - shell: ${{ matrix.shell }} run: | python -m pip install --upgrade pip wheel python -m pip install -e ".[all]" python -m pip install "pyinstaller>=6" - name: Build standalone binary - shell: ${{ matrix.shell }} run: python installer/build.py --archive - name: Upload build artifact diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index dd8f9ff..6880702 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -229,7 +229,7 @@ You need to install ``tox`` using one of the following approach: Running integration tests ------------------------- -Integration tests run against a local symbol store. +Integration tests run against a local symbol store. To run these tests: @@ -239,6 +239,51 @@ To run these tests: tox -e cli +Running functional tests +------------------------ + +Functional tests exercise the source readers (and emitters) against real +database servers. A docker compose stack under ``.docker/`` brings up the +required services (MSSQL, MySQL, PostgreSQL, and optionally Oracle) and +applies the per-DB init scripts in ``.docker//init/``. + +The ``functional`` docker compose profile covers MSSQL, MySQL and Postgres +(lightweight images). Oracle lives in its own ``oracle`` profile because the +image is ~10 GB and takes 3–5 min to boot. + +Make targets: + +.. code-block:: bash + + # full stack (mssql + mysql + postgres + oracle), then run all functional + # tests inside the dev container + make stack-up + make test-functional + + # lightweight subset — no Oracle (fast iteration) + make stack-up-light + make test-functional-light + + # Oracle-only (heavy, on-demand) + make stack-up-oracle + make test-oracle + + # tear down (keep volumes / wipe volumes) + make stack-down + make stack-reset + +The connection parameters consumed by the test fixtures are injected into +the dev container by ``.docker/docker-compose.yml`` (``MSSQL_*``, ``MYSQL_*``, +``PG_*``, ``ORACLE_*``). When a server is unreachable or its env vars are +missing, the corresponding fixture skips the test with a clear message +instead of failing — so the suite stays green when run outside the stack. + +Adding a new functional fixture: drop an init script under +``.docker//init/`` (executed automatically by the official images) and +mirror the existing tests in ``tests/functional/`` (one test module per +source DB, ``pytestmark = pytest.mark.functional``). + + Releases -------- diff --git a/Makefile b/Makefile index bb4955f..c472a8f 100644 --- a/Makefile +++ b/Makefile @@ -44,14 +44,16 @@ docker-clean: ## Remove the docker dev image and associated resources # `oracle` profile because the image is ~10 GB and takes 3-5 min to boot. # `stack-up` keeps the historical "everything" behaviour for local dev. -stack-up: ## Start the full functional DB stack (mssql, postgres, oracle) and wait until healthy - $(DOCKER_ENV) $(DOCKER_COMPOSE) --profile functional --profile oracle up -d --wait +stack-up: ## Start the full functional DB stack (mssql, mysql, postgres, oracle) and wait until healthy + $(DOCKER_ENV) $(DOCKER_COMPOSE) --profile functional --profile oracle up -d --wait mssql mysql postgres oracle + $(DOCKER_ENV) $(DOCKER_COMPOSE) --profile functional run --rm mssql-init -stack-up-light: ## Start only the lightweight functional DB stack (mssql, postgres) — no Oracle - $(DOCKER_ENV) $(DOCKER_COMPOSE) --profile functional up -d --wait +stack-up-light: ## Start only the lightweight functional DB stack (mssql, mysql, postgres) — no Oracle + $(DOCKER_ENV) $(DOCKER_COMPOSE) --profile functional up -d --wait mssql mysql postgres + $(DOCKER_ENV) $(DOCKER_COMPOSE) --profile functional run --rm mssql-init stack-up-oracle: ## Start only the Oracle container (heavy, ~3-5 min boot) — no mssql/postgres - $(DOCKER_ENV) $(DOCKER_COMPOSE) --profile oracle up -d --wait + $(DOCKER_ENV) $(DOCKER_COMPOSE) --profile oracle up -d --wait oracle stack-down: ## Stop the functional DB stack (keep volumes) $(DOCKER_COMPOSE) --profile functional --profile oracle down diff --git a/db2sql/application/dto/dump_request.py b/db2sql/application/dto/dump_request.py index 2b82ac5..22ebe6d 100644 --- a/db2sql/application/dto/dump_request.py +++ b/db2sql/application/dto/dump_request.py @@ -8,7 +8,7 @@ from dataclasses import dataclass, field from enum import Enum -from typing import Dict, FrozenSet, Mapping, Optional, Tuple +from typing import Mapping, Optional, Tuple from db2sql.domain.policy import FilterRules diff --git a/db2sql/application/ports/source_reader.py b/db2sql/application/ports/source_reader.py index 3f63029..aada7e1 100644 --- a/db2sql/application/ports/source_reader.py +++ b/db2sql/application/ports/source_reader.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Any, Iterable, Iterator, List, Protocol, Tuple +from typing import Any, Iterator, List, Protocol, Tuple from db2sql.domain.model import Column, Database, Table diff --git a/db2sql/application/use_cases/dump_database.py b/db2sql/application/use_cases/dump_database.py index 2a2b4c6..3f19964 100644 --- a/db2sql/application/use_cases/dump_database.py +++ b/db2sql/application/use_cases/dump_database.py @@ -39,9 +39,7 @@ def execute(self) -> Database: database = self._reader.collect_metadata() database = filter_database(database, self._request.filter_rules) if self._request.views: - self._logger.info( - f"materializing {len(self._request.views)} view export(s)" - ) + self._logger.info(f"materializing {len(self._request.views)} view export(s)") materialize_views(self._reader, database, self._request.views) self._emit(database) return database @@ -68,9 +66,7 @@ def _emit(self, database: Database) -> None: def _emit_data(self, database: Database) -> None: options = self._request.options - view_options = { - (v.target_schema, v.target_table): v for v in self._request.views - } + view_options = {(v.target_schema, v.target_table): v for v in self._request.views} for schema_name, schema in database.schemas.items(): for table_name, table in schema.tables.items(): view = view_options.get((schema_name, table_name)) @@ -78,9 +74,7 @@ def _emit_data(self, database: Database) -> None: limit = self._resolve_limit(view, options, schema_name, table_name) if limit == 0: continue - self._logger.info( - f"dumping rows from {schema_name}.{table_name} ({fmt.value})" - ) + self._logger.info(f"dumping rows from {schema_name}.{table_name} ({fmt.value})") if table.source_query is not None: rows = self._reader.iter_query_rows(table.source_query, limit=limit) else: diff --git a/db2sql/application/use_cases/materialize_views.py b/db2sql/application/use_cases/materialize_views.py index c6db235..85e48d0 100644 --- a/db2sql/application/use_cases/materialize_views.py +++ b/db2sql/application/use_cases/materialize_views.py @@ -68,8 +68,6 @@ def _apply_primary_key(columns: List[Column], primary_key: Iterable[str]) -> Non for name in pk: column = by_name.get(name) if column is None: - raise ValueError( - f"primary_key references unknown column '{name}' in view export" - ) + raise ValueError(f"primary_key references unknown column '{name}' in view export") column.constraint = "PRIMARY KEY" column.nullable = False diff --git a/db2sql/application/use_cases/migrate_database.py b/db2sql/application/use_cases/migrate_database.py index 6b52f50..3f25590 100644 --- a/db2sql/application/use_cases/migrate_database.py +++ b/db2sql/application/use_cases/migrate_database.py @@ -57,9 +57,7 @@ def execute(self) -> Database: database = self._reader.collect_metadata() database = filter_database(database, self._request.filter_rules) if self._request.views: - self._logger.info( - f"materializing {len(self._request.views)} view export(s)" - ) + self._logger.info(f"materializing {len(self._request.views)} view export(s)") materialize_views(self._reader, database, self._request.views) self._emit_and_load(database) return database @@ -84,18 +82,14 @@ def _emit_and_load(self, database: Database) -> None: def _load_data(self, database: Database) -> None: options = self._request.options - view_options = { - (v.target_schema, v.target_table): v for v in self._request.views - } + view_options = {(v.target_schema, v.target_table): v for v in self._request.views} for schema_name, schema in database.schemas.items(): for table_name, table in schema.tables.items(): view = view_options.get((schema_name, table_name)) limit = self._resolve_limit(view, options, schema_name, table_name) if limit == 0: continue - self._logger.info( - f"bulk-loading rows into {schema_name}.{table_name}" - ) + self._logger.info(f"bulk-loading rows into {schema_name}.{table_name}") if table.source_query is not None: rows = self._reader.iter_query_rows(table.source_query, limit=limit) else: diff --git a/db2sql/domain/__init__.py b/db2sql/domain/__init__.py index 4eb3c2b..2d5e61e 100644 --- a/db2sql/domain/__init__.py +++ b/db2sql/domain/__init__.py @@ -8,7 +8,7 @@ DuplicatedTableError, ) from .model import Column, Database, ForeignKey, Schema, Table -from .policy import FilterRules, filter_database, normalize_identifier, to_snake_case +from .policy import filter_database, FilterRules, normalize_identifier, to_snake_case __all__ = [ "Column", diff --git a/db2sql/domain/model/column.py b/db2sql/domain/model/column.py index f511a30..7172e28 100644 --- a/db2sql/domain/model/column.py +++ b/db2sql/domain/model/column.py @@ -2,7 +2,7 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Optional from .foreign_key import ForeignKey diff --git a/db2sql/domain/policy/__init__.py b/db2sql/domain/policy/__init__.py index 87b0467..f067f88 100644 --- a/db2sql/domain/policy/__init__.py +++ b/db2sql/domain/policy/__init__.py @@ -1,7 +1,7 @@ """Pure domain policies (identifier normalization, filtering rules).""" from .dependency_order import drop_order, topological_order -from .filter import FilterRules, filter_database +from .filter import filter_database, FilterRules from .identifier import normalize_identifier, to_snake_case __all__ = [ diff --git a/db2sql/infrastructure/config/errors.py b/db2sql/infrastructure/config/errors.py index 99d5e52..5d5dba2 100644 --- a/db2sql/infrastructure/config/errors.py +++ b/db2sql/infrastructure/config/errors.py @@ -2,8 +2,8 @@ from __future__ import annotations -from typing import Union from pathlib import Path +from typing import Union class ConfigError(Exception): diff --git a/db2sql/infrastructure/config/loader.py b/db2sql/infrastructure/config/loader.py index 24e3350..86b8579 100644 --- a/db2sql/infrastructure/config/loader.py +++ b/db2sql/infrastructure/config/loader.py @@ -12,7 +12,11 @@ from db2sql import const -from .errors import ConfigInvalidError, ConfigMissingError, ConfigUnsupportedFileExtensionError +from .errors import ( + ConfigInvalidError, + ConfigMissingError, + ConfigUnsupportedFileExtensionError, +) from .schema import AppConfig PathLike = Union[str, Path] diff --git a/db2sql/infrastructure/config/schema.py b/db2sql/infrastructure/config/schema.py index 1e6ea57..40260a8 100644 --- a/db2sql/infrastructure/config/schema.py +++ b/db2sql/infrastructure/config/schema.py @@ -88,9 +88,7 @@ def _validate_on_existing(cls, value: Any) -> Any: if value is None: return "fail" if value not in ("fail", "drop", "truncate"): - raise ValueError( - "dump.on_existing must be one of 'fail', 'drop', 'truncate'" - ) + raise ValueError("dump.on_existing must be one of 'fail', 'drop', 'truncate'") return value @field_validator( @@ -132,9 +130,7 @@ def _validate_on_existing(cls, value: Any) -> Any: if value is None: return "fail" if value not in ("fail", "drop", "truncate"): - raise ValueError( - "migrate.on_existing must be one of 'fail', 'drop', 'truncate'" - ) + raise ValueError("migrate.on_existing must be one of 'fail', 'drop', 'truncate'") return value diff --git a/db2sql/infrastructure/logging/__init__.py b/db2sql/infrastructure/logging/__init__.py index 6e10ad6..6a5882a 100644 --- a/db2sql/infrastructure/logging/__init__.py +++ b/db2sql/infrastructure/logging/__init__.py @@ -1,7 +1,9 @@ """Logging adapter — ConsoleLogger implements the application Logger port.""" -from .colors import Palette, color_enabled, init_colorama, is_terminal +from .colors import color_enabled, init_colorama, is_terminal, Palette from .console_logger import ( + ConsoleLogger, + InvalidLogLevel, LEVEL_DEBUG, LEVEL_ERROR, LEVEL_NOTICE, @@ -10,8 +12,6 @@ LEVEL_TRACE, LEVEL_VERBOSE, LEVEL_WARNING, - ConsoleLogger, - InvalidLogLevel, ) __all__ = [ diff --git a/db2sql/infrastructure/logging/console_logger.py b/db2sql/infrastructure/logging/console_logger.py index 434b620..97859ce 100644 --- a/db2sql/infrastructure/logging/console_logger.py +++ b/db2sql/infrastructure/logging/console_logger.py @@ -6,7 +6,7 @@ import traceback from typing import IO, Optional -from .colors import RESET, Palette, color_enabled +from .colors import color_enabled, Palette, RESET LEVEL_QUIET = 80 LEVEL_ERROR = 70 @@ -66,7 +66,9 @@ def from_verbosity( raise InvalidLogLevel(level_name) from exc stream: IO[str] if log_file: - stream = open(log_file, "wt", encoding="utf-8") # noqa: SIM115 — owned for process lifetime + stream = open( + log_file, "wt", encoding="utf-8" + ) # noqa: SIM115 — owned for process lifetime else: stream = sys.stdout return cls(level=level, stream=stream) diff --git a/db2sql/infrastructure/output/rotating_file_sink.py b/db2sql/infrastructure/output/rotating_file_sink.py index e069fd6..42accde 100644 --- a/db2sql/infrastructure/output/rotating_file_sink.py +++ b/db2sql/infrastructure/output/rotating_file_sink.py @@ -68,10 +68,7 @@ def boundary(self) -> None: if not self._buffer: return buffered_size = sum(len(s.encode("utf-8")) for s in self._buffer) - if ( - self._current_size > 0 - and self._current_size + buffered_size > self._max_bytes - ): + if self._current_size > 0 and self._current_size + buffered_size > self._max_bytes: self._rotate() self._flush_buffer() diff --git a/db2sql/infrastructure/output/stream_sink.py b/db2sql/infrastructure/output/stream_sink.py index 3952715..1860c58 100644 --- a/db2sql/infrastructure/output/stream_sink.py +++ b/db2sql/infrastructure/output/stream_sink.py @@ -18,7 +18,9 @@ def __init__(self, path: Optional[str] = None) -> None: def __enter__(self) -> "StreamSink": if self._path: - self._stream = open(self._path, "w", encoding="utf-8") # noqa: SIM115 — closed in __exit__ + self._stream = open( + self._path, "w", encoding="utf-8" + ) # noqa: SIM115 — closed in __exit__ self._owns_stream = True else: self._stream = sys.stdout diff --git a/db2sql/infrastructure/persistence/mssql/reader.py b/db2sql/infrastructure/persistence/mssql/reader.py index 6d1f685..b43056a 100644 --- a/db2sql/infrastructure/persistence/mssql/reader.py +++ b/db2sql/infrastructure/persistence/mssql/reader.py @@ -67,9 +67,7 @@ def collect_metadata(self) -> Database: raise SourceReaderError("failed to collect database information") from exception return database - def iter_rows( - self, schema: str, table: Table, limit: int = -1 - ) -> Iterator[Tuple[Any, ...]]: + def iter_rows(self, schema: str, table: Table, limit: int = -1) -> Iterator[Tuple[Any, ...]]: session = self._ensure_session() columns = ", ".join(f"[{name}]" for name in table.columns) top = f"TOP {limit} " if limit and limit > 0 else "" @@ -81,12 +79,8 @@ def iter_rows( def describe_query(self, query: str) -> List[Column]: return query_introspection.describe_query(self._ensure_session(), query) - def iter_query_rows( - self, query: str, limit: int = -1 - ) -> Iterator[Tuple[Any, ...]]: - yield from query_introspection.iter_query_rows( - self._ensure_session(), query, limit=limit - ) + def iter_query_rows(self, query: str, limit: int = -1) -> Iterator[Tuple[Any, ...]]: + yield from query_introspection.iter_query_rows(self._ensure_session(), query, limit=limit) def _read_schemas(self, database: Database) -> None: try: diff --git a/db2sql/infrastructure/persistence/mysql/reader.py b/db2sql/infrastructure/persistence/mysql/reader.py index bbabd7d..015ffc2 100644 --- a/db2sql/infrastructure/persistence/mysql/reader.py +++ b/db2sql/infrastructure/persistence/mysql/reader.py @@ -163,9 +163,7 @@ def _read_indexes(self, database: Database) -> None: if table: table.add_index(row.INDEX_NAME, row.COLUMN_NAME) - def iter_rows( - self, schema: str, table: Table, limit: int = -1 - ) -> Iterator[Tuple[Any, ...]]: + def iter_rows(self, schema: str, table: Table, limit: int = -1) -> Iterator[Tuple[Any, ...]]: session = self._ensure_session() columns = ", ".join(f"`{name}`" for name in table.columns) suffix = f" LIMIT {limit}" if limit and limit > 0 else "" @@ -177,9 +175,5 @@ def iter_rows( def describe_query(self, query: str) -> List[Column]: return query_introspection.describe_query(self._ensure_session(), query) - def iter_query_rows( - self, query: str, limit: int = -1 - ) -> Iterator[Tuple[Any, ...]]: - yield from query_introspection.iter_query_rows( - self._ensure_session(), query, limit=limit - ) + def iter_query_rows(self, query: str, limit: int = -1) -> Iterator[Tuple[Any, ...]]: + yield from query_introspection.iter_query_rows(self._ensure_session(), query, limit=limit) diff --git a/db2sql/infrastructure/persistence/oracle/reader.py b/db2sql/infrastructure/persistence/oracle/reader.py index 5010676..0c45736 100644 --- a/db2sql/infrastructure/persistence/oracle/reader.py +++ b/db2sql/infrastructure/persistence/oracle/reader.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Any, Iterator, List, Optional, Tuple +from typing import Any, Dict, Iterator, List, Optional, Tuple from sqlalchemy import create_engine, engine, text from sqlalchemy.orm.session import Session, sessionmaker @@ -13,6 +13,7 @@ from db2sql.infrastructure.persistence import query_introspection from db2sql.infrastructure.persistence.errors import SourceReaderError + def _normalize_oracle_type(raw: str) -> str: """Normalize Oracle data-type strings so the postgres emitter can map them. @@ -147,12 +148,9 @@ def collect_metadata(self) -> Database: def _read_schemas(self, database: Database) -> None: owner = self._schema_filter - params: dict = {} + params: Dict[str, Any] = {} if owner: - query = ( - "SELECT DISTINCT OWNER FROM ALL_TABLES " - "WHERE OWNER = :owner ORDER BY OWNER" - ) + query = "SELECT DISTINCT OWNER FROM ALL_TABLES " "WHERE OWNER = :owner ORDER BY OWNER" params["owner"] = owner else: query = ( @@ -165,11 +163,11 @@ def _read_schemas(self, database: Database) -> None: except Exception as exc: raise SourceReaderError(f"Error connecting to database {exc}") from exc for row in rows: - database.add_schema(Schema(row.OWNER)) + database.add_schema(Schema(row.owner)) def _read_tables(self, database: Database) -> None: owner = self._schema_filter - params: dict = {} + params: Dict[str, Any] = {} clauses = ["t.IOT_NAME IS NULL"] if owner: clauses.append("t.OWNER = :owner") @@ -187,11 +185,11 @@ def _read_tables(self, database: Database) -> None: params, ) for row in rows: - database.add_table(row.OWNER, Table(row.TABLE_NAME)) + database.add_table(row.owner, Table(row.table_name)) def _read_columns(self, database: Database) -> None: owner = self._schema_filter - params: dict = {} + params: Dict[str, Any] = {} if owner: owner_clause = "c.OWNER = :owner" params["owner"] = owner @@ -209,31 +207,31 @@ def _read_columns(self, database: Database) -> None: params, ) for row in rows: - table = database.get_table(row.OWNER, row.TABLE_NAME) + table = database.get_table(row.owner, row.table_name) if table is None: continue - default = row.DATA_DEFAULT + default = row.data_default if isinstance(default, str): default = default.strip().rstrip(";").strip() or None - data_type = (row.DATA_TYPE or "").lower() - char_length = row.CHAR_LENGTH if "char" in data_type else -1 + data_type = (row.data_type or "").lower() + char_length = row.char_length if "char" in data_type else -1 if not char_length: char_length = -1 table.add_column( Column( - name=row.COLUMN_NAME, - type=_normalize_oracle_type(row.DATA_TYPE), + name=row.column_name, + type=_normalize_oracle_type(row.data_type), default=default, - nullable=row.NULLABLE == "Y", + nullable=row.nullable == "Y", char_length=char_length, - precision=row.DATA_PRECISION, - scale=row.DATA_SCALE, + precision=row.data_precision, + scale=row.data_scale, ) ) def _read_constraints(self, database: Database) -> None: owner = self._schema_filter - params: dict = {} + params: Dict[str, Any] = {} if owner: owner_clause = "c.OWNER = :owner" params["owner"] = owner @@ -256,16 +254,16 @@ def _read_constraints(self, database: Database) -> None: params, ) for row in rows: - table = database.get_table(row.OWNER, row.TABLE_NAME) + table = database.get_table(row.owner, row.table_name) if table is None: continue - column = table.get_column(row.COLUMN_NAME) + column = table.get_column(row.column_name) if column is not None: - column.constraint = row.CONSTRAINT_TYPE + column.constraint = row.constraint_type def _read_foreign_keys(self, database: Database) -> None: owner = self._schema_filter - params: dict = {} + params: Dict[str, Any] = {} if owner: owner_clause = "c.OWNER = :owner" params["owner"] = owner @@ -291,17 +289,17 @@ def _read_foreign_keys(self, database: Database) -> None: params, ) for row in rows: - table = database.get_table(row.OWNER, row.TABLE_NAME) + table = database.get_table(row.owner, row.table_name) if table is None: continue - column = table.get_column(row.COLUMN_NAME) + column = table.get_column(row.column_name) if column is None: continue - column.foreign_key = ForeignKey(row.REF_OWNER, row.REF_TABLE, row.REF_COLUMN) + column.foreign_key = ForeignKey(row.ref_owner, row.ref_table, row.ref_column) def _read_indexes(self, database: Database) -> None: owner = self._schema_filter - params: dict = {} + params: Dict[str, Any] = {} if owner: owner_clause = "i.TABLE_OWNER = :owner" params["owner"] = owner @@ -322,14 +320,14 @@ def _read_indexes(self, database: Database) -> None: params, ) for row in rows: - table = database.get_table(row.TABLE_OWNER, row.TABLE_NAME) + table = database.get_table(row.table_owner, row.table_name) if table is not None: - table.add_index(row.INDEX_NAME, row.COLUMN_NAME) + table.add_index(row.index_name, row.column_name) def _read_identity_columns(self, database: Database) -> None: """Detect 12c+ identity columns. Silently skipped on older Oracle versions.""" owner = self._schema_filter - params: dict = {} + params: Dict[str, Any] = {} if owner: owner_clause = "OWNER = :owner" params["owner"] = owner @@ -347,16 +345,14 @@ def _read_identity_columns(self, database: Database) -> None: except Exception: return for row in rows: - table = database.get_table(row.OWNER, row.TABLE_NAME) + table = database.get_table(row.owner, row.table_name) if table is None: continue - column = table.get_column(row.COLUMN_NAME) + column = table.get_column(row.column_name) if column is not None: column.identity = True - def iter_rows( - self, schema: str, table: Table, limit: int = -1 - ) -> Iterator[Tuple[Any, ...]]: + def iter_rows(self, schema: str, table: Table, limit: int = -1) -> Iterator[Tuple[Any, ...]]: session = self._ensure_session() columns = ", ".join(f'"{name}"' for name in table.columns) query = f'SELECT {columns} FROM "{schema}"."{table.name}"' @@ -369,9 +365,5 @@ def iter_rows( def describe_query(self, query: str) -> List[Column]: return query_introspection.describe_query(self._ensure_session(), query) - def iter_query_rows( - self, query: str, limit: int = -1 - ) -> Iterator[Tuple[Any, ...]]: - yield from query_introspection.iter_query_rows( - self._ensure_session(), query, limit=limit - ) + def iter_query_rows(self, query: str, limit: int = -1) -> Iterator[Tuple[Any, ...]]: + yield from query_introspection.iter_query_rows(self._ensure_session(), query, limit=limit) diff --git a/db2sql/infrastructure/persistence/postgres/reader.py b/db2sql/infrastructure/persistence/postgres/reader.py index aba6253..3672c23 100644 --- a/db2sql/infrastructure/persistence/postgres/reader.py +++ b/db2sql/infrastructure/persistence/postgres/reader.py @@ -69,9 +69,9 @@ def _read_schemas_and_tables(self, database: Database) -> None: ) ) for row in rows: - if row.TABLE_SCHEMA not in database.schemas: - database.add_schema(Schema(row.TABLE_SCHEMA)) - database.add_table(row.TABLE_SCHEMA, Table(row.TABLE_NAME)) + if row.table_schema not in database.schemas: + database.add_schema(Schema(row.table_schema)) + database.add_table(row.table_schema, Table(row.table_name)) def _read_columns(self, database: Database) -> None: rows = self._ensure_session().execute( @@ -85,19 +85,19 @@ def _read_columns(self, database: Database) -> None: ) ) for row in rows: - table = database.get_table(row.TABLE_SCHEMA, row.TABLE_NAME) + table = database.get_table(row.table_schema, row.table_name) if table is None: continue column = Column( - name=row.COLUMN_NAME, - type=row.DATA_TYPE, - default=row.COLUMN_DEFAULT, - nullable=row.IS_NULLABLE == "YES", - char_length=row.CHARACTER_MAXIMUM_LENGTH or -1, - precision=row.NUMERIC_PRECISION, - scale=row.NUMERIC_SCALE, + name=row.column_name, + type=row.data_type, + default=row.column_default, + nullable=row.is_nullable == "YES", + char_length=row.character_maximum_length or -1, + precision=row.numeric_precision, + scale=row.numeric_scale, ) - if row.IS_IDENTITY == "YES": + if row.is_identity == "YES": column.identity = True table.add_column(column) @@ -114,11 +114,11 @@ def _read_constraints(self, database: Database) -> None: ) ) for row in rows: - table = database.get_table(row.TABLE_SCHEMA, row.TABLE_NAME) + table = database.get_table(row.table_schema, row.table_name) if table: - column = table.get_column(row.COLUMN_NAME) + column = table.get_column(row.column_name) if column: - column.constraint = row.CONSTRAINT_TYPE + column.constraint = row.constraint_type def _read_foreign_keys(self, database: Database) -> None: rows = self._ensure_session().execute( @@ -138,13 +138,13 @@ def _read_foreign_keys(self, database: Database) -> None: ) ) for row in rows: - table = database.get_table(row.TABLE_SCHEMA, row.TABLE_NAME) + table = database.get_table(row.table_schema, row.table_name) if table is None: continue - column = table.get_column(row.COLUMN_NAME) + column = table.get_column(row.column_name) if column is None: continue - column.foreign_key = ForeignKey(row.REF_SCHEMA, row.REF_TABLE, row.REF_COLUMN) + column.foreign_key = ForeignKey(row.ref_schema, row.ref_table, row.ref_column) def _read_indexes(self, database: Database) -> None: rows = self._ensure_session().execute( @@ -167,9 +167,7 @@ def _read_indexes(self, database: Database) -> None: if table: table.add_index(row.index_name, row.column_name) - def iter_rows( - self, schema: str, table: Table, limit: int = -1 - ) -> Iterator[Tuple[Any, ...]]: + def iter_rows(self, schema: str, table: Table, limit: int = -1) -> Iterator[Tuple[Any, ...]]: session = self._ensure_session() columns = ", ".join(f'"{name}"' for name in table.columns) suffix = f" LIMIT {limit}" if limit and limit > 0 else "" @@ -181,9 +179,5 @@ def iter_rows( def describe_query(self, query: str) -> List[Column]: return query_introspection.describe_query(self._ensure_session(), query) - def iter_query_rows( - self, query: str, limit: int = -1 - ) -> Iterator[Tuple[Any, ...]]: - yield from query_introspection.iter_query_rows( - self._ensure_session(), query, limit=limit - ) + def iter_query_rows(self, query: str, limit: int = -1) -> Iterator[Tuple[Any, ...]]: + yield from query_introspection.iter_query_rows(self._ensure_session(), query, limit=limit) diff --git a/db2sql/infrastructure/persistence/query_introspection.py b/db2sql/infrastructure/persistence/query_introspection.py index 83e3eaf..673a49a 100644 --- a/db2sql/infrastructure/persistence/query_introspection.py +++ b/db2sql/infrastructure/persistence/query_introspection.py @@ -63,9 +63,7 @@ def describe_query(session: Session, query: str) -> List[Column]: return columns -def iter_query_rows( - session: Session, query: str, limit: int = -1 -) -> Iterator[Tuple[Any, ...]]: +def iter_query_rows(session: Session, query: str, limit: int = -1) -> Iterator[Tuple[Any, ...]]: """Execute ``query`` and yield rows, stopping after ``limit`` if positive.""" result: Result[Any] = session.execute(text(query)) try: diff --git a/db2sql/infrastructure/persistence/sqlite/reader.py b/db2sql/infrastructure/persistence/sqlite/reader.py index 328f800..3a19fe8 100644 --- a/db2sql/infrastructure/persistence/sqlite/reader.py +++ b/db2sql/infrastructure/persistence/sqlite/reader.py @@ -124,9 +124,7 @@ def _read_foreign_keys(self, database: Database, table_name: str) -> None: continue column.foreign_key = ForeignKey(self._schema, ref_table, ref_col) - def iter_rows( - self, schema: str, table: Table, limit: int = -1 - ) -> Iterator[Tuple[Any, ...]]: + def iter_rows(self, schema: str, table: Table, limit: int = -1) -> Iterator[Tuple[Any, ...]]: session = self._ensure_session() columns = ", ".join(f'"{name}"' for name in table.columns) suffix = f" LIMIT {limit}" if limit and limit > 0 else "" @@ -139,9 +137,5 @@ def iter_rows( def describe_query(self, query: str) -> List[Column]: return query_introspection.describe_query(self._ensure_session(), query) - def iter_query_rows( - self, query: str, limit: int = -1 - ) -> Iterator[Tuple[Any, ...]]: - yield from query_introspection.iter_query_rows( - self._ensure_session(), query, limit=limit - ) + def iter_query_rows(self, query: str, limit: int = -1) -> Iterator[Tuple[Any, ...]]: + yield from query_introspection.iter_query_rows(self._ensure_session(), query, limit=limit) diff --git a/db2sql/infrastructure/plugins/__init__.py b/db2sql/infrastructure/plugins/__init__.py index 91f4abb..fc6fd36 100644 --- a/db2sql/infrastructure/plugins/__init__.py +++ b/db2sql/infrastructure/plugins/__init__.py @@ -1,21 +1,21 @@ """Plugin discovery for readers, emitters and writers.""" from .registry import ( - EMITTERS_GROUP, - READERS_GROUP, - WRITERS_GROUP, - UnknownEmitterError, - UnknownReaderError, - UnknownWriterError, available_emitters, available_readers, available_writers, + EMITTERS_GROUP, get_source_reader, get_sql_emitter, get_target_writer, + READERS_GROUP, register_emitter, register_reader, register_writer, + UnknownEmitterError, + UnknownReaderError, + UnknownWriterError, + WRITERS_GROUP, ) __all__ = [ diff --git a/db2sql/infrastructure/plugins/registry.py b/db2sql/infrastructure/plugins/registry.py index 7332986..256597d 100644 --- a/db2sql/infrastructure/plugins/registry.py +++ b/db2sql/infrastructure/plugins/registry.py @@ -13,7 +13,7 @@ from __future__ import annotations from importlib import metadata -from typing import Any, Callable, Dict, List +from typing import Any, Callable, cast, Dict, List from db2sql.application.ports import Logger, SourceReader, SqlEmitter, TargetWriter from db2sql.infrastructure.config import AppConfig @@ -97,7 +97,7 @@ def get_source_reader(name: str, config: AppConfig, logger: Logger) -> SourceRea return _manual_readers[name](config, logger) eps = _load_entry_points(READERS_GROUP) if name in eps: - return eps[name](config, logger) + return cast(SourceReader, eps[name](config, logger)) raise UnknownReaderError(name, available_readers()) @@ -106,7 +106,7 @@ def get_sql_emitter(name: str, **kwargs: Any) -> SqlEmitter: return _manual_emitters[name](**kwargs) eps = _load_entry_points(EMITTERS_GROUP) if name in eps: - return eps[name](**kwargs) + return cast(SqlEmitter, eps[name](**kwargs)) raise UnknownEmitterError(name, available_emitters()) @@ -115,5 +115,5 @@ def get_target_writer(name: str, config: AppConfig, logger: Logger) -> TargetWri return _manual_writers[name](config, logger) eps = _load_entry_points(WRITERS_GROUP) if name in eps: - return eps[name](config, logger) + return cast(TargetWriter, eps[name](config, logger)) raise UnknownWriterError(name, available_writers()) diff --git a/db2sql/infrastructure/writer/mssql/writer.py b/db2sql/infrastructure/writer/mssql/writer.py index c41ccd5..3def634 100644 --- a/db2sql/infrastructure/writer/mssql/writer.py +++ b/db2sql/infrastructure/writer/mssql/writer.py @@ -106,9 +106,7 @@ def bulk_load( raw_cursor.executemany(insert_sql, batch) total += len(batch) except Exception as exc: - raise TargetWriterExecutionError( - f"INSERT into {qualified} failed: {exc}" - ) from exc + raise TargetWriterExecutionError(f"INSERT into {qualified} failed: {exc}") from exc self._logger.info(f"loaded {total} row(s) into {schema}.{table.name}") # ---- helpers ---------------------------------------------------------- diff --git a/db2sql/infrastructure/writer/postgres/writer.py b/db2sql/infrastructure/writer/postgres/writer.py index 5618ffd..e1364a7 100644 --- a/db2sql/infrastructure/writer/postgres/writer.py +++ b/db2sql/infrastructure/writer/postgres/writer.py @@ -104,9 +104,7 @@ def bulk_load( try: raw.copy_expert(copy_sql, buffer) except Exception as exc: - raise TargetWriterExecutionError( - f"COPY into {qualified} failed: {exc}" - ) from exc + raise TargetWriterExecutionError(f"COPY into {qualified} failed: {exc}") from exc self._logger.info(f"loaded {count} row(s) into {schema}.{table.name}") # ---- helpers ---------------------------------------------------------- diff --git a/db2sql/interface/cli/init_command.py b/db2sql/interface/cli/init_command.py index 9ffa97d..6d70138 100644 --- a/db2sql/interface/cli/init_command.py +++ b/db2sql/interface/cli/init_command.py @@ -4,7 +4,6 @@ import argparse import json -import os import sys from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Tuple @@ -100,7 +99,7 @@ def _ask_csv_list(prompter: Prompter, message: str) -> List[str]: if not prompter.confirm("Add more entries?", default=False): break # de-duplicate while preserving order - seen: set = set() + seen: set[str] = set() deduped: List[str] = [] for item in accumulated: if item not in seen: @@ -138,9 +137,7 @@ def _ask_server(prompter: Prompter, driver: str) -> Dict[str, Any]: elif driver == "oracle": server["hostname"] = prompter.text("Hostname", validate=_validate_required) port_default = str(_DEFAULT_PORTS.get(driver, 1521)) - server["port"] = int( - prompter.text("Port", default=port_default, validate=_validate_int) - ) + server["port"] = int(prompter.text("Port", default=port_default, validate=_validate_int)) server["username"] = prompter.text("Username", default="") mode = prompter.select( "Identify the database via", @@ -158,9 +155,7 @@ def _ask_server(prompter: Prompter, driver: str) -> Dict[str, Any]: # mssql / mysql / postgres / unknown network driver server["hostname"] = prompter.text("Hostname", validate=_validate_required) port_default = str(_DEFAULT_PORTS.get(driver, 0)) if driver in _DEFAULT_PORTS else "" - server["port"] = int( - prompter.text("Port", default=port_default, validate=_validate_int) - ) + server["port"] = int(prompter.text("Port", default=port_default, validate=_validate_int)) server["dbname"] = prompter.text("Database name", validate=_validate_required) server["username"] = prompter.text("Username", default="") password = _ask_password(prompter) @@ -200,9 +195,7 @@ def _ask_mapping_schemas(prompter: Prompter) -> Dict[str, str]: def _ask_table_overrides(prompter: Prompter) -> Dict[str, Dict[str, Any]]: tables: Dict[str, Dict[str, Any]] = {} while prompter.confirm("Add an override for a specific table?", default=False): - name = prompter.text( - "Table name (bare or schema.table)", validate=_validate_required - ) + name = prompter.text("Table name (bare or schema.table)", validate=_validate_required) override: Dict[str, Any] = {} fmt_choice = prompter.select( f"data_format for '{name}'", @@ -211,9 +204,7 @@ def _ask_table_overrides(prompter: Prompter) -> Dict[str, Dict[str, Any]]: ) if fmt_choice != "(inherit)": override["data_format"] = fmt_choice - limit_raw = prompter.text( - f"limit_records for '{name}' (empty = inherit)", default="" - ) + limit_raw = prompter.text(f"limit_records for '{name}' (empty = inherit)", default="") if limit_raw.strip(): try: override["limit_records"] = int(limit_raw) @@ -229,9 +220,7 @@ def _ask_table_overrides(prompter: Prompter) -> Dict[str, Dict[str, Any]]: return tables -def _ask_dump_section( - prompter: Prompter, target: str -) -> Tuple[Dict[str, Any], DataFormat]: +def _ask_dump_section(prompter: Prompter, target: str) -> Tuple[Dict[str, Any], DataFormat]: dump: Dict[str, Any] = {} if prompter.confirm("Preserve identifier case?", default=False): @@ -283,9 +272,7 @@ def build_config(prompter: Prompter) -> Tuple[AppConfig, OutputFormat]: if not emitters: raise InitAborted() - output_format = prompter.select( - "Output format", choices=["yaml", "json"], default="yaml" - ) + output_format = prompter.select("Output format", choices=["yaml", "json"], default="yaml") driver = prompter.select( "Source database driver", @@ -301,9 +288,7 @@ def build_config(prompter: Prompter) -> Tuple[AppConfig, OutputFormat]: server = _ask_server(prompter, driver) dump, _fmt = _ask_dump_section(prompter, target) - output_file = prompter.text( - "Default SQL output file (empty to write to stdout)", default="" - ) + output_file = prompter.text("Default SQL output file (empty to write to stdout)", default="") data: Dict[str, Any] = { "driver": driver, @@ -327,7 +312,8 @@ def serialize_config(config: AppConfig, output_format: OutputFormat) -> str: payload = config.model_dump(mode="json", exclude_defaults=True) if output_format == "json": return json.dumps(payload, indent=2, sort_keys=False) + "\n" - return yaml.safe_dump(payload, sort_keys=False, default_flow_style=False) + rendered: str = yaml.safe_dump(payload, sort_keys=False, default_flow_style=False) + return rendered def _write_output( @@ -342,9 +328,7 @@ def _write_output( path = Path(output_path) if path.exists() and not force: - if not prompter.confirm( - f"File {path} already exists. Overwrite?", default=False - ): + if not prompter.confirm(f"File {path} already exists. Overwrite?", default=False): raise InitAborted() path.parent.mkdir(parents=True, exist_ok=True) path.write_text(rendered, encoding="utf-8") diff --git a/db2sql/interface/cli/parser.py b/db2sql/interface/cli/parser.py index e36b5d0..4f6eabf 100644 --- a/db2sql/interface/cli/parser.py +++ b/db2sql/interface/cli/parser.py @@ -9,7 +9,12 @@ from db2sql import const from db2sql.application.dto import DataFormat -from db2sql.infrastructure.config import AppConfig, ConfigError, load_config, merge_cli_overrides +from db2sql.infrastructure.config import ( + AppConfig, + ConfigError, + load_config, + merge_cli_overrides, +) from db2sql.infrastructure.plugins import ( available_emitters, available_readers, @@ -44,9 +49,7 @@ def __init__(self, message: str) -> None: class MsDumpToPGArgumentParser(argparse.ArgumentParser): """Builds an :class:`AppConfig` from CLI args + config file + env.""" - def parse_args_with_config( - self, args: Optional[Sequence[str]] = None - ) -> argparse.Namespace: + def parse_args_with_config(self, args: Optional[Sequence[str]] = None) -> argparse.Namespace: options = super().parse_args(args) if "help" in options and options.help: @@ -61,9 +64,8 @@ def parse_args_with_config( # The validate subcommand accepts the config file as a positional arg; # let it win over -C / env / default lookup if provided. config_file = options.config_file - if ( - getattr(options, "command", None) == COMMAND_VALIDATE - and getattr(options, "validate_config_file", None) + if getattr(options, "command", None) == COMMAND_VALIDATE and getattr( + options, "validate_config_file", None ): config_file = options.validate_config_file @@ -144,53 +146,78 @@ def _add_dump_options(parser: argparse.ArgumentParser) -> None: action=OnceArgument, ) parser.add_argument( - "-H", "--host", - metavar="HOSTNAME", dest="hostname", type=str, + "-H", + "--host", + metavar="HOSTNAME", + dest="hostname", + type=str, help=f"Database server host name. [env var: {const.ENV_DB2SQL_HOST}]", default=os.getenv(const.ENV_DB2SQL_HOST), action=OnceArgument, ) parser.add_argument( - "-P", "--port", - metavar="PORT", dest="port", type=int, + "-P", + "--port", + metavar="PORT", + dest="port", + type=int, help=f"Database server port. [env var: {const.ENV_DB2SQL_PORT}]", default=os.getenv(const.ENV_DB2SQL_PORT), action=OnceArgument, ) parser.add_argument( - "-d", "--dbname", - metavar="DBNAME", dest="dbname", type=str, + "-d", + "--dbname", + metavar="DBNAME", + dest="dbname", + type=str, help=f"Database name to connect to. [env var: {const.ENV_DB2SQL_DBNAME}]", default=os.getenv(const.ENV_DB2SQL_DBNAME), action=OnceArgument, ) parser.add_argument( - "-u", "--username", - metavar="USERNAME", dest="username", type=str, + "-u", + "--username", + metavar="USERNAME", + dest="username", + type=str, help=f"Database user name. [env var: {const.ENV_DB2SQL_USER}]", default=os.getenv(const.ENV_DB2SQL_USER), action=OnceArgument, ) parser.add_argument( - "-p", "--password", - metavar="PASSWORD", dest="password", type=str, + "-p", + "--password", + metavar="PASSWORD", + dest="password", + type=str, help=f"Database password. [env var: {const.ENV_DB2SQL_PASSWORD}]", default=os.getenv(const.ENV_DB2SQL_PASSWORD), action=OnceArgument, ) parser.add_argument( - "-W", "--ask-password", - dest="ask_password", action="store_true", default=False, + "-W", + "--ask-password", + dest="ask_password", + action="store_true", + default=False, help="Force password prompt.", ) parser.add_argument( - "-f", "--file", - metavar="PATH", dest="output_file_name", type=str, default=None, + "-f", + "--file", + metavar="PATH", + dest="output_file_name", + type=str, + default=None, help="Output file. If not provided, script is printed to standard output.", ) parser.add_argument( "--split-size", - metavar="SIZE", dest="split_size", type=_parse_size, default=None, + metavar="SIZE", + dest="split_size", + type=_parse_size, + default=None, help=( "Split the dump into multiple files when the current file exceeds " "SIZE. Accepts a byte count or a suffixed value (K/M/G). Requires -f." @@ -210,12 +237,16 @@ def _add_dump_options(parser: argparse.ArgumentParser) -> None: ) parser.add_argument( "--preserve-case", - dest="preserve_case", action=BooleanAction, default=None, + dest="preserve_case", + action=BooleanAction, + default=None, help="Preserve identifier case. When disabled, names are converted to snake_case.", ) parser.add_argument( "--transaction", - dest="dump_use_transaction", action=BooleanAction, default=None, + dest="dump_use_transaction", + action=BooleanAction, + default=None, help=( "Wrap the dump in a transaction (BEGIN/COMMIT). Disable with " "--no-transaction when the SQL is consumed by a tool that manages " @@ -223,55 +254,89 @@ def _add_dump_options(parser: argparse.ArgumentParser) -> None: ), ) parser.add_argument( - "-n", "--max-records", - dest="limit_records", type=int, default=None, + "-n", + "--max-records", + dest="limit_records", + type=int, + default=None, help="Limit the number of rows from each table. -1 means no limit.", ) parser.add_argument( "--data-format", dest="data_format", - choices=[fmt.value for fmt in DataFormat], default=None, + choices=[fmt.value for fmt in DataFormat], + default=None, help="Default output format for table data: copy (faster) or insert.", ) parser.add_argument( - "-i", "--include-schemas", - metavar="NAME", dest="include_schemas", type=str, - action="append", nargs="+", default=None, + "-i", + "--include-schemas", + metavar="NAME", + dest="include_schemas", + type=str, + action="append", + nargs="+", + default=None, help="Schema names to include during export (repeatable, comma separated).", ) parser.add_argument( - "-x", "--exclude-schemas", - metavar="NAME", dest="exclude_schemas", type=str, - action="append", nargs="+", default=None, + "-x", + "--exclude-schemas", + metavar="NAME", + dest="exclude_schemas", + type=str, + action="append", + nargs="+", + default=None, help="Schema names to exclude during export (repeatable, comma separated).", ) parser.add_argument( - "-I", "--include-tables", - metavar="NAME", dest="include_tables", type=str, - action="append", nargs="+", default=None, + "-I", + "--include-tables", + metavar="NAME", + dest="include_tables", + type=str, + action="append", + nargs="+", + default=None, help="Table names to include during export (repeatable, comma separated).", ) parser.add_argument( - "-X", "--exclude-tables", - metavar="NAME", dest="exclude_tables", type=str, - action="append", nargs="+", default=None, + "-X", + "--exclude-tables", + metavar="NAME", + dest="exclude_tables", + type=str, + action="append", + nargs="+", + default=None, help="Table names to exclude during export (repeatable, comma separated).", ) parser.add_argument( - "-C", "--config-file", - metavar="PATH", dest="config_file", type=str, + "-C", + "--config-file", + metavar="PATH", + dest="config_file", + type=str, help=f"Configuration file to use. [env var: {const.ENV_DB2SQL_CONFIG}]", ) parser.add_argument( - "-L", "--log-file", - metavar="PATH", dest="log_file", type=str, + "-L", + "--log-file", + metavar="PATH", + dest="log_file", + type=str, help="Send log output to PATH instead of stdout.", action=OnceArgument, ) parser.add_argument( - "-V", "--verbosity", - metavar="LEVEL", dest="verbosity", default="status", - nargs="?", type=str, + "-V", + "--verbosity", + metavar="LEVEL", + dest="verbosity", + default="status", + nargs="?", + type=str, help=( "Level of detail of the output. Valid options from less verbose to " "more verbose: -Vquiet, -Verror, -Vwarning, -Vnotice, -Vstatus, " @@ -279,7 +344,10 @@ def _add_dump_options(parser: argparse.ArgumentParser) -> None: ), ) parser.add_argument( - "--version", dest="version", action="store_true", default=False, + "--version", + dest="version", + action="store_true", + default=False, help="Output version information and exit.", ) @@ -437,7 +505,9 @@ def _add_migrate_subparser(subparsers: Any) -> None: ) migrate_parser.add_argument( "--transaction", - dest="use_transaction", action=BooleanAction, default=None, + dest="use_transaction", + action=BooleanAction, + default=None, help=( "Wrap the emitted SQL in a transaction (BEGIN/COMMIT). Disable " "with --no-transaction to let the target driver auto-commit each " @@ -458,13 +528,19 @@ def _add_init_subparser(subparsers: Any) -> None: formatter_class=SmartFormatter, ) init_parser.add_argument( - "-o", "--output", - dest="init_output", metavar="PATH", type=str, default=None, + "-o", + "--output", + dest="init_output", + metavar="PATH", + type=str, + default=None, help="Write the generated config to PATH (default: stdout).", ) init_parser.add_argument( "--force", - dest="init_force", action="store_true", default=False, + dest="init_force", + action="store_true", + default=False, help="Overwrite the output file without asking for confirmation.", ) diff --git a/db2sql/interface/cli/runner.py b/db2sql/interface/cli/runner.py index 0ff42c0..c698cab 100644 --- a/db2sql/interface/cli/runner.py +++ b/db2sql/interface/cli/runner.py @@ -16,16 +16,16 @@ to_dump_request, to_migrate_request, ) -from db2sql.infrastructure.logging import ConsoleLogger, Palette, init_colorama +from db2sql.infrastructure.logging import ConsoleLogger, init_colorama, Palette from db2sql.infrastructure.output import ExecutingSink, RotatingFileSink, StreamSink from db2sql.infrastructure.persistence.errors import SourceReaderError from db2sql.infrastructure.plugins import ( + get_source_reader, + get_sql_emitter, + get_target_writer, UnknownEmitterError, UnknownReaderError, UnknownWriterError, - get_sql_emitter, - get_source_reader, - get_target_writer, ) from db2sql.infrastructure.writer import TargetWriterError @@ -38,12 +38,12 @@ ) from .init_command import run_init from .parser import ( + AbortExecution, + build_parser, COMMAND_INIT, COMMAND_MIGRATE, COMMAND_VALIDATE, - AbortExecution, CommandLineError, - build_parser, ) from .validate_command import run_validate @@ -107,9 +107,7 @@ def _execute(self, config: AppConfig) -> None: ) request = to_dump_request(config) if request.split_size is not None and not request.output_file: - raise CommandLineError( - "--split-size requires -f/--file (cannot rotate stdout)." - ) + raise CommandLineError("--split-size requires -f/--file (cannot rotate stdout).") with self._open_dump_sink(request) as sink: use_case = DumpDatabaseUseCase( reader=reader, diff --git a/db2sql/interface/cli/validate_command.py b/db2sql/interface/cli/validate_command.py index fd377b9..8d5f638 100644 --- a/db2sql/interface/cli/validate_command.py +++ b/db2sql/interface/cli/validate_command.py @@ -17,21 +17,20 @@ from __future__ import annotations import argparse -from typing import Dict, List, Tuple +from typing import List from db2sql.application.dto import DumpRequest from db2sql.application.ports import SourceReader -from db2sql.domain.model import Database +from db2sql.domain.model import Database, Table from db2sql.domain.policy import filter_database from db2sql.infrastructure.config import AppConfig, to_dump_request from db2sql.infrastructure.logging import ConsoleLogger, Palette from db2sql.infrastructure.persistence.errors import SourceReaderError from db2sql.infrastructure.plugins import ( - UnknownEmitterError, - UnknownReaderError, available_emitters, available_readers, get_source_reader, + UnknownReaderError, ) from .exit_codes import ERROR_GENERAL, ERROR_INVALID_CONFIGURATION, SUCCESS @@ -75,8 +74,7 @@ def _check_plugin_names(config: AppConfig, logger: ConsoleLogger) -> int | None: emitters = available_emitters() if config.driver not in readers: logger.error( - f"unknown driver {config.driver!r}; known: " - f"{', '.join(sorted(readers)) or '(none)'}" + f"unknown driver {config.driver!r}; known: " f"{', '.join(sorted(readers)) or '(none)'}" ) return ERROR_INVALID_CONFIGURATION if config.target not in emitters: @@ -93,9 +91,7 @@ def _check_plugin_names(config: AppConfig, logger: ConsoleLogger) -> int | None: # --------------------------------------------------------------------------- # -def _run_dry_run( - config: AppConfig, logger: ConsoleLogger, *, with_counts: bool -) -> int: +def _run_dry_run(config: AppConfig, logger: ConsoleLogger, *, with_counts: bool) -> int: request = to_dump_request(config) _print_header(logger, "Source connection") _print_connection(logger, config) @@ -139,9 +135,7 @@ def _print_connection(logger: ConsoleLogger, config: AppConfig) -> None: _print_kv(logger, "options", pretty) -def _print_filter_summary( - logger: ConsoleLogger, raw: Database, filtered: Database -) -> None: +def _print_filter_summary(logger: ConsoleLogger, raw: Database, filtered: Database) -> None: _print_header(logger, "Filtering") total_schemas = len(raw.schemas) total_tables = sum(len(s.tables) for s in raw.schemas.values()) @@ -192,13 +186,11 @@ def _print_plan( count_str = "" if with_counts: count_str = f", count={_count_rows(reader, schema_name, schema.tables[table_name], limit, logger)}" - logger.info( - f" - {table_name} (format={fmt}, {limit_str}{count_str})" - ) + logger.info(f" - {table_name} (format={fmt}, {limit_str}{count_str})") def _count_rows( - reader: SourceReader, schema: str, table: object, limit: int, logger: ConsoleLogger + reader: SourceReader, schema: str, table: Table, limit: int, logger: ConsoleLogger ) -> str: """Return a row count for the table. diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py index 8baaf78..a39956e 100644 --- a/tests/functional/conftest.py +++ b/tests/functional/conftest.py @@ -83,3 +83,39 @@ def oracle_config() -> AppConfig: options={"service_name": env["ORACLE_SERVICE"], "owner": env["ORACLE_USER"]}, ), ) + + +@pytest.fixture(scope="session") +def postgres_config() -> AppConfig: + env = _require_env( + "PG_HOST", "PG_PORT", "PG_USER", "PG_PASSWORD", "PG_DATABASE" + ) + _check_tcp(env["PG_HOST"], int(env["PG_PORT"])) + return AppConfig( + driver="postgres", + server=ServerConfig( + hostname=env["PG_HOST"], + port=int(env["PG_PORT"]), + username=env["PG_USER"], + password=env["PG_PASSWORD"], + dbname=env["PG_DATABASE"], + ), + ) + + +@pytest.fixture(scope="session") +def mysql_config() -> AppConfig: + env = _require_env( + "MYSQL_HOST", "MYSQL_PORT", "MYSQL_USER", "MYSQL_PASSWORD", "MYSQL_DATABASE" + ) + _check_tcp(env["MYSQL_HOST"], int(env["MYSQL_PORT"])) + return AppConfig( + driver="mysql", + server=ServerConfig( + hostname=env["MYSQL_HOST"], + port=int(env["MYSQL_PORT"]), + username=env["MYSQL_USER"], + password=env["MYSQL_PASSWORD"], + dbname=env["MYSQL_DATABASE"], + ), + ) diff --git a/tests/functional/test_mysql_functional.py b/tests/functional/test_mysql_functional.py new file mode 100644 index 0000000..0cf34b6 --- /dev/null +++ b/tests/functional/test_mysql_functional.py @@ -0,0 +1,108 @@ +"""Functional tests for the MySQL source reader and the postgres emitter. + +Run via ``make test-functional`` after ``make stack-up`` (or ``stack-up-light``). + +The fixture comes from ``.docker/mysql/init/01-schema.sql``: a ``db2sqltest`` +database with a ``type_matrix`` table (covering MySQL-native source types +referenced by ``PostgresSqlEmitter.DEFAULT_TYPE_MAP``) plus the standard +author/book relational fixture. + +MySQL has no notion of "schema" separate from "database", so the reader +exposes the database name as a single schema entry. +""" + +from __future__ import annotations + +import pytest + +from db2sql.infrastructure.config import AppConfig +from db2sql.infrastructure.emit.postgres.emitter import PostgresSqlEmitter +from db2sql.infrastructure.persistence.mysql.reader import MySQLSourceReader + +pytestmark = pytest.mark.functional + + +# Expected source-column → mysql-reported-type for every column in +# .docker/mysql/init/01-schema.sql:type_matrix. +# MySQL's INFORMATION_SCHEMA.COLUMNS.DATA_TYPE returns the canonical +# lowercase name without precision (e.g. "varchar" not "varchar(64)"), so +# we match those 1:1 against the keys of PostgresSqlEmitter.DEFAULT_TYPE_MAP. +EXPECTED_TYPES = { + "c_bit": "bit", + "c_tinyint": "tinyint", + "c_smallint": "smallint", + "c_mediumint": "mediumint", + "c_int": "int", + "c_bigint": "bigint", + "c_decimal": "decimal", + "c_numeric": "decimal", # MySQL aliases NUMERIC → DECIMAL + "c_float": "float", + "c_double": "double", + "c_char": "char", + "c_varchar": "varchar", + "c_text": "text", + "c_mediumtext": "mediumtext", + "c_longtext": "longtext", + "c_binary": "binary", + "c_varbinary": "varbinary", + "c_blob": "blob", + "c_date": "date", + "c_time": "time", + "c_datetime": "datetime", + "c_timestamp": "timestamp", + "c_json": "json", +} + + +@pytest.fixture(scope="module") +def mysql_metadata(mysql_config: AppConfig, null_logger): + pytest.importorskip("pymysql") + reader = MySQLSourceReader(mysql_config, null_logger) + return reader.collect_metadata() + + +def test_mysql_schema_and_tables(mysql_metadata) -> None: + # The reader uses the database name as the schema key. + assert "db2sqltest" in mysql_metadata.schemas + schema = mysql_metadata.schemas["db2sqltest"] + assert {"type_matrix", "author", "book"}.issubset(schema.tables.keys()) + + +def test_mysql_type_matrix_columns_present(mysql_metadata) -> None: + table = mysql_metadata.schemas["db2sqltest"].get_table("type_matrix") + assert table is not None + for column_name in EXPECTED_TYPES: + assert column_name in table.columns, f"column {column_name} missing" + + +def test_mysql_default_type_mapping(mysql_metadata) -> None: + table = mysql_metadata.schemas["db2sqltest"].get_table("type_matrix") + assert table is not None + emitter = PostgresSqlEmitter() + for column_name, expected_source in EXPECTED_TYPES.items(): + column = table.columns[column_name] + assert column.type.lower() == expected_source, ( + f"{column_name}: reader returned {column.type!r}, expected {expected_source!r}" + ) + expected_target = PostgresSqlEmitter.DEFAULT_TYPE_MAP[expected_source] + rendered = emitter.column_definition(column) + assert expected_target.split("(")[0] in rendered, ( + f"{column_name}: rendered={rendered!r}, expected target type {expected_target!r}" + ) + + +def test_mysql_identity_and_pk(mysql_metadata) -> None: + table = mysql_metadata.schemas["db2sqltest"].get_table("type_matrix") + assert table is not None + pk_id = table.columns["id"] + assert pk_id.identity is True, "AUTO_INCREMENT should mark the column as identity" + assert pk_id.constraint == "PRIMARY KEY" + + +def test_mysql_foreign_key_and_index(mysql_metadata) -> None: + book = mysql_metadata.schemas["db2sqltest"].get_table("book") + assert book is not None + fk = book.columns["author_id"].foreign_key + assert fk is not None + assert (fk.schema, fk.table, fk.column) == ("db2sqltest", "author", "id") + assert any("title" in cols for cols in book.indexes.values()) diff --git a/tests/functional/test_postgres_functional.py b/tests/functional/test_postgres_functional.py new file mode 100644 index 0000000..943265a --- /dev/null +++ b/tests/functional/test_postgres_functional.py @@ -0,0 +1,196 @@ +"""Functional tests for the PostgreSQL source reader. + +Covers two pipelines: + +1. ``PostgresSourceReader`` → ``PostgresSqlEmitter`` (the canonical export + target — types should round-trip through ``DEFAULT_TYPE_MAP`` unchanged). +2. ``PostgresSourceReader`` → ``MssqlSqlEmitter`` (the pg → mssql migration + path: PG-native types must map to a non-empty MSSQL target type via + ``MssqlSqlEmitter.DEFAULT_TYPE_MAP``). + +Run via ``make test-functional`` after ``make stack-up`` (or ``stack-up-light``). + +The fixture comes from ``.docker/postgres/init/01-schema.sql`` which creates +an ``apptest`` schema with ``type_matrix``/``author``/``book`` tables. +""" + +from __future__ import annotations + +import pytest + +from db2sql.infrastructure.config import AppConfig +from db2sql.infrastructure.emit.mssql.emitter import MssqlSqlEmitter +from db2sql.infrastructure.emit.postgres.emitter import PostgresSqlEmitter +from db2sql.infrastructure.persistence.postgres.reader import PostgresSourceReader + +pytestmark = pytest.mark.functional + + +# Expected source-column → pg-reported-type for every column in +# .docker/postgres/init/01-schema.sql:apptest.type_matrix. PG's +# INFORMATION_SCHEMA.COLUMNS.DATA_TYPE reports the canonical lowercase name +# (e.g. "timestamp with time zone", "double precision"); we match those 1:1 +# against the keys of PostgresSqlEmitter.DEFAULT_TYPE_MAP. +EXPECTED_TYPES = { + "c_boolean": "boolean", + "c_smallint": "smallint", + "c_integer": "integer", + "c_bigint": "bigint", + "c_real": "real", + "c_double": "double precision", + "c_numeric": "numeric", + "c_decimal": "numeric", + "c_char": "character", + "c_varchar": "character varying", + "c_text": "text", + "c_bytea": "bytea", + "c_date": "date", + "c_time": "time without time zone", + "c_timestamp": "timestamp without time zone", + "c_timestamptz": "timestamp with time zone", + "c_uuid": "uuid", + "c_json": "json", + "c_jsonb": "jsonb", + "c_xml": "xml", +} + + +@pytest.fixture(scope="module") +def postgres_metadata(postgres_config: AppConfig, null_logger): + pytest.importorskip("psycopg2") + reader = PostgresSourceReader(postgres_config, null_logger) + return reader.collect_metadata() + + +def test_postgres_schema_and_tables(postgres_metadata) -> None: + assert "apptest" in postgres_metadata.schemas + schema = postgres_metadata.schemas["apptest"] + assert {"type_matrix", "author", "book"}.issubset(schema.tables.keys()) + + +def test_postgres_type_matrix_columns_present(postgres_metadata) -> None: + table = postgres_metadata.schemas["apptest"].get_table("type_matrix") + assert table is not None + for column_name in EXPECTED_TYPES: + assert column_name in table.columns, f"column {column_name} missing" + + +def test_postgres_default_type_mapping(postgres_metadata) -> None: + """Each PG-reported source type maps to a non-empty PG target column.""" + table = postgres_metadata.schemas["apptest"].get_table("type_matrix") + assert table is not None + emitter = PostgresSqlEmitter() + for column_name, expected_source in EXPECTED_TYPES.items(): + column = table.columns[column_name] + assert column.type.lower() == expected_source, ( + f"{column_name}: reader returned {column.type!r}, expected {expected_source!r}" + ) + rendered = emitter.column_definition(column) + assert rendered.strip(), f"{column_name}: emitter produced empty definition" + # The column name must always appear; the rendered type must not be + # empty after the name token. + tokens = rendered.split() + assert len(tokens) >= 2, f"{column_name}: rendered={rendered!r}" + + +def test_postgres_identity_and_pk(postgres_metadata) -> None: + table = postgres_metadata.schemas["apptest"].get_table("type_matrix") + assert table is not None + pk_id = table.columns["id"] + assert pk_id.identity is True + assert pk_id.constraint == "PRIMARY KEY" + + +def test_postgres_foreign_key_and_index(postgres_metadata) -> None: + book = postgres_metadata.schemas["apptest"].get_table("book") + assert book is not None + fk = book.columns["author_id"].foreign_key + assert fk is not None + assert (fk.schema, fk.table, fk.column) == ("apptest", "author", "id") + assert any("title" in cols for cols in book.indexes.values()) + + +# --------------------------------------------------------------------------- +# pg → mssql: same source metadata, but rendered through the MSSQL emitter. +# --------------------------------------------------------------------------- + +# Subset of EXPECTED_TYPES whose PG name is a direct key of +# MssqlSqlEmitter.DEFAULT_TYPE_MAP. Multi-word PG types ("double precision", +# "timestamp with time zone") and "character"/"character varying" are not in +# the MSSQL map verbatim, so we cover those via the no-empty-render check in +# ``test_pg_to_mssql_default_type_mapping``. +PG_TO_MSSQL_EXPECTED = { + "c_boolean": "bit", + "c_smallint": "smallint", + "c_bigint": "bigint", + "c_real": "real", + "c_numeric": "numeric", + "c_decimal": "numeric", + "c_text": "nvarchar(max)", + "c_bytea": "varbinary(max)", + "c_date": "date", + "c_uuid": "uniqueidentifier", + "c_json": "nvarchar(max)", + "c_jsonb": "nvarchar(max)", + "c_xml": "xml", +} + + +def test_pg_to_mssql_default_type_mapping(postgres_metadata) -> None: + """Each PG source column renders to a non-empty MSSQL definition. + + For columns whose PG type appears verbatim in ``MssqlSqlEmitter.DEFAULT_TYPE_MAP`` + we additionally assert the expected target type is present in the output. + """ + table = postgres_metadata.schemas["apptest"].get_table("type_matrix") + assert table is not None + emitter = MssqlSqlEmitter() + for column_name in EXPECTED_TYPES: + column = table.columns[column_name] + rendered = emitter.column_definition(column) + assert rendered.strip(), f"{column_name}: mssql emitter produced empty definition" + tokens = rendered.split() + assert len(tokens) >= 2, f"{column_name}: rendered={rendered!r}" + expected_target = PG_TO_MSSQL_EXPECTED.get(column_name) + if expected_target is not None: + assert expected_target.split("(")[0] in rendered, ( + f"{column_name}: rendered={rendered!r}, " + f"expected target type {expected_target!r}" + ) + + +def test_pg_to_mssql_identity_renders_as_identity(postgres_metadata) -> None: + """PG ``GENERATED ALWAYS AS IDENTITY`` must become ``IDENTITY(1,1)`` for MSSQL.""" + table = postgres_metadata.schemas["apptest"].get_table("type_matrix") + assert table is not None + pk_id = table.columns["id"] + rendered = MssqlSqlEmitter().column_definition(pk_id) + assert "IDENTITY(1,1)" in rendered, rendered + + +def test_pg_to_mssql_table_emits_qualified_names(postgres_metadata) -> None: + """Smoke-test the full table emission: schema-qualified, bracket-quoted.""" + import io + + class _Sink: + def __init__(self) -> None: + self.buf = io.StringIO() + + def write(self, data: str) -> None: + self.buf.write(data) + + def boundary(self) -> None: + pass + + def value(self) -> str: + return self.buf.getvalue() + + sink = _Sink() + emitter = MssqlSqlEmitter() + emitter.emit_tables(postgres_metadata, sink) + out = sink.value() + assert "[apptest].[type_matrix]" in out + assert "[apptest].[author]" in out + assert "[apptest].[book]" in out + # IDENTITY rendered on the PK column + assert "IDENTITY(1,1)" in out diff --git a/tests/unit/infrastructure/persistence/test_oracle_reader.py b/tests/unit/infrastructure/persistence/test_oracle_reader.py index cfe1ca2..12ef905 100644 --- a/tests/unit/infrastructure/persistence/test_oracle_reader.py +++ b/tests/unit/infrastructure/persistence/test_oracle_reader.py @@ -28,61 +28,61 @@ def _build_reader(**server_kwargs) -> OracleSourceReader: def _full_plan() -> FakeSession: session = FakeSession() - session.add("FROM ALL_TABLES", [FakeRow(OWNER="HR", TABLE_NAME="EMP")]) + session.add("FROM ALL_TABLES", [FakeRow(owner="HR", table_name="EMP")]) session.add( "FROM ALL_TABLES t", - [FakeRow(OWNER="HR", TABLE_NAME="EMP")], + [FakeRow(owner="HR", table_name="EMP")], ) session.add( "FROM ALL_TAB_COLUMNS", [ FakeRow( - OWNER="HR", - TABLE_NAME="EMP", - COLUMN_NAME="ID", - DATA_DEFAULT=None, - NULLABLE="N", - DATA_TYPE="NUMBER", - DATA_LENGTH=22, - CHAR_LENGTH=0, - DATA_PRECISION=10, - DATA_SCALE=0, + owner="HR", + table_name="EMP", + column_name="ID", + data_default=None, + nullable="N", + data_type="NUMBER", + data_length=22, + char_length=0, + data_precision=10, + data_scale=0, ), FakeRow( - OWNER="HR", - TABLE_NAME="EMP", - COLUMN_NAME="NAME", - DATA_DEFAULT="'unknown' ", - NULLABLE="Y", - DATA_TYPE="VARCHAR2", - DATA_LENGTH=100, - CHAR_LENGTH=100, - DATA_PRECISION=None, - DATA_SCALE=None, + owner="HR", + table_name="EMP", + column_name="NAME", + data_default="'unknown' ", + nullable="Y", + data_type="VARCHAR2", + data_length=100, + char_length=100, + data_precision=None, + data_scale=None, ), FakeRow( - OWNER="HR", - TABLE_NAME="EMP", - COLUMN_NAME="HIRED_AT", - DATA_DEFAULT=None, - NULLABLE="Y", - DATA_TYPE="DATE", - DATA_LENGTH=7, - CHAR_LENGTH=0, - DATA_PRECISION=None, - DATA_SCALE=None, + owner="HR", + table_name="EMP", + column_name="HIRED_AT", + data_default=None, + nullable="Y", + data_type="DATE", + data_length=7, + char_length=0, + data_precision=None, + data_scale=None, ), FakeRow( - OWNER="HR", - TABLE_NAME="EMP", - COLUMN_NAME="DEPT_ID", - DATA_DEFAULT=None, - NULLABLE="Y", - DATA_TYPE="NUMBER", - DATA_LENGTH=22, - CHAR_LENGTH=0, - DATA_PRECISION=10, - DATA_SCALE=0, + owner="HR", + table_name="EMP", + column_name="DEPT_ID", + data_default=None, + nullable="Y", + data_type="NUMBER", + data_length=22, + char_length=0, + data_precision=10, + data_scale=0, ), ], ) @@ -90,34 +90,34 @@ def _full_plan() -> FakeSession: "c.CONSTRAINT_TYPE = 'R'", [ FakeRow( - OWNER="HR", - TABLE_NAME="EMP", - COLUMN_NAME="DEPT_ID", - POSITION=1, - REF_OWNER="HR", - REF_TABLE="DEPT", - REF_COLUMN="ID", + owner="HR", + table_name="EMP", + column_name="DEPT_ID", + position=1, + ref_owner="HR", + ref_table="DEPT", + ref_column="ID", ) ], ) session.add( "c.CONSTRAINT_TYPE IN ('P', 'U')", - [FakeRow(OWNER="HR", TABLE_NAME="EMP", COLUMN_NAME="ID", CONSTRAINT_TYPE="PRIMARY KEY")], + [FakeRow(owner="HR", table_name="EMP", column_name="ID", constraint_type="PRIMARY KEY")], ) session.add( "FROM ALL_INDEXES", [ FakeRow( - TABLE_OWNER="HR", - TABLE_NAME="EMP", - INDEX_NAME="IDX_EMP_NAME", - COLUMN_NAME="NAME", + table_owner="HR", + table_name="EMP", + index_name="IDX_EMP_NAME", + column_name="NAME", ) ], ) session.add( "FROM ALL_TAB_IDENTITY_COLS", - [FakeRow(OWNER="HR", TABLE_NAME="EMP", COLUMN_NAME="ID")], + [FakeRow(owner="HR", table_name="EMP", column_name="ID")], ) return session @@ -186,8 +186,8 @@ def test_collect_metadata_builds_full_database() -> None: def test_owner_option_constrains_schema_filter() -> None: reader = _build_reader(options={"owner": "hr"}) session = FakeSession() - session.add("FROM ALL_TABLES", [FakeRow(OWNER="HR", TABLE_NAME="EMP")]) - session.add("FROM ALL_TABLES t", [FakeRow(OWNER="HR", TABLE_NAME="EMP")]) + session.add("FROM ALL_TABLES", [FakeRow(owner="HR", table_name="EMP")]) + session.add("FROM ALL_TABLES t", [FakeRow(owner="HR", table_name="EMP")]) session.add("FROM ALL_TAB_COLUMNS", []) session.add("c.CONSTRAINT_TYPE = 'R'", []) session.add("c.CONSTRAINT_TYPE IN ('P', 'U')", []) @@ -219,7 +219,7 @@ def execute(self, *_args, **_kwargs): def test_iter_rows_uses_oracle_fetch_first_for_limit() -> None: reader = _build_reader() session = FakeSession() - session.add("FETCH FIRST", [FakeRow(ID=1, NAME="Alice")]) + session.add("FETCH FIRST", [FakeRow(id=1, name="Alice")]) install_fake_session(reader, session) from db2sql.domain.model import Column, Table @@ -239,22 +239,22 @@ def test_identity_columns_query_is_optional() -> None: """Older Oracle versions lack ALL_TAB_IDENTITY_COLS — the reader must tolerate it.""" reader = _build_reader() session = FakeSession() - session.add("FROM ALL_TABLES", [FakeRow(OWNER="HR", TABLE_NAME="EMP")]) - session.add("FROM ALL_TABLES t", [FakeRow(OWNER="HR", TABLE_NAME="EMP")]) + session.add("FROM ALL_TABLES", [FakeRow(owner="HR", table_name="EMP")]) + session.add("FROM ALL_TABLES t", [FakeRow(owner="HR", table_name="EMP")]) session.add( "FROM ALL_TAB_COLUMNS", [ FakeRow( - OWNER="HR", - TABLE_NAME="EMP", - COLUMN_NAME="ID", - DATA_DEFAULT=None, - NULLABLE="N", - DATA_TYPE="NUMBER", - DATA_LENGTH=22, - CHAR_LENGTH=0, - DATA_PRECISION=10, - DATA_SCALE=0, + owner="HR", + table_name="EMP", + column_name="ID", + data_default=None, + nullable="N", + data_type="NUMBER", + data_length=22, + char_length=0, + data_precision=10, + data_scale=0, ) ], ) @@ -315,77 +315,77 @@ def test_collect_metadata_skips_rows_for_missing_columns_and_tables() -> None: reader = _build_reader() session = _FakeSession() - session.add("DISTINCT OWNER FROM ALL_TABLES", [_FakeRow(OWNER="HR")]) - session.add("FROM ALL_TABLES t", [_FakeRow(OWNER="HR", TABLE_NAME="EMP")]) + session.add("DISTINCT OWNER FROM ALL_TABLES", [_FakeRow(owner="HR")]) + session.add("FROM ALL_TABLES t", [_FakeRow(owner="HR", table_name="EMP")]) session.add( "FROM ALL_TAB_COLUMNS", [ _FakeRow( - OWNER="HR", - TABLE_NAME="EMP", - COLUMN_NAME="ID", - DATA_DEFAULT=None, - NULLABLE="N", - DATA_TYPE="NUMBER", - DATA_LENGTH=22, - CHAR_LENGTH=0, - DATA_PRECISION=10, - DATA_SCALE=0, + owner="HR", + table_name="EMP", + column_name="ID", + data_default=None, + nullable="N", + data_type="NUMBER", + data_length=22, + char_length=0, + data_precision=10, + data_scale=0, ), # Char column with 0 length → exercises the "if not char_length" branch _FakeRow( - OWNER="HR", - TABLE_NAME="EMP", - COLUMN_NAME="MEMO", - DATA_DEFAULT=None, - NULLABLE="Y", - DATA_TYPE="CHAR", - DATA_LENGTH=10, - CHAR_LENGTH=0, - DATA_PRECISION=None, - DATA_SCALE=None, + owner="HR", + table_name="EMP", + column_name="MEMO", + data_default=None, + nullable="Y", + data_type="CHAR", + data_length=10, + char_length=0, + data_precision=None, + data_scale=None, ), # Column for an unknown table — must be ignored _FakeRow( - OWNER="HR", - TABLE_NAME="GHOST", - COLUMN_NAME="X", - DATA_DEFAULT=None, - NULLABLE="Y", - DATA_TYPE="NUMBER", - DATA_LENGTH=22, - CHAR_LENGTH=0, - DATA_PRECISION=10, - DATA_SCALE=0, + owner="HR", + table_name="GHOST", + column_name="X", + data_default=None, + nullable="Y", + data_type="NUMBER", + data_length=22, + char_length=0, + data_precision=10, + data_scale=0, ), ], ) # Constraint referencing a non-existent column — must be skipped session.add( "c.CONSTRAINT_TYPE IN ('P', 'U')", - [_FakeRow(OWNER="HR", TABLE_NAME="EMP", COLUMN_NAME="MISSING", CONSTRAINT_TYPE="UNIQUE")], + [_FakeRow(owner="HR", table_name="EMP", column_name="MISSING", constraint_type="UNIQUE")], ) # FK referencing a non-existent column AND a non-existent table session.add( "c.CONSTRAINT_TYPE = 'R'", [ _FakeRow( - OWNER="HR", - TABLE_NAME="EMP", - COLUMN_NAME="MISSING", - POSITION=1, - REF_OWNER="HR", - REF_TABLE="DEPT", - REF_COLUMN="ID", + owner="HR", + table_name="EMP", + column_name="MISSING", + position=1, + ref_owner="HR", + ref_table="DEPT", + ref_column="ID", ), _FakeRow( - OWNER="HR", - TABLE_NAME="GHOST", - COLUMN_NAME="X", - POSITION=1, - REF_OWNER="HR", - REF_TABLE="DEPT", - REF_COLUMN="ID", + owner="HR", + table_name="GHOST", + column_name="X", + position=1, + ref_owner="HR", + ref_table="DEPT", + ref_column="ID", ), ], ) @@ -393,7 +393,7 @@ def test_collect_metadata_skips_rows_for_missing_columns_and_tables() -> None: # Identity row pointing to an unknown column session.add( "FROM ALL_TAB_IDENTITY_COLS", - [_FakeRow(OWNER="HR", TABLE_NAME="EMP", COLUMN_NAME="MISSING")], + [_FakeRow(owner="HR", table_name="EMP", column_name="MISSING")], ) install_fake_session(reader, session) db = reader.collect_metadata() @@ -410,7 +410,7 @@ def test_collect_metadata_wraps_unexpected_exception_post_schemas() -> None: reader = _build_reader() session = _FakeSession() - session.add("DISTINCT OWNER FROM ALL_TABLES", [_FakeRow(OWNER="HR")]) + session.add("DISTINCT OWNER FROM ALL_TABLES", [_FakeRow(owner="HR")]) original_execute = session.execute diff --git a/tests/unit/infrastructure/persistence/test_postgres_reader.py b/tests/unit/infrastructure/persistence/test_postgres_reader.py index 93f487f..e10f53c 100644 --- a/tests/unit/infrastructure/persistence/test_postgres_reader.py +++ b/tests/unit/infrastructure/persistence/test_postgres_reader.py @@ -31,50 +31,50 @@ def _populated_session() -> FakeSession: session.add( "FROM INFORMATION_SCHEMA.TABLES", [ - FakeRow(TABLE_SCHEMA="public", TABLE_NAME="author"), - FakeRow(TABLE_SCHEMA="public", TABLE_NAME="book"), - FakeRow(TABLE_SCHEMA="other", TABLE_NAME="thing"), + FakeRow(table_schema="public", table_name="author"), + FakeRow(table_schema="public", table_name="book"), + FakeRow(table_schema="other", table_name="thing"), ], ) session.add( "FROM INFORMATION_SCHEMA.COLUMNS", [ FakeRow( - TABLE_SCHEMA="public", - TABLE_NAME="author", - COLUMN_NAME="id", - COLUMN_DEFAULT=None, - IS_NULLABLE="NO", - DATA_TYPE="integer", - CHARACTER_MAXIMUM_LENGTH=None, - NUMERIC_PRECISION=32, - NUMERIC_SCALE=0, - IS_IDENTITY="YES", + table_schema="public", + table_name="author", + column_name="id", + column_default=None, + is_nullable="NO", + data_type="integer", + character_maximum_length=None, + numeric_precision=32, + numeric_scale=0, + is_identity="YES", ), FakeRow( - TABLE_SCHEMA="public", - TABLE_NAME="book", - COLUMN_NAME="author_id", - COLUMN_DEFAULT="0", - IS_NULLABLE="NO", - DATA_TYPE="integer", - CHARACTER_MAXIMUM_LENGTH=None, - NUMERIC_PRECISION=32, - NUMERIC_SCALE=0, - IS_IDENTITY="NO", + table_schema="public", + table_name="book", + column_name="author_id", + column_default="0", + is_nullable="NO", + data_type="integer", + character_maximum_length=None, + numeric_precision=32, + numeric_scale=0, + is_identity="NO", ), # Column for a table the reader never collected FakeRow( - TABLE_SCHEMA="ghost", - TABLE_NAME="x", - COLUMN_NAME="dropped", - COLUMN_DEFAULT=None, - IS_NULLABLE="YES", - DATA_TYPE="text", - CHARACTER_MAXIMUM_LENGTH=None, - NUMERIC_PRECISION=None, - NUMERIC_SCALE=None, - IS_IDENTITY="NO", + table_schema="ghost", + table_name="x", + column_name="dropped", + column_default=None, + is_nullable="YES", + data_type="text", + character_maximum_length=None, + numeric_precision=None, + numeric_scale=None, + is_identity="NO", ), ], ) @@ -82,17 +82,17 @@ def _populated_session() -> FakeSession: "INFORMATION_SCHEMA.TABLE_CONSTRAINTS", [ FakeRow( - TABLE_SCHEMA="public", - TABLE_NAME="author", - COLUMN_NAME="id", - CONSTRAINT_TYPE="PRIMARY KEY", + table_schema="public", + table_name="author", + column_name="id", + constraint_type="PRIMARY KEY", ), # missing column → silently ignored FakeRow( - TABLE_SCHEMA="public", - TABLE_NAME="author", - COLUMN_NAME="missing", - CONSTRAINT_TYPE="UNIQUE", + table_schema="public", + table_name="author", + column_name="missing", + constraint_type="UNIQUE", ), ], ) @@ -100,30 +100,30 @@ def _populated_session() -> FakeSession: "REFERENTIAL_CONSTRAINTS", [ FakeRow( - TABLE_SCHEMA="public", - TABLE_NAME="book", - COLUMN_NAME="author_id", - REF_SCHEMA="public", - REF_TABLE="author", - REF_COLUMN="id", + table_schema="public", + table_name="book", + column_name="author_id", + ref_schema="public", + ref_table="author", + ref_column="id", ), # column missing → ignored FakeRow( - TABLE_SCHEMA="public", - TABLE_NAME="book", - COLUMN_NAME="zzz", - REF_SCHEMA="public", - REF_TABLE="author", - REF_COLUMN="id", + table_schema="public", + table_name="book", + column_name="zzz", + ref_schema="public", + ref_table="author", + ref_column="id", ), # table missing FakeRow( - TABLE_SCHEMA="public", - TABLE_NAME="phantom", - COLUMN_NAME="x", - REF_SCHEMA="public", - REF_TABLE="author", - REF_COLUMN="id", + table_schema="public", + table_name="phantom", + column_name="x", + ref_schema="public", + ref_table="author", + ref_column="id", ), ], ) diff --git a/tox.ini b/tox.ini index e3bc4e1..e6623c9 100644 --- a/tox.ini +++ b/tox.ini @@ -84,7 +84,8 @@ max-line-length = 100 # E203: Whitespace before ':' # E501: Line too long # W503: Line break occurred before a binary operator -ignore = E203,E501,W503 +# E704: Multiple statements on one line (def) — idiomatic for Protocol stubs (`def foo(...): ...`) +ignore = E203,E501,W503,E704 [testenv:docs] basepython = python3 @@ -138,11 +139,14 @@ envdir = {toxworkdir}/functional deps = -r{toxinidir}/requirements.txt pytest pymssql + pymysql + cryptography oracledb psycopg2-binary -e {toxinidir} passenv = MSSQL_* + MYSQL_* ORACLE_* PG_* commands = @@ -162,16 +166,19 @@ commands = pytest -m oracle {posargs} {toxinidir}/tests/functional [testenv:functional-light] -# Functional tests excluding the Oracle subset: needs only MSSQL + Postgres. +# Functional tests excluding the Oracle subset: needs MSSQL + MySQL + Postgres. basepython = python3 envdir = {toxworkdir}/functional-light deps = -r{toxinidir}/requirements.txt pytest pymssql + pymysql + cryptography psycopg2-binary -e {toxinidir} passenv = MSSQL_* + MYSQL_* PG_* commands = pytest -m "functional and not oracle" {posargs} {toxinidir}/tests/functional