test: improve function test coverage

.docker/Dockerfile

@@ -21,6 +21,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
         unixodbc-dev \
     && rm -rf /var/lib/apt/lists/*
 
+RUN pip install --no-cache-dir tox
+
 RUN groupadd --gid ${GROUP_ID} dev \
     && useradd --uid ${USER_ID} --gid ${GROUP_ID} --create-home --shell /bin/bash dev
 

.docker/docker-compose.yml

@@ -1,3 +1,5 @@
+name: python-db2sql
+
 services:
   dev:
     build:
@@ -12,6 +14,7 @@ services:
       - ..:/workspace
     environment:
       DEVCONTAINER: "true"
+      PIPUSER: "0"
       HOME: /home/dev
       MSSQL_HOST: mssql
       MSSQL_PORT: "1433"
@@ -28,7 +31,16 @@ services:
       PG_USER: postgres
       PG_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
       PG_DATABASE: db2sqltarget
+      PG_SOURCE_SCHEMA: apptest
+      MYSQL_HOST: mysql
+      MYSQL_PORT: "3306"
+      MYSQL_USER: root
+      MYSQL_PASSWORD: ${MYSQL_ROOT_PASSWORD:-mysqlpw}
+      MYSQL_DATABASE: db2sqltest
     init: true
+    tty: true
+    stdin_open: true
+    command: ["sleep", "infinity"]
 
   mssql:
     profiles: ["functional"]
@@ -38,7 +50,7 @@ services:
       MSSQL_SA_PASSWORD: ${MSSQL_SA_PASSWORD:-Db2sqlTest!Strong}
       MSSQL_PID: Developer
     ports:
-      - "${MSSQL_PORT:-1433}:1433"
+      - "${MSSQL_PORT:-11433}:1433"
     volumes:
       - mssql-data:/var/opt/mssql
     healthcheck:
@@ -82,7 +94,7 @@ services:
       APP_USER: ${ORACLE_APP_USER:-apptest}
       APP_USER_PASSWORD: ${ORACLE_APP_PASSWORD:-apptestpw}
     ports:
-      - "${ORACLE_PORT:-1521}:1521"
+      - "${ORACLE_PORT:-11521}:1521"
     volumes:
       - ./oracle/init:/container-entrypoint-initdb.d:ro
       - oracle-data:/opt/oracle/oradata
@@ -93,6 +105,26 @@ services:
       retries: 60
       start_period: 60s
 
+  mysql:
+    profiles: ["functional"]
+    image: mysql:8.4
+    environment:
+      MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-mysqlpw}
+      MYSQL_DATABASE: db2sqltest
+    ports:
+      - "${MYSQL_PORT:-13306}:3306"
+    volumes:
+      - ./mysql/init:/docker-entrypoint-initdb.d:ro
+      - mysql-data:/var/lib/mysql
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - mysqladmin ping -h 127.0.0.1 -uroot -p"$$MYSQL_ROOT_PASSWORD" --silent
+      interval: 5s
+      timeout: 5s
+      retries: 30
+      start_period: 20s
+
   postgres:
     profiles: ["functional"]
     image: postgres:16-alpine
@@ -101,7 +133,7 @@ services:
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
       POSTGRES_DB: db2sqltarget
     ports:
-      - "${POSTGRES_PORT:-5432}:5432"
+      - "${PG_PORT:-15432}:5432"
     volumes:
       - ./postgres/init:/docker-entrypoint-initdb.d:ro
       - postgres-data:/var/lib/postgresql/data
@@ -113,5 +145,6 @@ services:
 
 volumes:
   mssql-data:
+  mysql-data:
   oracle-data:
   postgres-data:

.docker/mysql/init/01-schema.sql

@@ -0,0 +1,90 @@
+-- Functional-test fixture for MySQL.
+--
+-- The MySQLSourceReader uses the connection's database as the "schema"
+-- (MySQL has no notion of schema separate from database), so all fixtures
+-- live in the ``db2sqltest`` database created from MYSQL_DATABASE.
+--
+-- Coverage: every MySQL source type referenced by
+-- ``PostgresSqlEmitter.DEFAULT_TYPE_MAP`` that is native to MySQL, plus the
+-- standard relational fixture (author/book + foreign key + secondary index).
+
+CREATE DATABASE IF NOT EXISTS db2sqltest;
+USE db2sqltest;
+
+DROP TABLE IF EXISTS book;
+DROP TABLE IF EXISTS author;
+DROP TABLE IF EXISTS type_matrix;
+
+-- Type-coverage table -------------------------------------------------------
+CREATE TABLE type_matrix (
+    id              INT NOT NULL AUTO_INCREMENT,
+    c_bit           BIT(1)        NULL,
+    c_tinyint       TINYINT       NULL,
+    c_smallint      SMALLINT      NULL,
+    c_mediumint     MEDIUMINT     NULL,
+    c_int           INT           NULL,
+    c_bigint        BIGINT        NULL,
+    c_decimal       DECIMAL(12,4) NULL,
+    c_numeric       NUMERIC(18,6) NULL,
+    c_float         FLOAT         NULL,
+    c_double        DOUBLE        NULL,
+    c_char          CHAR(8)       NULL,
+    c_varchar       VARCHAR(64)   NULL,
+    c_text          TEXT          NULL,
+    c_mediumtext    MEDIUMTEXT    NULL,
+    c_longtext      LONGTEXT      NULL,
+    c_binary        BINARY(8)     NULL,
+    c_varbinary     VARBINARY(64) NULL,
+    c_blob          BLOB          NULL,
+    c_date          DATE          NULL,
+    c_time          TIME          NULL,
+    c_datetime      DATETIME      NULL,
+    c_timestamp     TIMESTAMP     NULL DEFAULT NULL,
+    c_json          JSON          NULL,
+    PRIMARY KEY (id)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- Relational mini-fixture (parallels mssql/oracle/postgres fixtures)
+CREATE TABLE author (
+    id          INT NOT NULL AUTO_INCREMENT,
+    name        VARCHAR(120) NOT NULL,
+    birth_year  INT NULL,
+    PRIMARY KEY (id)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+CREATE TABLE book (
+    id          INT NOT NULL AUTO_INCREMENT,
+    author_id   INT NOT NULL,
+    title       VARCHAR(200) NOT NULL,
+    PRIMARY KEY (id),
+    CONSTRAINT fk_book_author FOREIGN KEY (author_id) REFERENCES author (id)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+CREATE INDEX idx_book_title ON book (title);
+
+-- Representative payload (one populated row + one all-NULL row)
+INSERT INTO type_matrix
+    (c_bit, c_tinyint, c_smallint, c_mediumint, c_int, c_bigint,
+     c_decimal, c_numeric, c_float, c_double,
+     c_char, c_varchar, c_text, c_mediumtext, c_longtext,
+     c_binary, c_varbinary, c_blob,
+     c_date, c_time, c_datetime, c_timestamp, c_json)
+VALUES
+    (b'1', 7, 32100, 8388600, 2147483640, 9000000000000000000,
+     1234.5678, 1234567.890123, 1.5, 3.141592653589793,
+     'fixed', 'with accent é', 'long text payload', 'medium text', 'long text',
+     UNHEX('DEADBEEFCAFEBABE'), UNHEX('01020304'), UNHEX('05060708'),
+     '2024-01-31', '14:30:00', '2024-01-31 14:30:00',
+     '2024-01-31 14:30:00', JSON_OBJECT('k', 'v')),
+    (NULL, NULL, NULL, NULL, NULL, NULL,
+     NULL, NULL, NULL, NULL,
+     NULL, NULL, NULL, NULL, NULL,
+     NULL, NULL, NULL,
+     NULL, NULL, NULL, NULL, NULL);
+
+INSERT INTO author (name, birth_year) VALUES ('Alice', 1980);
+INSERT INTO author (name, birth_year) VALUES ('Bob', NULL);
+
+INSERT INTO book (author_id, title) VALUES (1, 'First');
+INSERT INTO book (author_id, title) VALUES (1, 'Second\'s ride');
+INSERT INTO book (author_id, title) VALUES (2, 'Bob book');

.docker/postgres/init/01-schema.sql

@@ -1,11 +1,93 @@
--- Target PostgreSQL database used by functional tests to validate a generated
--- dump. The dump is supposed to be self-sufficient (CREATE SCHEMA + DDL +
--- COPY/INSERT), so this init script only ensures the database exists and is
--- otherwise empty.
+-- Functional-test fixture for PostgreSQL.
+--
+-- The "db2sqltarget" database (created from POSTGRES_DB by the entrypoint) is
+-- used both as:
+--   * a TARGET database for dumps produced by db2sql (apply step) — left
+--     untouched outside the apptest schema below;
+--   * a SOURCE database whose ``apptest`` schema mirrors the mssql/oracle
+--     fixtures (type_matrix + author/book) so the PostgreSQL reader can be
+--     exercised by the functional suite.
+--
+-- The reader filters out pg_catalog/information_schema/pg_toast, so the
+-- ``apptest`` schema is the only one surfaced by collect_metadata().
+-- Note: types covered are the PG-native ones referenced by
+-- ``PostgresSqlEmitter.DEFAULT_TYPE_MAP`` (no MSSQL/Oracle-only types).
 
--- The database "db2sqltarget" is created from POSTGRES_DB by the entrypoint,
--- so nothing else is strictly required here. We keep the file so the volume
--- mount /docker-entrypoint-initdb.d is non-empty and to host any future
--- preconditioning (extensions, roles, etc.).
+CREATE SCHEMA IF NOT EXISTS apptest;
 
-SELECT 'postgres target ready' AS status;
+DROP TABLE IF EXISTS apptest.book;
+DROP TABLE IF EXISTS apptest.author;
+DROP TABLE IF EXISTS apptest.type_matrix;
+
+-- Type-coverage table -------------------------------------------------------
+CREATE TABLE apptest.type_matrix (
+    id                 INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    c_boolean          BOOLEAN,
+    c_smallint         SMALLINT,
+    c_integer          INTEGER,
+    c_bigint           BIGINT,
+    c_real             REAL,
+    c_double           DOUBLE PRECISION,
+    c_numeric          NUMERIC(18, 6),
+    c_decimal          DECIMAL(12, 4),
+    c_char             CHAR(8),
+    c_varchar          VARCHAR(64),
+    c_text             TEXT,
+    c_bytea            BYTEA,
+    c_date             DATE,
+    c_time             TIME,
+    c_timestamp        TIMESTAMP,
+    c_timestamptz      TIMESTAMP WITH TIME ZONE,
+    c_uuid             UUID,
+    c_json             JSON,
+    c_jsonb            JSONB,
+    c_xml              XML
+);
+
+-- Relational mini-fixture (parallels mssql/oracle/sqlite fixtures)
+CREATE TABLE apptest.author (
+    id          INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    name        VARCHAR(120) NOT NULL,
+    birth_year  INTEGER
+);
+
+CREATE TABLE apptest.book (
+    id          INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    author_id   INTEGER NOT NULL,
+    title       VARCHAR(200) NOT NULL,
+    CONSTRAINT fk_book_author FOREIGN KEY (author_id) REFERENCES apptest.author (id)
+);
+
+CREATE INDEX idx_book_title ON apptest.book (title);
+
+-- Representative payload (one populated row + one all-NULL row)
+INSERT INTO apptest.type_matrix
+    (c_boolean, c_smallint, c_integer, c_bigint,
+     c_real, c_double, c_numeric, c_decimal,
+     c_char, c_varchar, c_text, c_bytea,
+     c_date, c_time, c_timestamp, c_timestamptz,
+     c_uuid, c_json, c_jsonb, c_xml)
+VALUES
+    (TRUE, 32100, 2147483640, 9000000000000000000,
+     1.5, 3.141592653589793, 1234567.890123, 1234.5678,
+     'fixed', 'with accent é', 'long text payload', '\xDEADBEEFCAFEBABE',
+     DATE '2024-01-31', TIME '14:30:00',
+     TIMESTAMP '2024-01-31 14:30:00',
+     TIMESTAMP WITH TIME ZONE '2024-01-31 14:30:00+01',
+     '11111111-2222-3333-4444-555555555555',
+     '{"k": "v"}', '{"k": "v"}',
+     XMLPARSE(DOCUMENT '<root><k>v</k></root>')),
+    (NULL, NULL, NULL, NULL,
+     NULL, NULL, NULL, NULL,
+     NULL, NULL, NULL, NULL,
+     NULL, NULL, NULL, NULL,
+     NULL, NULL, NULL, NULL);
+
+INSERT INTO apptest.author (name, birth_year) VALUES ('Alice', 1980);
+INSERT INTO apptest.author (name, birth_year) VALUES ('Bob', NULL);
+
+INSERT INTO apptest.book (author_id, title) VALUES (1, 'First');
+INSERT INTO apptest.book (author_id, title) VALUES (1, 'Second''s ride');
+INSERT INTO apptest.book (author_id, title) VALUES (2, 'Bob book');
+
+SELECT 'postgres apptest fixture loaded' AS status;

.github/workflows/ci.yml

@@ -75,7 +75,7 @@ jobs:
         run: sphinx-build -b html -W --keep-going docs docs/_build/html
 
   functional:
-    name: Functional tests (MSSQL + Postgres, no Oracle)
+    name: Functional tests (MSSQL + MySQL + Postgres, no Oracle)
     runs-on: ubuntu-latest
     # Light stack (~2-3 min boot). Runs on every push to main, and on PRs
     # carrying the `run-functional` label. Oracle is on-demand only — see
@@ -98,6 +98,12 @@ jobs:
       POSTGRES_PASSWORD: postgres
       PG_PASSWORD: postgres
       PG_DATABASE: db2sqltarget
+      MYSQL_HOST: localhost
+      MYSQL_PORT: "3306"
+      MYSQL_USER: root
+      MYSQL_ROOT_PASSWORD: mysqlpw
+      MYSQL_PASSWORD: mysqlpw
+      MYSQL_DATABASE: db2sqltest
     steps:
       - uses: actions/checkout@v4
 
@@ -112,8 +118,11 @@ jobs:
       - name: Install package and test dependencies
         run: pip install -e ".[all]" pytest
 
-      - name: Start docker stack (mssql + postgres) and wait until healthy
-        run: docker compose -f .docker/docker-compose.yml --profile functional up -d --wait
+      - name: Start docker stack (mssql + mysql + postgres) and wait until healthy
+        run: docker compose -f .docker/docker-compose.yml --profile functional up -d --wait mssql mysql postgres
+
+      - name: Apply MSSQL init schema
+        run: docker compose -f .docker/docker-compose.yml --profile functional run --rm mssql-init
 
       - name: Run functional tests (excluding Oracle)
         run: pytest -m "functional and not oracle" tests/functional -v

.github/workflows/release-binaries.yml

@@ -39,19 +39,16 @@ jobs:
           - label: windows-x86_64
             os: windows-latest
             python: "3.12"
-            shell: pwsh
             archive-glob: "installer/dist/db2sql-*-windows-*.zip"
             container: ""
           - label: linux-x86_64
             os: ubuntu-latest
             python: "3.12"
-            shell: bash
             archive-glob: "installer/dist/db2sql-*-linux-*.tar.gz"
             container: "python:3.12-bookworm"
           - label: macos-arm64
             os: macos-14  # Apple Silicon runner
             python: "3.12"
-            shell: bash
             archive-glob: "installer/dist/db2sql-*-macos-*.tar.gz"
             container: ""
 
@@ -79,7 +76,6 @@ jobs:
           rm -rf /var/lib/apt/lists/*
 
       - name: Show toolchain info
-        shell: ${{ matrix.shell }}
         run: |
           python --version
           python -c "import platform; print('platform:', platform.platform()); print('machine:', platform.machine())"
@@ -89,14 +85,12 @@ jobs:
         run: ldd --version | head -1
 
       - name: Install build dependencies
-        shell: ${{ matrix.shell }}
         run: |
           python -m pip install --upgrade pip wheel
           python -m pip install -e ".[all]"
           python -m pip install "pyinstaller>=6"
 
       - name: Build standalone binary
-        shell: ${{ matrix.shell }}
         run: python installer/build.py --archive
 
       - name: Upload build artifact