diff --git a/conf/jvm.conf b/conf/jvm.conf index fcc3742..86d3aa6 100644 --- a/conf/jvm.conf +++ b/conf/jvm.conf @@ -3,6 +3,8 @@ -Dfile.encoding=UTF-8 -Duser.timezone=UTC +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager + -Xms8g -Xmx60g diff --git a/conf/pixels-sink.aws.properties b/conf/pixels-sink.aws.properties index cd9f98d..a6ef8b3 100644 --- a/conf/pixels-sink.aws.properties +++ b/conf/pixels-sink.aws.properties @@ -1,5 +1,8 @@ # engine | kafka | storage sink.datasource=storage +sink.storage.mode=stream +# CDC dialect for envelope normalization: mysql | postgresql +sink.debezium.dialect=postgresql # -1 means no limit, Only implement in retina sink mode yet: sink.datasource.rate.limit=200000 sink.monitor.report.file=/home/ubuntu/pixels-sink/result1k2_feb/rate_8192tile_3.csv @@ -20,11 +23,11 @@ sink.retina.trans.request.batch.size=100 sink.trans.mode=batch sink.monitor.report.enable=true # trino for freshness query -trino.url=jdbc:trino://realtime-pixels-coordinator:8080/pixels/pixels_bench -# trino.url=jdbc:trino://realtime-pixels-coordinator:8080/pixels/pixels_bench_sf10x -trino.user=pixels -trino.password=password -trino.parallel=1 +sink.query.url=jdbc:trino://realtime-pixels-coordinator:8080/pixels/pixels_bench +# sink.query.url=jdbc:trino://realtime-pixels-coordinator:8080/pixels/pixels_bench_sf10x +sink.query.user=pixels +sink.query.password=password +sink.query.parallel=1 # row or txn or embed sink.monitor.freshness.level=embed sink.monitor.freshness.verbose=true @@ -41,8 +44,7 @@ bootstrap.servers=realtime-kafka-2:29092 group.id=3078 auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -#value.deserializer=io.pixelsdb.pixels.writer.deserializer.RowChangeEventAvroDeserializer -value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventJsonDeserializer +sink.kafka.value.format=json # Topic & Database Config topic.prefix=postgresql.oltp_server consumer.capture_database=pixels_bench_sf1x @@ -78,8 +80,6 @@ sink.flink.server.port=9091 sink.registry.url=http://localhost:8080/apis/registry/v2 # Transaction Config transaction.topic.suffix=transaction -#transaction.topic.value.deserializer=io.pixelsdb.pixels.writer.deserializer.TransactionAvroMessageDeserializer -transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.TransactionJsonMessageDeserializer sink.trans.batch.size=100 # Sink Metrics @@ -90,7 +90,6 @@ sink.monitor.freshness.interval=1000 # Interact with other rpc sink.rpc.enable=true -sink.rpc.mock.delay=20 # debezium engine config debezium.name=testEngine debezium.connector.class=io.debezium.connector.postgresql.PostgresConnector diff --git a/conf/pixels-sink.ch.properties b/conf/pixels-sink.ch.properties index c9a4fbe..7312a01 100644 --- a/conf/pixels-sink.ch.properties +++ b/conf/pixels-sink.ch.properties @@ -1,5 +1,8 @@ # engine | kafka | storage sink.datasource=storage +sink.storage.mode=stream +# CDC dialect for envelope normalization: mysql | postgresql +sink.debezium.dialect=postgresql sink.mode=retina #sink.datasource=engine #sink.mode=proto @@ -23,10 +26,10 @@ sink.trans.mode=batch # sink.trans.mode=record sink.monitor.report.enable=true # trino for freshness query -trino.url=jdbc:trino://realtime-pixels-coordinator:8080/pixels/pixels_bench -trino.user=pixels -trino.password=password -trino.parallel=1 +sink.query.url=jdbc:trino://realtime-pixels-coordinator:8080/pixels/pixels_bench +sink.query.user=pixels +sink.query.password=password +sink.query.parallel=1 # row or txn or embed sink.monitor.freshness.level=embed sink.monitor.freshness.verbose=true @@ -42,8 +45,7 @@ bootstrap.servers=realtime-kafka-2:29092 group.id=3078 auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -#value.deserializer=io.pixelsdb.pixels.writer.deserializer.RowChangeEventAvroDeserializer -value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventJsonDeserializer +sink.kafka.value.format=json # Topic & Database Config topic.prefix=postgresql.oltp_server consumer.capture_database=pixels_bench_sf1x @@ -75,8 +77,6 @@ sink.flink.server.port=9091 sink.registry.url=http://localhost:8080/apis/registry/v2 # Transaction Config transaction.topic.suffix=transaction -#transaction.topic.value.deserializer=io.pixelsdb.pixels.writer.deserializer.TransactionAvroMessageDeserializer -transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.TransactionJsonMessageDeserializer sink.trans.batch.size=100 # Sink Metrics @@ -87,7 +87,6 @@ sink.monitor.freshness.interval=1000 # Interact with other rpc sink.rpc.enable=true -sink.rpc.mock.delay=20 # debezium engine config debezium.name=testEngine debezium.connector.class=io.debezium.connector.postgresql.PostgresConnector diff --git a/conf/pixels-sink.flink.properties.chbench b/conf/pixels-sink.flink.properties.chbench index 6c87406..c862997 100644 --- a/conf/pixels-sink.flink.properties.chbench +++ b/conf/pixels-sink.flink.properties.chbench @@ -1,5 +1,6 @@ # engine | kafka | storage sink.datasource=storage +sink.storage.mode=stream # -1 means no limit, Only implement in retina sink mode yet sink.datasource.rate.limit=2000 # Sink Config: retina | csv | proto | flink | none @@ -11,10 +12,10 @@ sink.monitor.report.enable=true sink.monitor.report.file=/home/ubuntu/pixels-sink/result_delta/rate_test.csv sink.monitor.freshness.file=/home/ubuntu/pixels-sink/result_delta/fresh_batch10k.csv # trino for freshness query -trino.url=jdbc:trino://realtime-pixels-coordinator:8080/delta_lake/chbenchmark_w10000 -trino.user=pixels -trino.password=password -trino.parallel=1 +sink.query.url=jdbc:trino://realtime-pixels-coordinator:8080/delta_lake/chbenchmark_w10000 +sink.query.user=pixels +sink.query.password=password +sink.query.parallel=1 # row or txn or embed sink.monitor.freshness.level=embed sink.monitor.freshness.embed.warmup=10 diff --git a/conf/pixels-sink.flink.properties.hybench b/conf/pixels-sink.flink.properties.hybench index eb03619..d809dc6 100644 --- a/conf/pixels-sink.flink.properties.hybench +++ b/conf/pixels-sink.flink.properties.hybench @@ -1,5 +1,6 @@ # engine | kafka | storage sink.datasource=storage +sink.storage.mode=stream # -1 means no limit, Only implement in retina sink mode yet sink.datasource.rate.limit=500 sink.datasource.rate.limit.type=guava @@ -12,10 +13,10 @@ sink.monitor.report.enable=true sink.monitor.report.file=/home/ubuntu/pixels-sink/result_delta/rate_hybench_rate500.csv sink.monitor.freshness.file=/home/ubuntu/pixels-sink/result_delta/fresh_hybench_rate500.csv # trino for freshness query -trino.url=jdbc:trino://realtime-pixels-coordinator:8080/delta_lake/hybench_sf1000 -trino.user=pixels -trino.password=password -trino.parallel=1 +sink.query.url=jdbc:trino://realtime-pixels-coordinator:8080/delta_lake/hybench_sf1000 +sink.query.user=pixels +sink.query.password=password +sink.query.parallel=1 # row or txn or embed sink.monitor.freshness.level=embed sink.monitor.freshness.embed.warmup=10 diff --git a/conf/pixels-sink.hudi.properties b/conf/pixels-sink.hudi.properties index 8ab4983..801922c 100644 --- a/conf/pixels-sink.hudi.properties +++ b/conf/pixels-sink.hudi.properties @@ -1,5 +1,6 @@ # engine | kafka | storage sink.datasource=storage +sink.storage.mode=stream # -1 means no limit, Only implement in retina sink mode yet sink.datasource.rate.limit=2000 # Sink Config: retina | csv | proto | flink | none @@ -10,11 +11,11 @@ sink.trans.mode=batch sink.monitor.report.enable=true sink.monitor.report.file=/home/ubuntu/pixels-sink/result_delta/rate_test.csv sink.monitor.freshness.file=/home/ubuntu/pixels-sink/result_delta/fresh_batch10k.csv -# trino for freshness query -trino.url=jdbc:hive2://172.31.16.214:10000/hudi_hybench_sf1333_2 -trino.user=pixels -trino.password= -trino.parallel=1 +# HiveServer2 for Hudi RO incremental freshness (requires: mvn -Phudi-hive package) +sink.query.url=jdbc:hive2://172.31.16.214:10000/hudi_hybench_sf1333_2 +sink.query.user=pixels +sink.query.password= +sink.query.parallel=1 # row or txn or embed sink.monitor.freshness.level=embed sink.monitor.freshness.embed.warmup=0 diff --git a/conf/pixels-sink.mysql.properties b/conf/pixels-sink.mysql.properties new file mode 100644 index 0000000..58e5e1e --- /dev/null +++ b/conf/pixels-sink.mysql.properties @@ -0,0 +1,102 @@ +# engine | kafka | storage +sink.datasource=engine +sink.datasource.engine.format=connect +# CDC dialect for envelope normalization: mysql | postgresql +sink.debezium.dialect=mysql +sink.storage.mode=stream +# -1 means no limit, Only implement in retina sink mode yet +sink.datasource.rate.limit=100000 +# Sink Config: retina | csv | proto | flink | none +sink.mode=retina +sink.retina.client=8 +sink.retina.log.queue=false +## batch or single or record, batch is recommend. record is faster, but doesn't have ACID feature +sink.trans.mode=batch +sink.monitor.report.enable=true +sink.monitor.report.file=/home/ubuntu/pixels-sink/resulti7i/100k_rate_mysql.csv +sink.monitor.freshness.file=/home/ubuntu/pixels-sink/resulti7i/100k_freshness_mysql.csv +# trino for freshness query +sink.query.url=jdbc:trino://realtime-kafka-2:8080/pixels/pixels_bench_sf10x +sink.query.user=pixels +sink.query.password=password +sink.query.parallel=8 +# row or txn or embed +sink.monitor.freshness.level=embed +sink.monitor.freshness.embed.warmup=10 +sink.monitor.freshness.embed.static=false +sink.monitor.freshness.embed.snapshot=true +sink.monitor.freshness.embed.tablelist=binlog_test +sink.monitor.freshness.verbose=true +sink.monitor.freshness.timestamp=true +sink.storage.loop=true +# Kafka Config +bootstrap.servers=realtime-kafka-2:29092 +group.id=3078 +auto.offset.reset=earliest +key.deserializer=org.apache.kafka.common.serialization.StringDeserializer +sink.kafka.value.format=json +# Topic & Database Config +topic.prefix=mysql-cdc +consumer.capture_database=cdc_verify +consumer.include_tables= +sink.csv.path=./data +sink.csv.enable_header=false +## Retina Config +sink.retina.embedded=false +# stub or stream +sink.retina.mode=stream +sink.remote.host=localhost +sink.remote.port=29422 +sink.timeout.ms=5000 +sink.flush.interval.ms=50 +sink.flush.batch.size=10 +sink.max.retries=3 +## writer commit +# sync or async +sink.commit.method=sync +sink.commit.batch.size=10 +sink.commit.batch.worker=32 +sink.commit.batch.delay=3000 +## Proto Config +sink.proto.dir=file:///home/ubuntu/disk1/hybench/ +sink.proto.data=hybench10_10 +sink.proto.maxRecords=100000 +## Flink Config +sink.flink.server.port=9091 +## Schema Registry +sink.registry.url=http://localhost:8080/apis/registry/v2 +# Transaction Config +transaction.topic.suffix=transaction +sink.trans.batch.size=100 + +# Sink Metrics +sink.monitor.enable=true +sink.monitor.port=9464 +sink.monitor.report.interval=10000 +sink.monitor.freshness.interval=1000 + +# Interact with other rpc +sink.rpc.enable=true +# debezium engine config +debezium.name=mysql-cdc-engine +debezium.connector.class=io.debezium.connector.mysql.MySqlConnector +debezium.provide.transaction.metadata=true +debezium.offset.storage=org.apache.kafka.connect.storage.FileOffsetBackingStore +debezium.offset.storage.file.filename=/tmp/pixels-sink-mysql-offsets.dat +debezium.offset.flush.interval.ms=60000 +debezium.schema.history.internal=io.debezium.storage.file.history.FileSchemaHistory +debezium.schema.history.internal.file.filename=/tmp/pixels-sink-mysql-schema-history.dat +debezium.database.hostname=tdsql +debezium.database.port=6060 +debezium.database.user=tdsql +debezium.database.password=tdsql123 +debezium.database.server.id=5401 +debezium.database.include.list=cdc_verify +debezium.snapshot.mode=never +debezium.key.converter=org.apache.kafka.connect.json.JsonConverter +debezium.value.converter=org.apache.kafka.connect.json.JsonConverter +debezium.topic.prefix=mysql-cdc +debezium.transforms=topicRouting +debezium.transforms.topicRouting.type=org.apache.kafka.connect.transforms.RegexRouter +debezium.transforms.topicRouting.regex=mysql-cdc\\.cdc_verify\\.(.*) +debezium.transforms.topicRouting.replacement=mysql-cdc.cdc_verify.$1 diff --git a/conf/pixels-sink.pg.properties b/conf/pixels-sink.pg.properties index effc171..741aa2c 100644 --- a/conf/pixels-sink.pg.properties +++ b/conf/pixels-sink.pg.properties @@ -1,5 +1,9 @@ # engine | kafka | storage sink.datasource=engine +sink.datasource.engine.format=connect +# CDC dialect for envelope normalization: mysql | postgresql +sink.debezium.dialect=postgresql +sink.storage.mode=stream # -1 means no limit, Only implement in retina sink mode yet sink.datasource.rate.limit=100000 # Sink Config: retina | csv | proto | flink | none @@ -12,11 +16,11 @@ sink.monitor.report.enable=true sink.monitor.report.file=/home/ubuntu/pixels-sink/resulti7i/100k_rate_2.csv sink.monitor.freshness.file=/home/ubuntu/pixels-sink/resulti7i/100k_freshness_2.csv # trino for freshness query -trino.url=jdbc:trino://realtime-kafka-2:8080/pixels/pixels_bench_sf10x -# trino.url=jdbc:trino://realtime-pixels-coordinator:8080/pixels/pixels_bench_sf10x -trino.user=pixels -trino.password=password -trino.parallel=8 +sink.query.url=jdbc:trino://realtime-kafka-2:8080/pixels/pixels_bench_sf10x +# sink.query.url=jdbc:trino://realtime-pixels-coordinator:8080/pixels/pixels_bench_sf10x +sink.query.user=pixels +sink.query.password=password +sink.query.parallel=8 # row or txn or embed sink.monitor.freshness.level=embed sink.monitor.freshness.embed.warmup=10 @@ -31,8 +35,7 @@ bootstrap.servers=realtime-kafka-2:29092 group.id=3078 auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -#value.deserializer=io.pixelsdb.pixels.writer.deserializer.RowChangeEventAvroDeserializer -value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventJsonDeserializer +sink.kafka.value.format=json # Topic & Database Config topic.prefix=postgresql.oltp_server consumer.capture_database=pixels_bench_sf1x @@ -66,8 +69,6 @@ sink.flink.server.port=9091 sink.registry.url=http://localhost:8080/apis/registry/v2 # Transaction Config transaction.topic.suffix=transaction -#transaction.topic.value.deserializer=io.pixelsdb.pixels.writer.deserializer.TransactionAvroMessageDeserializer -transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.TransactionJsonMessageDeserializer sink.trans.batch.size=100 # Sink Metrics @@ -78,7 +79,6 @@ sink.monitor.freshness.interval=1000 # Interact with other rpc sink.rpc.enable=true -sink.rpc.mock.delay=20 # debezium engine config debezium.name=testEngine debezium.connector.class=io.debezium.connector.postgresql.PostgresConnector diff --git a/docs/configuration.md b/docs/configuration.md index 845af65..79cd195 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -12,14 +12,18 @@ Values are loaded by `PixelsSinkConfig` and mapped from keys in the properties f | --- | --- | --- | | `sink.datasource` | `engine` | Source type: `engine`, `kafka`, or `storage`. | | `sink.mode` | `retina` | Sink type: `retina`, `csv`, `proto`, `flink`, or `none`. | +| `sink.datasource.decode.threads` | `4` | Decode thread pool size. Engine uses `StreamOrderedDecoder` (multi logical stream); Storage uses `OrderedBatchDecoder` (single-key batch). | +| `sink.datasource.engine.format` | `connect` | Engine wire format. Only `connect` is allowed; other values fail fast on resolve. | | `sink.datasource.rate.limit` | `-1` | Rate limit for source ingestion. `-1` disables. | | `sink.datasource.rate.limit.type` | `semaphore` | Rate limiter type used by `FlushRateLimiterFactory`. 'guava' or 'semaphore'| ### Notes on `sink.datasource` -- `engine` reads CDC logs directly from Debezium Engine. +- `engine` reads CDC logs directly from Debezium Engine. Currently requires `sink.datasource.engine.format=connect`. Json/Avro Engine paths are reserved and will reuse the same `conversion.debezium` converter interfaces later. - `storage` reads CDC logs from files dumped by `sink.proto` output; schema reference: [sink.proto](https://github.com/pixelsdb/pixels/blob/master/proto/sink.proto). - `kafka` reads from a set of Kafka topics; this mode is deprecated and not actively tested. +- Engine and Kafka records are normalized to the canonical `SinkProto` contract before reaching writers. +- Storage row records use canonical `SinkProto`; `SourceInfo.schema` may be empty for MySQL. ### Notes on `sink.mode` @@ -50,10 +54,16 @@ Notes on `sink.trans.mode`: | Key | Default | Notes | | --- | --- | --- | +| `sink.datasource.engine.format` | `connect` | Only `connect` is runnable today. Non-`connect` values fail fast. | +| `sink.debezium.dialect` | none | Sink-side CDC dialect for envelope normalization: `mysql` or `postgresql`. Used by Engine and Kafka conversion. If unset, falls back to inferring from `debezium.connector.class`. | | `debezium.name` | none | Engine name. | -| `debezium.connector.class` | none | Connector class, e.g. PostgreSQL connector. | +| `debezium.connector.class` | none | Debezium Engine connector class only (which DB the engine connects to). Not a Kafka setting. | | `debezium.*` | none | Standard Debezium engine properties. | +See `conf/pixels-sink.mysql.properties` for a TDSQL MySQL CDC example. + +The Debezium Connector reads the database Binlog or WAL. The local `conversion.debezium` package only converts envelopes already emitted by Debezium (`connect` / `json` / `avro` plus shared `support` / `dialect`). + ### Retina Sink | Key | Default | Notes | @@ -89,6 +99,7 @@ Notes on `sink.trans.mode`: | `sink.proto.dir` | required | Proto output or input directory. | | `sink.proto.data` | `data` | Data set name. | | `sink.proto.maxRecords` | `100000` | Max records per file. | +| `sink.storage.mode` | `stream` | Storage read mode: `stream` reads records incrementally; `memory` preloads all records before replay. | | `sink.storage.loop` | `false` | Whether to loop over stored files. | ### Flink Sink @@ -108,15 +119,27 @@ Kafka source is deprecated. | `group.id` | required | Consumer group id. | | `auto.offset.reset` | none | Standard Kafka consumer property. | | `key.deserializer` | `org.apache.kafka.common.serialization.StringDeserializer` | Kafka key deserializer. | -| `value.deserializer` | `io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventJsonDeserializer` | Kafka value deserializer for row events. | +| `sink.kafka.value.format` | `json` | Envelope format: `json` or `avro`. Kafka sources assemble `conversion.debezium` converters from this key. | +| `sink.debezium.dialect` | none | Upstream CDC dialect (`mysql` / `postgresql`). Required for Kafka transaction decoding; row events can also infer from `source.connector` when unset. | | `topic.prefix` | required | Topic prefix for table events. | | `consumer.capture_database` | required | Database name used to build topic names. | | `consumer.include_tables` | empty | Comma-separated table list, empty means all. | | `transaction.topic.suffix` | `transaction` | Suffix appended to transaction topics. | -| `transaction.topic.value.deserializer` | `io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventJsonDeserializer` | Deserializer for transaction messages. | | `transaction.topic.group_id` | `transaction_consumer` | Consumer group for transaction topic. | | `sink.registry.url` | required | Avro Schema registry endpoint. | +**Legacy Kafka deserializer migration** + +Prefer `sink.kafka.value.format`. Old keys `value.deserializer` and +`transaction.topic.value.deserializer` are no longer SPI; `PixelsSinkConfig` +migrates them at startup: + +| Condition | Behavior | +| --- | --- | +| `sink.kafka.value.format` is set; legacy keys still present | Warn and ignore legacy keys | +| Format unset; legacy class name maps to Json/Avro | Infer format and warn | +| Row/tx inferences conflict, or class name unrecognized | Fail fast | + **Reserved Configuration** | Key | Default | Notes | @@ -124,7 +147,6 @@ Kafka source is deprecated. | `sink.remote.host` | `localhost` | Sink server host. | | `sink.remote.port` | `9090` | Sink server port. | | `sink.rpc.enable` | `false` | Enable RPC simulation (for development). | -| `sink.rpc.mock.delay` | `0` | Artificial delay in ms. | **Monitoring and Metrics** @@ -146,13 +168,13 @@ Kafka source is deprecated. | `sink.monitor.freshness.verbose` | `false` | Verbose freshness logging. | | `sink.monitor.freshness.timestamp` | `false` | Include timestamps. | -Note: In the Retina paper experiments, `sink.monitor.freshness.level=embed` is used to query freshness from Trino. This requires the last column of each table to be `freshness_ts`. +Note: In the Retina paper experiments, `sink.monitor.freshness.level=embed` queries freshness through JDBC. This requires the last column of each table to be `freshness_ts`. Trino is available in the default build; HiveServer2 support for Hudi requires building with `-Phudi-hive`. -**Freshness Trino Settings** +**Freshness Query Settings** | Key | Default | Notes | | --- | --- | --- | -| `trino.url` | required for Trino-based freshness | JDBC URL. | -| `trino.user` | required for Trino-based freshness | Username. | -| `trino.password` | required for Trino-based freshness | Password. | -| `trino.parallel` | `1` | Parallel query count. | +| `sink.query.url` | required for embedded freshness | Trino or HiveServer2 JDBC URL. | +| `sink.query.user` | required for embedded freshness | Username. | +| `sink.query.password` | empty | Password. | +| `sink.query.parallel` | `1` | Parallel query count. | diff --git a/docs/overview.md b/docs/overview.md index c321e20..952cfa8 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -15,86 +15,83 @@ Pixels Sink uses a multi-stage pipeline. Each stage communicates via producer/co - Receives a `ConfigFactory` directly and builds the same sink pipeline. **Source** -The source stage pulls events and forwards raw payloads to providers. +The source stage owns transport lifecycle (Engine / Kafka / Storage). Payload +conversion is separate from transport: + +- Parallel decode infrastructure lives in `util.concurrent` + (`DecodeExecutors`, `StreamOrderedDecoder`, `OrderedBatchDecoder`) and is + sized by `sink.datasource.decode.threads`. +- Debezium envelopes (Engine Connect / Kafka JSON / Kafka Avro) convert through + `conversion.debezium` (`connect` / `json` / `avro` + `support` / `dialect`). +- Storage sink-proto bytes convert through `conversion.sinkproto` and stay + independent of the Debezium package. +- Engine event classification stays in `source.engine` (`ConnectEventClassifier`, + `DebeziumRecordType`). + +**Decode ordering (not enhanced beyond historical contract)** + +| Guaranteed | Not guaranteed | +| --- | --- | +| Same table (Engine) / same Storage key: FIFO relative to source scan or enqueue order | Cross-table global order | +| TX events FIFO on the TX stream | TX BEGIN/END vs row events global arrival order | +| | Same-table cross-PK-bucket write total order; `totalOrder` / `dataCollectionOrder` reorder | + +Two decoders have different roles (not a shared abstraction): + +| Component | Role | Stream / shard key | +| --- | --- | --- | +| `StreamOrderedDecoder` | Engine: ordered publish across multiple logical streams in a mixed batch | ROW = `SchemaTableName`; TX = singleton stream key | +| `OrderedBatchDecoder` | Storage: parallel decode inside one key consumer, then deliver in input order | Sharding already done by `queueMap`; decoder does not re-shard | **Source Inputs** | Source Type | Description | Related Config | | --- | --- | --- | -| `engine` | Debezium Engine reads WAL/binlog directly from a database | `debezium.*` | -| `kafka` | Kafka consumer reads change events from topics | `bootstrap.servers`, `group.id`, `topic.*` | +| `engine` | Debezium Engine reads WAL/binlog directly from a database | `debezium.*`, `sink.datasource.engine.format` | +| `kafka` | Kafka consumer reads change events from topics | `bootstrap.servers`, `group.id`, `topic.*`, `sink.kafka.value.format` | | `storage` | Reads from Pixels storage files containing serialized sink proto records | `sink.proto.*`, `sink.storage.loop` | **Source Outputs** -- The source does not parse events. It forwards raw records to providers. +- Kafka, Engine, and Storage sources publish only canonical + `RowChangeEvent` or `SinkProto.TransactionMetadata` objects. +- Protocol conversion is implemented in `conversion`; pipeline queues never receive + Kafka bytes, Engine `SourceRecord`, or Storage `ByteBuffer`. -**Provider** -Providers convert source records into Pixels events. +**Pipeline Queue** +Pipelines own bounded canonical-event queues. The queue provides backpressure, +ordered delivery, and lifecycle management without knowing the source protocol. ```mermaid classDiagram direction TB - class EventProvider~SOURCE_RECORD_T, TARGET_RECORD_T~ { - +run() + class BlockingBoundedQueue~T~ { + +put(T value) + +take() +close() - +processLoop() - +convertToTargetRecord() - +recordSerdEvent() - +putRawEvent() - +getRawEvent() - +pollRawEvent() - +putTargetEvent() - +getTargetEvent() } - class TableEventProvider~SOURCE_RECORD_T~ { + class TablePipeline { + +publish(RowChangeEvent) } - class TableEventEngineProvider~T~ { + class TransactionPipeline { + +publish(TransactionMetadata) } - class TableEventKafkaProvider~T~ { - } - class TableEventStorageProvider~T~ { + class TablePipelineManager { + +route(RowChangeEvent) } - class TransactionEventProvider~SOURCE_RECORD_T~ { - } - class TransactionEventEngineProvider~T~ { - } - class TransactionEventKafkaProvider~T~ { - } - class TransactionEventStorageProvider~T~ { - } - - EventProvider <|-- TableEventProvider - EventProvider <|-- TransactionEventProvider - - TableEventProvider <|-- TableEventEngineProvider - TableEventProvider <|-- TableEventKafkaProvider - TableEventProvider <|-- TableEventStorageProvider - - TransactionEventProvider <|-- TransactionEventEngineProvider - TransactionEventProvider <|-- TransactionEventKafkaProvider - TransactionEventProvider <|-- TransactionEventStorageProvider + TablePipelineManager --> TablePipeline + TablePipeline --> BlockingBoundedQueue + TransactionPipeline --> BlockingBoundedQueue ``` -Example mappings: - -| Provider | Source Type | Target Type | -| --- | --- | --- | -| `TableEventEngineProvider` | Debezium Struct | `RowChangeEvent` | -| `TableEventKafkaProvider` | Kafka topic | `RowChangeEvent` | -| `TableEventStorageProvider` | Proto bytes | `RowChangeEvent` | -| `TransactionEventEngineProvider` | Debezium Struct | `SinkProto.TransactionMetadata` | -| `TransactionEventKafkaProvider` | Kafka topic | `SinkProto.TransactionMetadata` | -| `TransactionEventStorageProvider` | Proto bytes | `SinkProto.TransactionMetadata` | - **Processor** -Processors pull events from providers and write to the sink writers. +Processors pull events from pipeline queues and write to the sink writers. -- `TableProcessor` instances are created by `TableProviderAndProcessorPipelineManager`. +- `TableProcessor` instances are created by `TablePipelineManager`. - There is typically one `TableProcessor` per table to maintain per-table ordering. -- `TransactionProcessor` is a singleton. +- `TransactionPipeline` owns the transaction queue and `TransactionProcessor`. **Writer** Writers implement `PixelsSinkWriter`: diff --git a/docs/usage.md b/docs/usage.md index 5beaa54..d27ead6 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -25,7 +25,8 @@ mvn -q -DskipTests exec:java \ **Pipeline Summary** - A source reads change events from Debezium Engine, Kafka, or storage files. -- A provider converts source records into Pixels events. +- Conversion normalizes source records into canonical Pixels events. +- Pipelines route events through bounded queues to their processors. - Processors enforce ordering by table and write to the configured sink. - The sink writer persists the events to Retina, CSV, Proto, Flink, or a no-op sink. diff --git a/pom.xml b/pom.xml index 2b965f2..f9203eb 100644 --- a/pom.xml +++ b/pom.xml @@ -34,14 +34,40 @@ true 0.9.0 - 3.8.0 - 5.8 + 3.9.1 1.18.42 3.2.3.Final + 2.16.2 + 2.6.2.Final + 1.11.3 + 3.6.1 + 3.12.0 + + 440 + 2.3.9 1.4.13 - 440 + + + + + com.fasterxml.jackson + jackson-bom + ${dep.jackson.version} + pom + import + + + io.netty + netty-bom + ${dep.netty.version} + pom + import + + + + io.pixelsdb @@ -54,56 +80,82 @@ true - io.pixelsdb - pixels-retina + io.etcd + jetcd-core true - test + - org.apache.hive - hive-jdbc - 2.3.9 + io.netty + netty-codec-http2 + runtime - io.etcd - jetcd-core - true + io.netty + netty-handler-proxy + runtime io.netty - netty-all + netty-resolver-dns + runtime - io.grpc - grpc-netty + io.netty + netty-transport-native-unix-common + runtime io.trino trino-jdbc - ${trino.version} + runtime - com.alibaba - fastjson - true + org.apache.logging.log4j + log4j-api + runtime org.apache.logging.log4j log4j-core true + runtime org.apache.logging.log4j log4j-slf4j-impl + runtime + + + + org.slf4j + log4j-over-slf4j + runtime + + + + org.apache.logging.log4j + log4j-jul ${dep.log4j.version} + runtime - + io.grpc grpc-netty-shaded true + runtime + + + + io.grpc + grpc-netty + runtime io.grpc @@ -116,14 +168,12 @@ true - com.google.protobuf - protobuf-java-util - 3.25.1 + io.grpc + grpc-api - - javax.annotation - javax.annotation-api + com.google.protobuf + protobuf-java @@ -144,16 +194,14 @@ connect-api ${dep.kafka.version} - - com.opencsv - opencsv - ${dep.opencsv.version} + org.slf4j + slf4j-api + com.google.guava guava - 33.2.0-jre @@ -163,6 +211,19 @@ provided + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + org.apache.avro + avro + ${dep.avro.version} + @@ -174,84 +235,81 @@ io.debezium debezium-embedded ${dep.debezium.version} + + + + ch.qos.reload4j + reload4j + + io.debezium - debezium-sink + debezium-connector-postgres ${dep.debezium.version} + runtime io.debezium - debezium-connector-postgres + debezium-connector-mysql ${dep.debezium.version} + runtime - junit - junit - test - true - - - org.junit.platform - junit-platform-launcher - test - true - - - org.junit.platform - junit-platform-runner + org.junit.jupiter + junit-jupiter + ${dep.junit.jupiter.version} test - true - org.junit.jupiter - junit-jupiter - 5.8.2 - test + io.apicurio + apicurio-registry-serdes-avro-serde + ${dep.apicurio.version} - org.junit.jupiter - junit-jupiter-api - 5.8.2 - test + io.apicurio + apicurio-registry-serde-common + ${dep.apicurio.version} - io.apicurio - apicurio-registry-serdes-avro-serde - 2.6.2.Final + apicurio-registry-client + ${dep.apicurio.version} io.prometheus simpleclient - 0.16.0 io.prometheus simpleclient_hotspot - 0.16.0 - io.prometheus simpleclient_httpserver - 0.16.0 + ${dep.prometheus.client.version} org.apache.commons commons-math3 - 3.6.1 + ${dep.commons-math3.version} + + + org.apache.commons + commons-lang3 + ${dep.commons-lang3.version} - + io.pixelsdb pixels-storage-localfs + runtime @@ -265,10 +323,21 @@ + + org.apache.maven.plugins + maven-surefire-plugin + + false + integration + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-exports=java.base/sun.nio.ch=ALL-UNNAMED + -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager + + org.apache.maven.plugins maven-shade-plugin - 3.2.0 + ${maven.plugin.shade.version} package @@ -278,6 +347,7 @@ pixels-sink ${project.basedir}/target + false true full @@ -287,6 +357,35 @@ META-INF/*.SF META-INF/*.DSA META-INF/*.RSA + module-info.class + META-INF/versions/**/module-info.class + + + + io.grpc:grpc-netty + + META-INF/services/io.grpc.ServerProvider + META-INF/services/io.grpc.ManagedChannelProvider + + + + + org.apache.hive:hive-jdbc + + org/apache/logging/log4j/** + org/apache/logging/slf4j/** + com/fasterxml/jackson/** + javax/servlet/** + org/openjdk/jol/** + META-INF/org/apache/logging/log4j/** + META-INF/services/org.apache.logging.log4j.* + META-INF/services/com.fasterxml.jackson.* + META-INF/services/javax.servlet.ServletContainerInitializer + META-INF/maven/org.apache.logging.log4j/** + META-INF/maven/com.fasterxml.jackson.core/** + META-INF/maven/javax.servlet/** + META-INF/maven/org.openjdk.jol/** @@ -296,6 +395,11 @@ ${mainClass} + + + true + @@ -335,4 +439,45 @@ + + + + + hudi-hive + + + org.apache.hive + hive-jdbc + ${dep.hive.jdbc.version} + standalone + runtime + + + * + * + + + + + + + integration-tests + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-exports=java.base/sun.nio.ch=ALL-UNNAMED + -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager + + + + + + \ No newline at end of file diff --git a/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkApp.java b/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkApp.java index 0453d6a..a55afb4 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkApp.java +++ b/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkApp.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink; import io.pixelsdb.pixels.sink.config.CommandLineConfig; @@ -53,8 +48,8 @@ public static void main(String[] args) throws IOException Runtime.getRuntime().addShutdownHook(new Thread(() -> { PixelsSinkConfig config = PixelsSinkConfigFactory.getInstance(); + sinkSource.close(); TransactionProxy.staticClose(); - sinkSource.stopProcessor(); LOGGER.info("Pixels Sink Server shutdown complete"); if (config.getSinkMonitorFreshnessLevel().equals("embed") && freshnessClient != null) { diff --git a/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkProvider.java b/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkProvider.java index ecd9364..2f808db 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkProvider.java +++ b/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkProvider.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink; import io.pixelsdb.pixels.common.sink.SinkProvider; @@ -42,7 +37,7 @@ public void start(ConfigFactory config) @Override public void shutdown() { - sinkSource.stopProcessor(); + sinkSource.close(); } @Override diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/CommandLineConfig.java b/src/main/java/io/pixelsdb/pixels/sink/config/CommandLineConfig.java index a8bf062..ab87dd6 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/CommandLineConfig.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/CommandLineConfig.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/ConfigKey.java b/src/main/java/io/pixelsdb/pixels/sink/config/ConfigKey.java index bb4ed66..461ff26 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/ConfigKey.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/ConfigKey.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/ConfigLoader.java b/src/main/java/io/pixelsdb/pixels/sink/config/ConfigLoader.java index 0461535..d5f1a66 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/ConfigLoader.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/ConfigLoader.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/EngineValueFormat.java b/src/main/java/io/pixelsdb/pixels/sink/config/EngineValueFormat.java new file mode 100644 index 0000000..e1d0303 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/config/EngineValueFormat.java @@ -0,0 +1,52 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.config; + +import java.util.Locale; + +/** + * Resolves {@code sink.datasource.engine.format}. Only {@code connect} is implemented. + */ +public final class EngineValueFormat +{ + public static final String CONNECT = "connect"; + + private EngineValueFormat() + { + } + + public static String resolve(String configuredFormat) + { + if (configuredFormat == null || configuredFormat.isBlank()) + { + return CONNECT; + } + String normalized = normalize(configuredFormat); + if (!CONNECT.equals(normalized)) + { + throw new IllegalArgumentException( + "sink.datasource.engine.format='" + configuredFormat + + "' is not implemented yet; only 'connect' is supported. " + + "Json/Avro Engine paths will reuse conversion.debezium converters later."); + } + return normalized; + } + + private static String normalize(String format) + { + return format.trim().toLowerCase(Locale.ROOT); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/KafkaValueFormat.java b/src/main/java/io/pixelsdb/pixels/sink/config/KafkaValueFormat.java new file mode 100644 index 0000000..0a0721e --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/config/KafkaValueFormat.java @@ -0,0 +1,51 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.config; + +import java.util.Locale; + +/** + * Resolves {@code sink.kafka.value.format}. Supported values: {@code json}, {@code avro}. + */ +public final class KafkaValueFormat +{ + public static final String JSON = "json"; + public static final String AVRO = "avro"; + + private KafkaValueFormat() + { + } + + public static String resolve(String configuredFormat) + { + if (configuredFormat == null || configuredFormat.isBlank()) + { + return JSON; + } + return normalize(configuredFormat); + } + + private static String normalize(String format) + { + String normalized = format.trim().toLowerCase(Locale.ROOT); + if (!JSON.equals(normalized) && !AVRO.equals(normalized)) + { + throw new IllegalArgumentException( + "sink.kafka.value.format must be json or avro, got: " + format); + } + return normalized; + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java index 4f75eeb..02108b7 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java @@ -1,39 +1,43 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config; import io.pixelsdb.pixels.common.utils.ConfigFactory; -import io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventJsonDeserializer; import io.pixelsdb.pixels.sink.writer.PixelsSinkMode; import io.pixelsdb.pixels.sink.writer.retina.RetinaServiceProxy; import io.pixelsdb.pixels.sink.writer.retina.TransactionMode; import lombok.Getter; import org.apache.kafka.common.serialization.StringDeserializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.List; +import java.util.Locale; +import java.util.Properties; @Getter public class PixelsSinkConfig { + private static final Logger LOGGER = LoggerFactory.getLogger(PixelsSinkConfig.class); + private static final String LEGACY_VALUE_DESERIALIZER = "value.deserializer"; + private static final String LEGACY_TX_VALUE_DESERIALIZER = + "transaction.topic.value.deserializer"; + private static final String KAFKA_VALUE_FORMAT_KEY = "sink.kafka.value.format"; + private final ConfigFactory config; @ConfigKey(value = "transaction.timeout", defaultValue = TransactionConfig.DEFAULT_TRANSACTION_TIME_OUT) @@ -117,9 +121,6 @@ public class PixelsSinkConfig @ConfigKey(value = "sink.rpc.enable", defaultValue = "false") private boolean rpcEnable; - @ConfigKey(value = "sink.rpc.mock.delay", defaultValue = "0") - private int mockRpcDelay; - @ConfigKey(value = "sink.trans.batch.size", defaultValue = "100") private int transBatchSize; @@ -136,6 +137,16 @@ public class PixelsSinkConfig @ConfigKey("debezium.topic.prefix") private String debeziumTopicPrefix; + /** + * Sink-side CDC dialect for envelope normalization: {@code mysql} or {@code postgresql}. + * Independent of {@code debezium.connector.class}, which is only for Debezium Engine. + */ + @ConfigKey(value = "sink.debezium.dialect", defaultValue = "") + private String debeziumDialect; + + @ConfigKey(value = "debezium.connector.class", defaultValue = "") + private String debeziumConnectorClass; + @ConfigKey("consumer.capture_database") private String captureDatabase; @@ -151,8 +162,11 @@ public class PixelsSinkConfig @ConfigKey(value = "key.deserializer", defaultClass = StringDeserializer.class) private String keyDeserializer; - @ConfigKey(value = "value.deserializer", defaultClass = RowChangeEventJsonDeserializer.class) - private String valueDeserializer; + /** + * Kafka envelope wire format: {@code json} or {@code avro}. + */ + @ConfigKey(value = "sink.kafka.value.format", defaultValue = KafkaValueFormat.JSON) + private String kafkaValueFormat; @ConfigKey(value = "sink.csv.path", defaultValue = PixelsSinkDefaultConfig.CSV_SINK_PATH) private String csvSinkPath; @@ -160,10 +174,6 @@ public class PixelsSinkConfig @ConfigKey(value = "transaction.topic.suffix", defaultValue = TransactionConfig.DEFAULT_TRANSACTION_TOPIC_SUFFIX) private String transactionTopicSuffix; - @ConfigKey(value = "transaction.topic.value.deserializer", - defaultClass = RowChangeEventJsonDeserializer.class) - private String transactionTopicValueDeserializer; - @ConfigKey(value = "transaction.topic.group_id", defaultValue = TransactionConfig.DEFAULT_TRANSACTION_TOPIC_GROUP_ID) private String transactionTopicGroupId; @@ -177,6 +187,16 @@ public class PixelsSinkConfig @ConfigKey(value = "sink.datasource", defaultValue = PixelsSinkDefaultConfig.DATA_SOURCE) private String dataSource; + /** + * Engine wire format. Currently only {@code connect} is implemented. + */ + @ConfigKey(value = "sink.datasource.engine.format", defaultValue = "connect") + private String engineFormat; + + @ConfigKey(value = "sink.datasource.decode.threads", + defaultValue = PixelsSinkDefaultConfig.SOURCE_DECODE_THREADS) + private int sourceDecodeThreads; + @ConfigKey(value = "sink.datasource.rate.limit", defaultValue = "-1") private int sourceRateLimit; @@ -196,6 +216,9 @@ public class PixelsSinkConfig @ConfigKey(value = "sink.storage.loop", defaultValue = "false") private boolean sinkStorageLoop; + @ConfigKey(value = "sink.storage.mode", defaultValue = PixelsSinkDefaultConfig.STORAGE_MODE) + private String sinkStorageMode; + @ConfigKey(value = "sink.monitor.freshness.level", defaultValue = "row") // row or txn or embed private String sinkMonitorFreshnessLevel; @ConfigKey(value = "sink.monitor.freshness.embed.warmup", defaultValue = "10") @@ -219,17 +242,17 @@ public class PixelsSinkConfig @ConfigKey(value = "sink.monitor.freshness.timestamp", defaultValue = "false") private boolean sinkMonitorFreshnessTimestamp; - @ConfigKey(value = "trino.url") - private String trinoUrl; + @ConfigKey(value = "sink.query.url") + private String sinkQueryUrl; - @ConfigKey(value = "trino.user") - private String trinoUser; + @ConfigKey(value = "sink.query.user") + private String sinkQueryUser; - @ConfigKey(value = "trino.password") - private String trinoPassword; + @ConfigKey(value = "sink.query.password") + private String sinkQueryPassword; - @ConfigKey(value = "trino.parallel", defaultValue = "1") - private int trinoParallel; + @ConfigKey(value = "sink.query.parallel", defaultValue = "1") + private int sinkQueryParallel; public PixelsSinkConfig(String configFilePath) throws IOException { @@ -249,10 +272,124 @@ public String[] getIncludeTables() return includeTablesRaw.isEmpty() ? new String[0] : includeTablesRaw.split(","); } - private void init() + /** + * Token used to select a {@code DebeziumSourceAdapter}. + * Prefers {@code sink.debezium.dialect}; falls back to {@code debezium.connector.class} + * so Engine-only configs keep working without duplicating the dialect. + */ + public String resolveDebeziumSourceDialect() { - ConfigLoader.load(this.config.extractPropertiesByPrefix("", false), this); + if (debeziumDialect != null && !debeziumDialect.isBlank()) + { + return debeziumDialect.trim(); + } + if (debeziumConnectorClass != null && !debeziumConnectorClass.isBlank()) + { + return debeziumConnectorClass.trim(); + } + return ""; + } + private void init() + { + Properties props = this.config.extractPropertiesByPrefix("", false); + ConfigLoader.load(props, this); + + if (this.sourceDecodeThreads <= 0) + { + throw new IllegalArgumentException( + "sink.datasource.decode.threads must be positive"); + } this.enableSourceRateLimit = this.sourceRateLimit >= 0; + this.kafkaValueFormat = resolveKafkaValueFormat(props); + this.engineFormat = EngineValueFormat.resolve(this.engineFormat); + } + + /** + * Migrates legacy {@code value.deserializer} / + * {@code transaction.topic.value.deserializer} class names to + * {@code sink.kafka.value.format}. Package-visible for unit tests. + */ + static String resolveKafkaValueFormat(Properties props) + { + String explicitFormat = trimToNull(props.getProperty(KAFKA_VALUE_FORMAT_KEY)); + String rowDeserializer = trimToNull(props.getProperty(LEGACY_VALUE_DESERIALIZER)); + String txDeserializer = trimToNull(props.getProperty(LEGACY_TX_VALUE_DESERIALIZER)); + boolean hasLegacy = rowDeserializer != null || txDeserializer != null; + + if (explicitFormat != null) + { + if (hasLegacy) + { + LOGGER.warn( + "Ignoring legacy {} / {} because {} is set to '{}'", + LEGACY_VALUE_DESERIALIZER, + LEGACY_TX_VALUE_DESERIALIZER, + KAFKA_VALUE_FORMAT_KEY, + explicitFormat); + } + return KafkaValueFormat.resolve(explicitFormat); + } + + if (!hasLegacy) + { + return KafkaValueFormat.resolve(null); + } + + String inferredRow = inferFormatFromDeserializer(rowDeserializer); + String inferredTx = inferFormatFromDeserializer(txDeserializer); + if (inferredRow == null && inferredTx == null) + { + throw new IllegalArgumentException( + "Unable to migrate legacy Kafka deserializer classes to " + + KAFKA_VALUE_FORMAT_KEY + ": value.deserializer=" + + rowDeserializer + ", transaction.topic.value.deserializer=" + + txDeserializer); + } + if (inferredRow != null && inferredTx != null && !inferredRow.equals(inferredTx)) + { + throw new IllegalArgumentException( + "Conflicting legacy Kafka deserializer formats: value.deserializer=" + + rowDeserializer + " (" + inferredRow + + "), transaction.topic.value.deserializer=" + + txDeserializer + " (" + inferredTx + ")"); + } + String inferred = inferredRow != null ? inferredRow : inferredTx; + LOGGER.warn( + "Migrating legacy Kafka deserializer class names to {}={}", + KAFKA_VALUE_FORMAT_KEY, inferred); + return inferred; + } + + private static String inferFormatFromDeserializer(String className) + { + if (className == null) + { + return null; + } + String simpleName = className; + int lastDot = className.lastIndexOf('.'); + if (lastDot >= 0 && lastDot + 1 < className.length()) + { + simpleName = className.substring(lastDot + 1); + } + String lower = simpleName.toLowerCase(Locale.ROOT); + boolean json = lower.contains("json"); + boolean avro = lower.contains("avro"); + if (json == avro) + { + return null; + } + return json ? KafkaValueFormat.JSON : KafkaValueFormat.AVRO; + } + + private static String trimToNull(String value) + { + if (value == null) + { + return null; + } + String trimmed = value.trim(); + return trimmed.isEmpty() ? null : trimmed; } } \ No newline at end of file diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java index 034ae78..4e5e1c5 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java @@ -1,30 +1,28 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config; public final class PixelsSinkConstants { public static final String ROW_RECORD_KAFKA_PROP_FACTORY = "row-record"; public static final String TRANSACTION_KAFKA_PROP_FACTORY = "transaction"; - public static final int MONITOR_NUM = 2; + public static final String KAFKA_VALUE_FORMAT = "pixels.sink.kafka.value.format"; + /** Sink-side CDC dialect for envelope normalization: {@code mysql} or {@code postgresql}. */ + public static final String SINK_DEBEZIUM_DIALECT = "sink.debezium.dialect"; + public static final String DEBEZIUM_CONNECTOR_CLASS = "debezium.connector.class"; public static final int MAX_QUEUE_SIZE = 1_000; public static final String SNAPSHOT_TX_PREFIX = "SNAPSHOT-"; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkDefaultConfig.java b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkDefaultConfig.java index 1a56ddd..585ce0a 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkDefaultConfig.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkDefaultConfig.java @@ -1,28 +1,24 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config; public class PixelsSinkDefaultConfig { public static final String DATA_SOURCE = "engine"; + public static final String SOURCE_DECODE_THREADS = "4"; public static final String PROPERTIES_PATH = "pixels-sink.properties"; public static final String CSV_SINK_PATH = "./data"; @@ -52,6 +48,6 @@ public class PixelsSinkDefaultConfig // Mock RPC public static final boolean SINK_RPC_ENABLED = true; - public static final int MOCK_RPC_DELAY = 100; public static final String MAX_RECORDS_PER_FILE = "100000"; + public static final String STORAGE_MODE = "stream"; } diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/TransactionConfig.java b/src/main/java/io/pixelsdb/pixels/sink/config/TransactionConfig.java index f516b3a..53585fe 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/TransactionConfig.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/TransactionConfig.java @@ -1,31 +1,23 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config; -import io.pixelsdb.pixels.sink.event.deserializer.TransactionJsonMessageDeserializer; - public class TransactionConfig { public static final String DEFAULT_TRANSACTION_TOPIC_SUFFIX = "transaction"; - public static final String DEFAULT_TRANSACTION_TOPIC_VALUE_DESERIALIZER = TransactionJsonMessageDeserializer.class.getName(); public static final String DEFAULT_TRANSACTION_TOPIC_GROUP_ID = "transaction_consumer"; public static final String DEFAULT_TRANSACTION_TIME_OUT = "300"; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/factory/KafkaPropFactory.java b/src/main/java/io/pixelsdb/pixels/sink/config/factory/KafkaPropFactory.java index cbace9f..e1eceb6 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/factory/KafkaPropFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/factory/KafkaPropFactory.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config.factory; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/factory/KafkaPropFactorySelector.java b/src/main/java/io/pixelsdb/pixels/sink/config/factory/KafkaPropFactorySelector.java index c0343f9..9d2431d 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/factory/KafkaPropFactorySelector.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/factory/KafkaPropFactorySelector.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config.factory; import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/factory/PixelsSinkConfigFactory.java b/src/main/java/io/pixelsdb/pixels/sink/config/factory/PixelsSinkConfigFactory.java index 56b3550..a25dfc3 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/factory/PixelsSinkConfigFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/factory/PixelsSinkConfigFactory.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config.factory; import io.pixelsdb.pixels.common.utils.ConfigFactory; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/factory/RowRecordKafkaPropFactory.java b/src/main/java/io/pixelsdb/pixels/sink/config/factory/RowRecordKafkaPropFactory.java index 6c1fbc0..be04556 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/factory/RowRecordKafkaPropFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/factory/RowRecordKafkaPropFactory.java @@ -1,27 +1,25 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config.factory; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; +import io.apicurio.registry.serde.SerdeConfig; import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; import java.util.Properties; @@ -33,6 +31,8 @@ static Properties getCommonKafkaProperties(PixelsSinkConfig config) kafkaProperties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, config.getBootstrapServers()); kafkaProperties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, config.getKeyDeserializer()); kafkaProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + kafkaProperties.put( + PixelsSinkConstants.KAFKA_VALUE_FORMAT, config.getKafkaValueFormat()); return kafkaProperties; } @@ -40,7 +40,16 @@ static Properties getCommonKafkaProperties(PixelsSinkConfig config) public Properties createKafkaProperties(PixelsSinkConfig config) { Properties kafkaProperties = getCommonKafkaProperties(config); - kafkaProperties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, config.getValueDeserializer()); + String dialect = config.resolveDebeziumSourceDialect(); + if (!dialect.isBlank()) + { + kafkaProperties.put(PixelsSinkConstants.SINK_DEBEZIUM_DIALECT, dialect); + } + if (config.getRegistryUrl() != null && !config.getRegistryUrl().isBlank()) + { + kafkaProperties.put(SerdeConfig.REGISTRY_URL, config.getRegistryUrl()); + } + kafkaProperties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class); kafkaProperties.put(ConsumerConfig.GROUP_ID_CONFIG, config.getGroupId()); kafkaProperties.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "30000"); diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/factory/TransactionKafkaPropFactory.java b/src/main/java/io/pixelsdb/pixels/sink/config/factory/TransactionKafkaPropFactory.java index 52949b7..464f162 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/factory/TransactionKafkaPropFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/factory/TransactionKafkaPropFactory.java @@ -1,27 +1,25 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config.factory; +import io.apicurio.registry.serde.SerdeConfig; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; import java.util.Properties; @@ -33,7 +31,16 @@ public class TransactionKafkaPropFactory implements KafkaPropFactory public Properties createKafkaProperties(PixelsSinkConfig config) { Properties kafkaProperties = getCommonKafkaProperties(config); - kafkaProperties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, config.getTransactionTopicValueDeserializer()); + String dialect = config.resolveDebeziumSourceDialect(); + if (!dialect.isBlank()) + { + kafkaProperties.put(PixelsSinkConstants.SINK_DEBEZIUM_DIALECT, dialect); + } + if (config.getRegistryUrl() != null && !config.getRegistryUrl().isBlank()) + { + kafkaProperties.put(SerdeConfig.REGISTRY_URL, config.getRegistryUrl()); + } + kafkaProperties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class); kafkaProperties.put(ConsumerConfig.GROUP_ID_CONFIG, config.getTransactionTopicGroupId() + "-" + config.getGroupId()); return kafkaProperties; } diff --git a/src/test/java/io/pixelsdb/pixels/sink/TestSplitString.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowConverter.java similarity index 57% rename from src/test/java/io/pixelsdb/pixels/sink/TestSplitString.java rename to src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowConverter.java index 5d32ba9..138e737 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/TestSplitString.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowConverter.java @@ -1,5 +1,5 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,24 +12,16 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ -package io.pixelsdb.pixels.sink; +package io.pixelsdb.pixels.sink.conversion.debezium; -import org.junit.Test; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; /** - * Created at: 29/04/2021 - * Author: hank + * Converts a Debezium row envelope (any wire format) into a canonical row event. */ -public class TestSplitString +@FunctionalInterface +public interface DebeziumRowConverter { - @Test - public void test() - { - String s = "1|3689999|O|224560.83|1996-01-02|5-LOW|Clerk#000095055|0|nstructions sleep furiously among |"; - String reg = "\\\\|"; - String[] splits = s.split(reg); - System.out.println(splits.length); - } + RowChangeEvent convert(I input) throws Exception; } diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumTransactionConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumTransactionConverter.java new file mode 100644 index 0000000..4b8893e --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumTransactionConverter.java @@ -0,0 +1,27 @@ +/* + * Copyright 2023 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.debezium; + +import io.pixelsdb.pixels.sink.SinkProto; + +/** + * Converts a Debezium transaction envelope (any wire format) into metadata. + */ +@FunctionalInterface +public interface DebeziumTransactionConverter +{ + SinkProto.TransactionMetadata convert(I input) throws Exception; +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroPayloadDecoder.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroPayloadDecoder.java new file mode 100644 index 0000000..4b6924c --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroPayloadDecoder.java @@ -0,0 +1,50 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.avro; + +import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; +import org.apache.avro.generic.GenericRecord; + +import java.util.HashMap; +import java.util.Map; + +/** + * Decodes Kafka Avro Confluent/Apicurio payloads to {@link GenericRecord}. + */ +public final class DebeziumAvroPayloadDecoder implements AutoCloseable +{ + private final AvroKafkaDeserializer avroDeserializer = + new AvroKafkaDeserializer<>(); + + public void configure(Map configs, boolean isKey) + { + Map enrichedConfig = new HashMap<>(configs); + enrichedConfig.put(SerdeConfig.CHECK_PERIOD_MS, SerdeConfig.CHECK_PERIOD_MS_DEFAULT); + avroDeserializer.configure(enrichedConfig, isKey); + } + + public GenericRecord decode(String topic, byte[] data) + { + return avroDeserializer.deserialize(topic, data); + } + + @Override + public void close() + { + avroDeserializer.close(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroRowConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroRowConverter.java new file mode 100644 index 0000000..34aaf4e --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroRowConverter.java @@ -0,0 +1,102 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.avro; + +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumRowConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumEnvelopeNormalizer; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRecordUtil; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRowRecordAssembler; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRowValueConverter; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import org.apache.avro.generic.GenericRecord; + +public final class DebeziumAvroRowConverter implements DebeziumRowConverter +{ + private final TableMetadataRegistry tableMetadataRegistry; + private final DebeziumSourceAdapter configuredAdapter; + private final DebeziumRowRecordAssembler rowRecordAssembler = + new DebeziumRowRecordAssembler(); + + public DebeziumAvroRowConverter( + TableMetadataRegistry tableMetadataRegistry, + DebeziumSourceAdapter configuredAdapter) + { + this.tableMetadataRegistry = tableMetadataRegistry; + this.configuredAdapter = configuredAdapter; + } + + @Override + public RowChangeEvent convert(GenericRecord record) throws SinkException + { + SinkProto.OperationType operation = DebeziumRecordUtil.getOperationType( + DebeziumRecordUtil.getStringSafely(record, "op")); + Object sourceObject = record.get("source"); + if (!(sourceObject instanceof GenericRecord source)) + { + throw new SinkException("Missing source field in row record"); + } + DebeziumSourceAdapter adapter = configuredAdapter == null + ? DebeziumEnvelopeNormalizer.adapterForSource(source) + : configuredAdapter; + SinkProto.SourceInfo sourceInfo = + DebeziumEnvelopeNormalizer.normalizeSource(source, adapter); + TableMetadata metadata = tableMetadataRegistry.getMetadata( + sourceInfo.getDb(), sourceInfo.getTable()); + TypeDescription schema = metadata.getTypeDescription(); + DebeziumRowValueConverter rowValueConverter = + new DebeziumRowValueConverter(schema); + + SinkProto.RowValue before = parseRowValue( + record.get("before"), operation, true, rowValueConverter); + SinkProto.RowValue after = parseRowValue( + record.get("after"), operation, false, rowValueConverter); + SinkProto.TransactionInfo transaction = record.get("transaction") == null + ? null + : DebeziumEnvelopeNormalizer.normalizeTransaction( + record.get("transaction"), adapter); + return rowRecordAssembler.assemble( + operation, sourceInfo, transaction, before, after, schema, metadata); + } + + private SinkProto.RowValue parseRowValue( + Object value, + SinkProto.OperationType operation, + boolean before, + DebeziumRowValueConverter converter) throws SinkException + { + boolean required = before + ? DebeziumRecordUtil.hasBeforeValue(operation) + : DebeziumRecordUtil.hasAfterValue(operation); + if (!required) + { + return null; + } + if (!(value instanceof GenericRecord row)) + { + throw new SinkException( + "Missing " + (before ? "before" : "after") + " image for " + operation); + } + SinkProto.RowValue.Builder builder = SinkProto.RowValue.newBuilder(); + converter.parse(row, builder); + return builder.build(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroTransactionConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroTransactionConverter.java new file mode 100644 index 0000000..462e994 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroTransactionConverter.java @@ -0,0 +1,42 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.avro; + +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumTransactionConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumEnvelopeNormalizer; +import org.apache.avro.generic.GenericRecord; + +public final class DebeziumAvroTransactionConverter + implements DebeziumTransactionConverter +{ + private final DebeziumSourceAdapter configuredAdapter; + + public DebeziumAvroTransactionConverter(DebeziumSourceAdapter configuredAdapter) + { + this.configuredAdapter = configuredAdapter; + } + + @Override + public SinkProto.TransactionMetadata convert(GenericRecord record) + { + return configuredAdapter == null + ? DebeziumEnvelopeNormalizer.normalizeTransactionMetadata(record) + : DebeziumEnvelopeNormalizer.normalizeTransactionMetadata( + record, configuredAdapter); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectRowConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectRowConverter.java new file mode 100644 index 0000000..ebc61e6 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectRowConverter.java @@ -0,0 +1,138 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.connect; + +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumRowConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumEnvelopeNormalizer; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRecordUtil; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRowRecordAssembler; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRowValueConverter; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.errors.DataException; +import org.apache.kafka.connect.source.SourceRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Converts Kafka Connect {@link SourceRecord} Debezium envelopes to row events. + */ +public final class DebeziumConnectRowConverter implements DebeziumRowConverter +{ + private static final Logger LOGGER = LoggerFactory.getLogger(DebeziumConnectRowConverter.class); + + private final TableMetadataRegistry tableMetadataRegistry; + private final DebeziumSourceAdapter configuredAdapter; + private final DebeziumRowRecordAssembler rowRecordAssembler = new DebeziumRowRecordAssembler(); + + public DebeziumConnectRowConverter( + TableMetadataRegistry tableMetadataRegistry, + DebeziumSourceAdapter configuredAdapter) + { + this.tableMetadataRegistry = tableMetadataRegistry; + this.configuredAdapter = configuredAdapter; + } + + @Override + public RowChangeEvent convert(SourceRecord sourceRecord) throws SinkException + { + if (!(sourceRecord.value() instanceof Struct value)) + { + throw new SinkException("Debezium row value must be a Struct"); + } + String op = DebeziumRecordUtil.getStringSafely(value, "op"); + SinkProto.OperationType operationType = DebeziumRecordUtil.getOperationType(op); + return buildRowRecord(value, operationType); + } + + private RowChangeEvent buildRowRecord( + Struct value, SinkProto.OperationType opType) throws SinkException + { + DebeziumSourceAdapter adapter; + SinkProto.SourceInfo sourceInfo; + try + { + Struct source = value.getStruct("source"); + if (source == null) + { + throw new SinkException("Missing source field in row record"); + } + adapter = configuredAdapter == null + ? DebeziumEnvelopeNormalizer.adapterForSource(source) + : configuredAdapter; + sourceInfo = DebeziumEnvelopeNormalizer.normalizeSource(source, adapter); + } catch (DataException | IllegalArgumentException e) + { + LOGGER.warn("Missing source field in row record"); + throw new SinkException(e); + } + + TableMetadata metadata = tableMetadataRegistry.getMetadata( + sourceInfo.getDb(), sourceInfo.getTable()); + TypeDescription typeDescription = metadata.getTypeDescription(); + DebeziumRowValueConverter rowDataParser = new DebeziumRowValueConverter(typeDescription); + + SinkProto.TransactionInfo transactionInfo = null; + try + { + Struct transaction = value.getStruct("transaction"); + if (transaction != null) + { + transactionInfo = DebeziumEnvelopeNormalizer.normalizeTransaction( + transaction, adapter); + } + } catch (DataException e) + { + LOGGER.warn("Missing transaction field in row record"); + } + + SinkProto.RowValue beforeValue = null; + if (DebeziumRecordUtil.hasBeforeValue(opType)) + { + Struct before = value.getStruct("before"); + if (before == null) + { + throw new SinkException("Missing before image for " + opType); + } + SinkProto.RowValue.Builder beforeBuilder = SinkProto.RowValue.newBuilder(); + rowDataParser.parse(before, beforeBuilder); + beforeValue = beforeBuilder.build(); + } + + SinkProto.RowValue afterValue = null; + if (DebeziumRecordUtil.hasAfterValue(opType)) + { + Struct after = value.getStruct("after"); + if (after == null) + { + throw new SinkException("Missing after image for " + opType); + } + SinkProto.RowValue.Builder afterBuilder = SinkProto.RowValue.newBuilder(); + rowDataParser.parse(after, afterBuilder); + afterValue = afterBuilder.build(); + } + + return rowRecordAssembler.assemble( + opType, sourceInfo, transactionInfo, beforeValue, afterValue, + typeDescription, metadata); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectTransactionConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectTransactionConverter.java new file mode 100644 index 0000000..461c2b0 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectTransactionConverter.java @@ -0,0 +1,49 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.connect; + +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumTransactionConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumEnvelopeNormalizer; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.source.SourceRecord; + +/** + * Converts Kafka Connect {@link SourceRecord} transaction envelopes. + */ +public final class DebeziumConnectTransactionConverter + implements DebeziumTransactionConverter +{ + private final DebeziumSourceAdapter configuredAdapter; + + public DebeziumConnectTransactionConverter(DebeziumSourceAdapter configuredAdapter) + { + this.configuredAdapter = configuredAdapter; + } + + @Override + public SinkProto.TransactionMetadata convert(SourceRecord sourceRecord) + { + if (!(sourceRecord.value() instanceof Struct value)) + { + throw new IllegalArgumentException("Debezium transaction value must be a Struct"); + } + return configuredAdapter == null + ? DebeziumEnvelopeNormalizer.normalizeTransactionMetadata(value) + : DebeziumEnvelopeNormalizer.normalizeTransactionMetadata(value, configuredAdapter); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceAdapter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceAdapter.java new file mode 100644 index 0000000..d64eca8 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceAdapter.java @@ -0,0 +1,44 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.dialect; + +import io.pixelsdb.pixels.sink.SinkProto; + +import java.util.Locale; + +public interface DebeziumSourceAdapter +{ + String connector(); + + default boolean supports(String connectorName) + { + if (connectorName == null || connectorName.isBlank()) + { + return false; + } + String normalized = connectorName.toLowerCase(Locale.ROOT); + return normalized.equals(connector()) || normalized.contains(connector()); + } + + String normalizeTransactionId(String sourceId); + + SinkProto.SourceInfo normalizeSource(DebeziumSourceMetadata source); + + default String normalizeDataCollection(String sourceDataCollection) + { + return sourceDataCollection; + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceAdapterRegistry.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceAdapterRegistry.java new file mode 100644 index 0000000..b002962 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceAdapterRegistry.java @@ -0,0 +1,58 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.dialect; + +import java.util.List; +import java.util.ServiceLoader; + +public final class DebeziumSourceAdapterRegistry +{ + private static final List ADAPTERS = ServiceLoader + .load(DebeziumSourceAdapter.class) + .stream() + .map(ServiceLoader.Provider::get) + .toList(); + + private DebeziumSourceAdapterRegistry() + { + } + + public static DebeziumSourceAdapter forSource(String connector) + { + return resolve(connector); + } + + public static DebeziumSourceAdapter resolve(String connector) + { + if (connector == null || connector.isBlank()) + { + throw new IllegalArgumentException("Unsupported Debezium connector: " + connector); + } + + List matches = ADAPTERS.stream() + .filter(adapter -> adapter.supports(connector)) + .toList(); + if (matches.size() == 1) + { + return matches.get(0); + } + if (matches.size() > 1) + { + throw new IllegalStateException("Multiple Debezium connector adapters support: " + connector); + } + throw new IllegalArgumentException("Unsupported Debezium connector: " + connector); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceMetadata.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceMetadata.java new file mode 100644 index 0000000..eda02f0 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceMetadata.java @@ -0,0 +1,24 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.dialect; + +public record DebeziumSourceMetadata( + String connector, + String db, + String schema, + String table) +{ +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/MySqlSourceAdapter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/MySqlSourceAdapter.java new file mode 100644 index 0000000..f35615e --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/MySqlSourceAdapter.java @@ -0,0 +1,48 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.dialect; + +import io.pixelsdb.pixels.sink.SinkProto; + +public final class MySqlSourceAdapter implements DebeziumSourceAdapter +{ + public static final MySqlSourceAdapter INSTANCE = new MySqlSourceAdapter(); + + public MySqlSourceAdapter() + { + } + + @Override + public String connector() + { + return "mysql"; + } + + @Override + public String normalizeTransactionId(String sourceId) + { + return sourceId; + } + + @Override + public SinkProto.SourceInfo normalizeSource(DebeziumSourceMetadata source) + { + return SinkProto.SourceInfo.newBuilder() + .setDb(source.db()) + .setTable(source.table()) + .build(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/PostgresSourceAdapter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/PostgresSourceAdapter.java new file mode 100644 index 0000000..77d6a0f --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/PostgresSourceAdapter.java @@ -0,0 +1,54 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.dialect; + +import io.pixelsdb.pixels.sink.SinkProto; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public final class PostgresSourceAdapter implements DebeziumSourceAdapter +{ + public static final PostgresSourceAdapter INSTANCE = new PostgresSourceAdapter(); + private static final Pattern TRANSACTION_WITH_LSN = Pattern.compile("^(\\d+):\\d+$"); + + public PostgresSourceAdapter() + { + } + + @Override + public String connector() + { + return "postgresql"; + } + + @Override + public String normalizeTransactionId(String sourceId) + { + Matcher matcher = TRANSACTION_WITH_LSN.matcher(sourceId); + return matcher.matches() ? matcher.group(1) : sourceId; + } + + @Override + public SinkProto.SourceInfo normalizeSource(DebeziumSourceMetadata source) + { + return SinkProto.SourceInfo.newBuilder() + .setDb(source.db()) + .setSchema(source.schema()) + .setTable(source.table()) + .build(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonRowConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonRowConverter.java new file mode 100644 index 0000000..d482131 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonRowConverter.java @@ -0,0 +1,124 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumRowConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumEnvelopeNormalizer; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRecordUtil; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRowRecordAssembler; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRowValueConverter; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; + +public final class DebeziumJsonRowConverter implements DebeziumRowConverter +{ + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private final TableMetadataRegistry tableMetadataRegistry; + private final DebeziumSourceAdapter configuredAdapter; + private final DebeziumRowRecordAssembler rowRecordAssembler = + new DebeziumRowRecordAssembler(); + + public DebeziumJsonRowConverter( + TableMetadataRegistry tableMetadataRegistry, + DebeziumSourceAdapter configuredAdapter) + { + this.tableMetadataRegistry = tableMetadataRegistry; + this.configuredAdapter = configuredAdapter; + } + + @Override + public RowChangeEvent convert(byte[] data) throws Exception + { + JsonNode rootNode = OBJECT_MAPPER.readTree(data); + JsonNode payloadNode = rootNode.path("payload"); + SinkProto.OperationType operation = DebeziumRecordUtil.getOperationType( + payloadNode.path("op").asText("")); + + if (!payloadNode.hasNonNull("source")) + { + throw new SinkException("Missing source field in row record"); + } + + JsonNode sourceNode = payloadNode.get("source"); + DebeziumSourceAdapter adapter = configuredAdapter == null + ? DebeziumEnvelopeNormalizer.adapterForSource(sourceNode) + : configuredAdapter; + SinkProto.SourceInfo sourceInfo = + DebeziumEnvelopeNormalizer.normalizeSource(sourceNode, adapter); + TableMetadata metadata = tableMetadataRegistry.getMetadata( + sourceInfo.getDb(), sourceInfo.getTable()); + TypeDescription schema = metadata.getTypeDescription(); + DebeziumRowValueConverter rowValueConverter = + new DebeziumRowValueConverter(schema); + + SinkProto.TransactionInfo transaction = payloadNode.hasNonNull("transaction") + ? DebeziumEnvelopeNormalizer.normalizeTransaction( + payloadNode.get("transaction"), adapter) + : null; + SinkProto.RowValue before = parseBefore( + payloadNode, operation, rowValueConverter); + SinkProto.RowValue after = parseAfter( + payloadNode, operation, rowValueConverter); + return rowRecordAssembler.assemble( + operation, sourceInfo, transaction, before, after, schema, metadata); + } + + private SinkProto.RowValue parseBefore( + JsonNode payload, + SinkProto.OperationType operation, + DebeziumRowValueConverter converter) throws SinkException + { + if (!DebeziumRecordUtil.hasBeforeValue(operation)) + { + return null; + } + JsonNode before = payload.get("before"); + if (before == null || before.isNull()) + { + throw new SinkException("Missing before image for " + operation); + } + SinkProto.RowValue.Builder builder = SinkProto.RowValue.newBuilder(); + converter.parse(before, builder); + return builder.build(); + } + + private SinkProto.RowValue parseAfter( + JsonNode payload, + SinkProto.OperationType operation, + DebeziumRowValueConverter converter) throws SinkException + { + if (!DebeziumRecordUtil.hasAfterValue(operation)) + { + return null; + } + JsonNode after = payload.get("after"); + if (after == null || after.isNull()) + { + throw new SinkException("Missing after image for " + operation); + } + SinkProto.RowValue.Builder builder = SinkProto.RowValue.newBuilder(); + converter.parse(after, builder); + return builder.build(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonTransactionConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonTransactionConverter.java new file mode 100644 index 0000000..0b5da07 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonTransactionConverter.java @@ -0,0 +1,52 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumTransactionConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumEnvelopeNormalizer; + +/** + * Converts Debezium JSON transaction envelopes ({@code byte[]} payload). + */ +public final class DebeziumJsonTransactionConverter + implements DebeziumTransactionConverter +{ + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private final DebeziumSourceAdapter configuredAdapter; + + public DebeziumJsonTransactionConverter(DebeziumSourceAdapter configuredAdapter) + { + this.configuredAdapter = configuredAdapter; + } + + @Override + public SinkProto.TransactionMetadata convert(byte[] data) throws Exception + { + if (configuredAdapter == null) + { + throw new IllegalStateException( + "No Debezium source adapter configured for JSON transaction conversion"); + } + JsonNode payload = OBJECT_MAPPER.readTree(data).path("payload"); + return DebeziumEnvelopeNormalizer.normalizeTransactionMetadata( + payload, configuredAdapter); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumEnvelopeNormalizer.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumEnvelopeNormalizer.java new file mode 100644 index 0000000..960b4a5 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumEnvelopeNormalizer.java @@ -0,0 +1,118 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.support; + +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapterRegistry; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceMetadata; + +public final class DebeziumEnvelopeNormalizer +{ + private DebeziumEnvelopeNormalizer() + { + } + + public static DebeziumSourceMetadata extractSource(Object source) + { + return new DebeziumSourceMetadata( + DebeziumRecordUtil.getStringSafely(source, "connector"), + DebeziumRecordUtil.getStringSafely(source, "db"), + DebeziumRecordUtil.getStringSafely(source, "schema"), + DebeziumRecordUtil.getStringSafely(source, "table")); + } + + public static DebeziumSourceAdapter adapterForSource(Object source) + { + String payloadConnector = DebeziumRecordUtil.getStringSafely(source, "connector"); + return DebeziumSourceAdapterRegistry.forSource(payloadConnector); + } + + public static DebeziumSourceAdapter adapterForTransaction(Object transaction) + { + Object source = DebeziumRecordUtil.getFieldSafely(transaction, "source"); + if (source == null) + { + throw new IllegalArgumentException( + "Transaction metadata requires an explicit source adapter"); + } + return adapterForSource(source); + } + + public static SinkProto.SourceInfo normalizeSource(Object source) + { + return normalizeSource(source, adapterForSource(source)); + } + + public static SinkProto.SourceInfo normalizeSource( + Object source, DebeziumSourceAdapter adapter) + { + DebeziumSourceMetadata metadata = extractSource(source); + if (!metadata.connector().isBlank() && + !adapter.connector().equalsIgnoreCase(metadata.connector())) + { + throw new IllegalArgumentException( + "Configured Debezium connector " + adapter.connector() + + " does not match source.connector " + metadata.connector()); + } + + return adapter.normalizeSource(metadata); + } + + public static SinkProto.TransactionInfo normalizeTransaction( + Object transaction, DebeziumSourceAdapter adapter) + { + String rawTransactionId = DebeziumRecordUtil.getStringSafely(transaction, "id"); + return SinkProto.TransactionInfo.newBuilder() + .setId(adapter.normalizeTransactionId(rawTransactionId)) + .setTotalOrder(DebeziumRecordUtil.getLongSafely(transaction, "total_order")) + .setDataCollectionOrder( + DebeziumRecordUtil.getLongSafely(transaction, "data_collection_order")) + .build(); + } + + public static SinkProto.TransactionMetadata normalizeTransactionMetadata( + Object transaction, DebeziumSourceAdapter adapter) + { + String rawTransactionId = DebeziumRecordUtil.getStringSafely(transaction, "id"); + SinkProto.TransactionMetadata.Builder builder = SinkProto.TransactionMetadata.newBuilder() + .setStatus(DebeziumRecordUtil.getStatusSafely(transaction, "status")) + .setId(adapter.normalizeTransactionId(rawTransactionId)) + .setEventCount(DebeziumRecordUtil.getLongSafely(transaction, "event_count")) + .setTimestamp(DebeziumRecordUtil.getLongSafely(transaction, "ts_ms")); + + Object collections = DebeziumRecordUtil.getFieldSafely(transaction, "data_collections"); + if (collections instanceof Iterable iterable) + { + for (Object item : iterable) + { + String sourceDataCollection = + DebeziumRecordUtil.getStringSafely(item, "data_collection"); + builder.addDataCollections(SinkProto.DataCollection.newBuilder() + .setDataCollection( + adapter.normalizeDataCollection(sourceDataCollection)) + .setEventCount( + DebeziumRecordUtil.getLongSafely(item, "event_count"))); + } + } + return builder.build(); + } + + public static SinkProto.TransactionMetadata normalizeTransactionMetadata(Object transaction) + { + return normalizeTransactionMetadata(transaction, adapterForTransaction(transaction)); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/DeserializerUtil.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRecordUtil.java similarity index 57% rename from src/main/java/io/pixelsdb/pixels/sink/event/deserializer/DeserializerUtil.java rename to src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRecordUtil.java index 81ef25d..75eaac2 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/DeserializerUtil.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRecordUtil.java @@ -1,33 +1,33 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ +package io.pixelsdb.pixels.sink.conversion.debezium.support; -package io.pixelsdb.pixels.sink.event.deserializer; - +import com.fasterxml.jackson.databind.JsonNode; import io.pixelsdb.pixels.sink.SinkProto; import org.apache.avro.generic.GenericRecord; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.source.SourceRecord; -public class DeserializerUtil +public final class DebeziumRecordUtil { - static public SinkProto.TransactionStatus getStatusSafely(T record, String field) + private DebeziumRecordUtil() + { + } + + public static SinkProto.TransactionStatus getStatusSafely(T record, String field) { String statusString = getStringSafely(record, field); if (statusString.equals("BEGIN")) @@ -52,6 +52,10 @@ public static Object getFieldSafely(T record, String field) } else if (record instanceof Struct struct) { return struct.get(field); + } else if (record instanceof JsonNode json) + { + JsonNode value = json.get(field); + return value == null || value.isNull() ? null : value; } else if (record instanceof SourceRecord sourceRecord) { return ((Struct) sourceRecord.value()).get(field); @@ -66,49 +70,53 @@ public static Object getFieldSafely(T record, String field) public static String getStringSafely(T record, String field) { Object value = getFieldSafely(record, field); + if (value instanceof JsonNode json) + { + return json.asText(""); + } return value != null ? value.toString() : ""; } public static Long getLongSafely(T record, String field) { Object value = getFieldSafely(record, field); + if (value instanceof JsonNode json) + { + return json.asLong(); + } return value instanceof Number ? ((Number) value).longValue() : 0L; } public static Integer getIntSafely(T record, String field) { Object value = getFieldSafely(record, field); + if (value instanceof JsonNode json) + { + return json.asInt(); + } return value instanceof Number ? ((Number) value).intValue() : 0; } - static public SinkProto.OperationType getOperationType(String op) + public static SinkProto.OperationType getOperationType(String op) { - op = op.toLowerCase(); - return switch (op) + return switch (op.toLowerCase()) { case "c" -> SinkProto.OperationType.INSERT; case "u" -> SinkProto.OperationType.UPDATE; case "d" -> SinkProto.OperationType.DELETE; case "r" -> SinkProto.OperationType.SNAPSHOT; - default -> throw new IllegalArgumentException(String.format("Can't convert %s to operation type", op)); + default -> throw new IllegalArgumentException( + String.format("Can't convert %s to operation type", op)); }; } - static public boolean hasBeforeValue(SinkProto.OperationType op) + public static boolean hasBeforeValue(SinkProto.OperationType op) { return op == SinkProto.OperationType.DELETE || op == SinkProto.OperationType.UPDATE; } - static public boolean hasAfterValue(SinkProto.OperationType op) + public static boolean hasAfterValue(SinkProto.OperationType op) { return op != SinkProto.OperationType.DELETE; } - - static public String getTransIdPrefix(String originTransID) - { - return originTransID.contains(":") - ? originTransID.substring(0, originTransID.indexOf(":")) - : originTransID; - } - } diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowRecordAssembler.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowRecordAssembler.java new file mode 100644 index 0000000..8aefa8e --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowRecordAssembler.java @@ -0,0 +1,66 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.support; + +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.event.RowChangeEventFactory; +import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; + +public final class DebeziumRowRecordAssembler +{ + public RowChangeEvent assemble( + SinkProto.OperationType operation, + SinkProto.SourceInfo source, + SinkProto.TransactionInfo transaction, + SinkProto.RowValue before, + SinkProto.RowValue after, + TypeDescription schema) throws SinkException + { + return assemble(operation, source, transaction, before, after, schema, null); + } + + public RowChangeEvent assemble( + SinkProto.OperationType operation, + SinkProto.SourceInfo source, + SinkProto.TransactionInfo transaction, + SinkProto.RowValue before, + SinkProto.RowValue after, + TypeDescription schema, + TableMetadata metadata) throws SinkException + { + SinkProto.RowRecord.Builder builder = SinkProto.RowRecord.newBuilder() + .setOp(operation) + .setSource(source); + if (transaction != null) + { + builder.setTransaction(transaction); + } + if (before != null) + { + builder.setBefore(before); + } + if (after != null) + { + builder.setAfter(after); + } + return metadata == null + ? RowChangeEventFactory.create(builder.build(), schema) + : RowChangeEventFactory.create(builder.build(), schema, metadata); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowDataParser.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowValueConverter.java similarity index 50% rename from src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowDataParser.java rename to src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowValueConverter.java index 5c85d0a..6297e57 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowDataParser.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowValueConverter.java @@ -1,49 +1,38 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - -package io.pixelsdb.pixels.sink.event.deserializer; +package io.pixelsdb.pixels.sink.conversion.debezium.support; import com.fasterxml.jackson.databind.JsonNode; import com.google.protobuf.ByteString; import io.pixelsdb.pixels.core.TypeDescription; import io.pixelsdb.pixels.sink.SinkProto; import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.connect.data.Field; -import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.Struct; -import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.time.LocalDate; import java.util.Base64; -import java.util.Collections; -import java.util.Map; -class RowDataParser +public class DebeziumRowValueConverter { private final TypeDescription schema; - public RowDataParser(TypeDescription schema) + public DebeziumRowValueConverter(TypeDescription schema) { this.schema = schema; } @@ -55,8 +44,15 @@ private static void buildFloat32(float value, SinkProto.ColumnValue.Builder colu private static void buildInt32(int value, int capacity, SinkProto.ColumnValue.Builder columnValueBuilder) { - int intBits = value; - byte[] bytes = ByteBuffer.allocate(capacity).putInt(intBits).array(); + ByteBuffer buffer = ByteBuffer.allocate(capacity); + switch (capacity) + { + case Byte.BYTES -> buffer.put((byte) value); + case Short.BYTES -> buffer.putShort((short) value); + case Integer.BYTES -> buffer.putInt(value); + default -> throw new IllegalArgumentException("Unsupported integer width: " + capacity); + } + byte[] bytes = buffer.array(); columnValueBuilder.setValue(ByteString.copyFrom(bytes)); } @@ -66,7 +62,7 @@ public void parse(GenericRecord record, SinkProto.RowValue.Builder builder) { String fieldName = schema.getFieldNames().get(i); TypeDescription fieldType = schema.getChildren().get(i); - builder.addValues(parseValue(record, fieldName, fieldType).build()); + builder.addValues(parseCanonicalValue(record.get(fieldName), fieldType).build()); } } @@ -85,9 +81,12 @@ public void parse(Struct record, SinkProto.RowValue.Builder builder) for (int i = 0; i < schema.getFieldNames().size(); i++) { String fieldName = schema.getFieldNames().get(i); - Field field = record.schema().field(fieldName); - Schema.Type fieldType = field.schema().type(); - builder.addValues(parseValue(record.get(fieldName), fieldName, fieldType).build()); + TypeDescription fieldType = schema.getChildren().get(i); + if (record.schema().field(fieldName) == null) + { + throw new IllegalArgumentException("Missing field in Debezium row schema: " + fieldName); + } + builder.addValues(parseCanonicalValue(record.get(fieldName), fieldType).build()); } } @@ -104,6 +103,16 @@ private SinkProto.ColumnValue.Builder parseValue(JsonNode valueNode, String fiel switch (type.getCategory()) { + case BYTE: + { + buildInt32(valueNode.asInt(), Byte.BYTES, columnValueBuilder); + break; + } + case SHORT: + { + buildInt32(valueNode.asInt(), Short.BYTES, columnValueBuilder); + break; + } case INT: { buildInt32(valueNode.asInt(), Integer.BYTES, columnValueBuilder); @@ -131,7 +140,7 @@ private SinkProto.ColumnValue.Builder parseValue(JsonNode valueNode, String fiel case STRING: case VARBINARY: { - String value = valueNode.asText().trim(); + String value = valueNode.asText(); columnValueBuilder.setValue(ByteString.copyFrom(value, StandardCharsets.UTF_8)); // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.STRING)); break; @@ -175,6 +184,7 @@ private SinkProto.ColumnValue.Builder parseValue(JsonNode valueNode, String fiel break; } case DATE: + case TIME: { buildInt32(valueNode.asInt(), Integer.BYTES, columnValueBuilder); // columnValueBuilder.setType(PixelsProto.Type.newBuilder() @@ -197,168 +207,112 @@ private SinkProto.ColumnValue.Builder parseValue(JsonNode valueNode, String fiel return columnValueBuilder; } - @Deprecated // TODO: use bit - private SinkProto.ColumnValue.Builder parseValue(GenericRecord record, String fieldName, TypeDescription fieldType) + private SinkProto.ColumnValue.Builder parseCanonicalValue(Object raw, TypeDescription type) { - SinkProto.ColumnValue.Builder columnValueBuilder = SinkProto.ColumnValue.newBuilder(); - // columnValueBuilder.setName(fieldName); - - Object raw = record.get(fieldName); if (raw == null) { - columnValueBuilder.setValue(ByteString.EMPTY); - return columnValueBuilder; + return SinkProto.ColumnValue.newBuilder() + .setValue(ByteString.EMPTY); + } + if (raw instanceof JsonNode jsonNode) + { + return parseValue(jsonNode, "", type); } - switch (fieldType.getCategory()) + SinkProto.ColumnValue.Builder builder = SinkProto.ColumnValue.newBuilder(); + switch (type.getCategory()) { + case BYTE: + buildInt32(((Number) raw).intValue(), Byte.BYTES, builder); + break; + case SHORT: + buildInt32(((Number) raw).intValue(), Short.BYTES, builder); + break; case INT: - { - int value = (int) raw; - columnValueBuilder.setValue(ByteString.copyFrom(Integer.toString(value), StandardCharsets.UTF_8)); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.INT)); + buildInt32(((Number) raw).intValue(), Integer.BYTES, builder); break; - } - case LONG: + case TIMESTAMP: { - long value = (long) raw; - columnValueBuilder.setValue(ByteString.copyFrom(Long.toString(value), StandardCharsets.UTF_8)); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.LONG)); + byte[] bytes = ByteBuffer.allocate(Long.BYTES) + .putLong(((Number) raw).longValue()).array(); + builder.setValue(ByteString.copyFrom(bytes)); break; } - + case DATE: + case TIME: + buildInt32(((Number) raw).intValue(), Integer.BYTES, builder); + break; + case CHAR: + case VARCHAR: case STRING: - { - String value = raw.toString(); - columnValueBuilder.setValue(ByteString.copyFrom(value, StandardCharsets.UTF_8)); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.STRING)); + builder.setValue(ByteString.copyFrom(raw.toString(), StandardCharsets.UTF_8)); break; - } - case DECIMAL: { - ByteBuffer buffer = (ByteBuffer) raw; - String decimalStr = new String(buffer.array(), StandardCharsets.UTF_8).trim(); - columnValueBuilder.setValue(ByteString.copyFrom(decimalStr, StandardCharsets.UTF_8)); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder() -// .setKind(PixelsProto.Type.Kind.DECIMAL) -// .setDimension(fieldType.getPrecision()) -// .setScale(fieldType.getScale())); - break; - } - - case DATE: - { - int epochDay = (int) raw; - String isoDate = LocalDate.ofEpochDay(epochDay).toString(); // e.g., "2025-07-03" - columnValueBuilder.setValue(ByteString.copyFrom(isoDate, StandardCharsets.UTF_8)); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.DATE)); + BigDecimal decimal = toBigDecimal(raw, type); + builder.setValue(ByteString.copyFrom( + decimal.toPlainString(), StandardCharsets.UTF_8)); break; } - case BINARY: - { - ByteBuffer buffer = (ByteBuffer) raw; - // encode as hex or base64 if needed, otherwise just dump as UTF-8 string if it's meant to be readable - String base64 = Base64.getEncoder().encodeToString(buffer.array()); - columnValueBuilder.setValue(ByteString.copyFrom(base64, StandardCharsets.UTF_8)); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.BINARY)); - break; - } - default: - throw new IllegalArgumentException("Unsupported type: " + fieldType.getCategory()); - } - - return columnValueBuilder; - } - - private SinkProto.ColumnValue.Builder parseValue(Object record, String fieldName, Schema.Type type) - { - // TODO(AntiO2) support pixels type - if (record == null) - { - return SinkProto.ColumnValue.newBuilder() - // .setName(fieldName) - .setValue(ByteString.EMPTY); - } - - SinkProto.ColumnValue.Builder columnValueBuilder = SinkProto.ColumnValue.newBuilder(); - switch (type) - { - case INT8: - case INT16: - { - buildInt32((Short) record, Integer.BYTES, columnValueBuilder); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.INT)); - break; - } - case INT32: - { - buildInt32((Integer) record, Integer.BYTES, columnValueBuilder); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.INT)); + case VARBINARY: + builder.setValue(ByteString.copyFrom(toBytes(raw))); break; - } - case INT64: + case DOUBLE: { - long value = (Long) record; - byte[] bytes = ByteBuffer.allocate(Long.BYTES).putLong(value).array(); - columnValueBuilder.setValue(ByteString.copyFrom(bytes)); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.LONG)); + long bits = Double.doubleToLongBits(((Number) raw).doubleValue()); + builder.setValue(ByteString.copyFrom( + ByteBuffer.allocate(Long.BYTES).putLong(bits).array())); break; } - case BYTES: - { - if (record instanceof BigDecimal) - { - float value = ((BigDecimal) record).floatValue(); - buildFloat32(value, columnValueBuilder); - break; - } - byte[] bytes = (byte[]) record; - columnValueBuilder.setValue(ByteString.copyFrom(bytes)); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.BYTE)); + case FLOAT: + buildFloat32(((Number) raw).floatValue(), builder); break; - } case BOOLEAN: - case STRING: - { - String value = (String) record; - columnValueBuilder.setValue(ByteString.copyFrom(value, StandardCharsets.UTF_8)); -// columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.STRING)); + builder.setValue(ByteString.copyFrom( + new byte[]{(byte) ((Boolean) raw ? 1 : 0)})); break; - } case STRUCT: - { - // You can recursively parse fields in a struct here throw new UnsupportedOperationException("STRUCT parsing not yet implemented"); - } - case FLOAT64: - { - double value = (double) record; - long doubleBits = Double.doubleToLongBits(value); - byte[] bytes = ByteBuffer.allocate(Long.BYTES).putLong(doubleBits).array(); - columnValueBuilder.setValue(ByteString.copyFrom(bytes)); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.DOUBLE)); - break; - } - case FLOAT32: - { - float value = (float) record; - buildFloat32(value, columnValueBuilder); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.FLOAT)); - break; - } default: - throw new IllegalArgumentException("Unsupported type: " + type); + throw new IllegalArgumentException( + "Unsupported canonical type: " + type.getCategory()); } + return builder; + } - return columnValueBuilder; + private BigDecimal toBigDecimal(Object raw, TypeDescription type) + { + if (raw instanceof BigDecimal decimal) + { + return decimal; + } + if (raw instanceof ByteBuffer buffer) + { + return new BigDecimal(new BigInteger(toBytes(buffer)), type.getScale()); + } + if (raw instanceof byte[] bytes) + { + return new BigDecimal(new BigInteger(bytes), type.getScale()); + } + return new BigDecimal(raw.toString()); } - private Map parseDeleteRecord() + private byte[] toBytes(Object raw) { - return Collections.singletonMap("__deleted", true); + if (raw instanceof byte[] bytes) + { + return bytes; + } + if (raw instanceof ByteBuffer buffer) + { + ByteBuffer copy = buffer.duplicate(); + byte[] bytes = new byte[copy.remaining()]; + copy.get(bytes); + return bytes; + } + return raw.toString().getBytes(StandardCharsets.UTF_8); } BigDecimal parseDecimal(JsonNode node, TypeDescription type) @@ -368,19 +322,4 @@ BigDecimal parseDecimal(JsonNode node, TypeDescription type) return new BigDecimal(new BigInteger(bytes), scale); } - private LocalDate parseDate(JsonNode node) - { - return LocalDate.ofEpochDay(node.asLong()); - } - - private byte[] parseBinary(JsonNode node) - { - try - { - return node.binaryValue(); - } catch (IOException e) - { - throw new RuntimeException("Binary parsing failed", e); - } - } -} \ No newline at end of file +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverter.java new file mode 100644 index 0000000..319c657 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverter.java @@ -0,0 +1,51 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.sinkproto; + +import com.google.protobuf.InvalidProtocolBufferException; +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.event.RowChangeEventFactory; +import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; + +import java.nio.ByteBuffer; + +public final class RowRecordConverter +{ + private final TableMetadataRegistry tableMetadataRegistry; + + public RowRecordConverter(TableMetadataRegistry tableMetadataRegistry) + { + this.tableMetadataRegistry = tableMetadataRegistry; + } + + public RowChangeEvent convert(SinkProto.RowRecord rowRecord) throws SinkException + { + SinkProto.SourceInfo source = rowRecord.getSource(); + TableMetadata metadata = tableMetadataRegistry.getMetadata( + source.getDb(), source.getTable()); + TypeDescription schema = metadata.getTypeDescription(); + return RowChangeEventFactory.create(rowRecord, schema, metadata); + } + + public SinkProto.RowRecord parse(ByteBuffer buffer) throws InvalidProtocolBufferException + { + return SinkProto.RowRecord.parseFrom(buffer.duplicate()); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/TransactionMetadataConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/TransactionMetadataConverter.java new file mode 100644 index 0000000..b40a070 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/TransactionMetadataConverter.java @@ -0,0 +1,34 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.sinkproto; + +import com.google.protobuf.InvalidProtocolBufferException; +import io.pixelsdb.pixels.sink.SinkProto; + +import java.nio.ByteBuffer; + +public final class TransactionMetadataConverter +{ + public SinkProto.TransactionMetadata convert(ByteBuffer buffer, int loopId) + throws InvalidProtocolBufferException + { + SinkProto.TransactionMetadata metadata = + SinkProto.TransactionMetadata.parseFrom(buffer.duplicate()); + return metadata.toBuilder() + .setId(metadata.getId() + "_" + loopId) + .build(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEvent.java b/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEvent.java index b3588fd..19336e7 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEvent.java +++ b/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEvent.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.event; import com.google.protobuf.ByteString; @@ -30,8 +25,6 @@ import io.pixelsdb.pixels.sink.exception.SinkException; import io.pixelsdb.pixels.sink.metadata.TableMetadata; import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; -import io.pixelsdb.pixels.sink.util.MetricsFacade; -import io.prometheus.client.Summary; import lombok.Getter; import lombok.Setter; @@ -47,7 +40,6 @@ public class RowChangeEvent @Getter private final SinkProto.RowRecord rowRecord; - private final MetricsFacade metricsFacade = MetricsFacade.getInstance(); @Getter private final TypeDescription schema; /** @@ -60,7 +52,6 @@ public class RowChangeEvent private String topic; @Getter private TableMetadata tableMetadata = null; - private Summary.Timer latencyTimer; private Map beforeValueMap; private Map afterValueMap; @Getter @@ -78,20 +69,31 @@ public class RowChangeEvent public RowChangeEvent(SinkProto.RowRecord rowRecord) throws SinkException { - this.rowRecord = rowRecord; TableMetadataRegistry tableMetadataRegistry = TableMetadataRegistry.Instance(); - this.schema = tableMetadataRegistry.getTypeDescription(getSchemaName(), getTable()); + this.rowRecord = rowRecord; + this.schema = tableMetadataRegistry.getTypeDescription( + rowRecord.getSource().getDb(), rowRecord.getSource().getTable()); + this.tableMetadata = tableMetadataRegistry.getMetadata( + rowRecord.getSource().getDb(), rowRecord.getSource().getTable()); init(); initIndexKey(); } public RowChangeEvent(SinkProto.RowRecord rowRecord, TypeDescription schema) throws SinkException + { + this(rowRecord, schema, TableMetadataRegistry.Instance().getMetadata( + rowRecord.getSource().getDb(), rowRecord.getSource().getTable())); + } + + public RowChangeEvent( + SinkProto.RowRecord rowRecord, + TypeDescription schema, + TableMetadata tableMetadata) throws SinkException { this.rowRecord = rowRecord; this.schema = schema; - + this.tableMetadata = tableMetadata; init(); - // initIndexKey(); } protected static int getBucketFromIndexKey(IndexProto.IndexKey indexKey) @@ -106,8 +108,7 @@ protected static int getBucketIdFromByteBuffer(ByteString byteString) private void init() throws SinkException { - TableMetadataRegistry tableMetadataRegistry = TableMetadataRegistry.Instance(); - this.tableId = tableMetadataRegistry.getTableId(getSchemaName(), getTable()); + this.tableId = tableMetadata == null ? 0 : tableMetadata.getTableId(); this.schemaTableName = new SchemaTableName(getSchemaName(), getTable()); initColumnValueMap(); @@ -141,9 +142,10 @@ public void initIndexKey() throws SinkException return; } - this.tableMetadata = TableMetadataRegistry.Instance().getMetadata( - this.rowRecord.getSource().getDb(), - this.rowRecord.getSource().getTable()); + if (this.tableMetadata == null) + { + throw new SinkException("Row change table metadata is missing"); + } if (!this.tableMetadata.hasPrimaryIndex()) { @@ -258,16 +260,14 @@ public String getTable() public String getFullTableName() { - // TODO(AntiO2): In postgresql, data collection uses schemaName as prefix, while MySQL uses DB as prefix. - return rowRecord.getSource().getSchema() + "." + rowRecord.getSource().getTable(); - // return getSchemaName() + "." + getTable(); + SinkProto.SourceInfo source = rowRecord.getSource(); + String namespace = source.getSchema().isBlank() ? source.getDb() : source.getSchema(); + return namespace + "." + source.getTable(); } - // TODO(AntiO2): How to Map Schema Names Between Source DB and Pixels public String getSchemaName() { return rowRecord.getSource().getDb(); - // return rowRecord.getSource().getSchema(); } public boolean hasError() @@ -310,20 +310,6 @@ public boolean hasAfterData() return isUpdate() || isInsert() || isSnapshot(); } - public void startLatencyTimer() - { - this.latencyTimer = metricsFacade.startProcessLatencyTimer(); - } - - public void endLatencyTimer() - { - if (latencyTimer != null) - { - this.latencyTimer.close(); - } - - } - public SinkProto.OperationType getOp() { return rowRecord.getOp(); diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactory.java b/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactory.java new file mode 100644 index 0000000..07eabdc --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactory.java @@ -0,0 +1,101 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.event; + +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; + +public final class RowChangeEventFactory +{ + private RowChangeEventFactory() + { + } + + public static RowChangeEvent create( + SinkProto.OperationType operation, + SinkProto.SourceInfo source, + SinkProto.TransactionInfo transaction, + SinkProto.RowValue before, + SinkProto.RowValue after, + TypeDescription schema) throws SinkException + { + if (operation == null) + { + throw new SinkException("Row change operation is missing"); + } + if (source == null) + { + throw new SinkException("Row change source is missing"); + } + + SinkProto.RowRecord.Builder builder = SinkProto.RowRecord.newBuilder() + .setOp(operation) + .setSource(source); + if (transaction != null) + { + builder.setTransaction(transaction); + } + if (before != null) + { + builder.setBefore(before); + } + if (after != null) + { + builder.setAfter(after); + } + return create(builder.build(), schema); + } + + public static RowChangeEvent create( + SinkProto.RowRecord rowRecord, + TypeDescription schema) throws SinkException + { + if (rowRecord == null) + { + throw new SinkException("Row change record is missing"); + } + if (schema == null) + { + throw new SinkException("Row change schema is missing"); + } + TableMetadataRegistry registry = TableMetadataRegistry.Instance(); + TableMetadata metadata = registry.getMetadata( + rowRecord.getSource().getDb(), rowRecord.getSource().getTable()); + return create(rowRecord, schema, metadata); + } + + public static RowChangeEvent create( + SinkProto.RowRecord rowRecord, + TypeDescription schema, + TableMetadata metadata) throws SinkException + { + if (rowRecord == null) + { + throw new SinkException("Row change record is missing"); + } + if (schema == null) + { + throw new SinkException("Row change schema is missing"); + } + + RowChangeEvent event = new RowChangeEvent(rowRecord, schema, metadata); + event.initIndexKey(); + return event; + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventAvroDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventAvroDeserializer.java deleted file mode 100644 index 7a2711a..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventAvroDeserializer.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - -import io.apicurio.registry.serde.SerdeConfig; -import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; -import io.pixelsdb.pixels.core.TypeDescription; -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.exception.SinkException; -import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; -import io.pixelsdb.pixels.sink.util.MetricsFacade; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.common.serialization.Deserializer; - -import java.util.HashMap; -import java.util.Map; - -public class RowChangeEventAvroDeserializer implements Deserializer -{ - - private final AvroKafkaDeserializer avroDeserializer = new AvroKafkaDeserializer<>(); - private final TableMetadataRegistry tableMetadataRegistry = TableMetadataRegistry.Instance(); - private final PixelsSinkConfig config = PixelsSinkConfigFactory.getInstance(); - - @Override - public void configure(Map configs, boolean isKey) - { - Map enrichedConfig = new HashMap<>(configs); - enrichedConfig.put(SerdeConfig.REGISTRY_URL, config.getRegistryUrl()); - enrichedConfig.put(SerdeConfig.CHECK_PERIOD_MS, SerdeConfig.CHECK_PERIOD_MS_DEFAULT); - avroDeserializer.configure(enrichedConfig, isKey); - } - - @Override - public RowChangeEvent deserialize(String topic, byte[] data) - { - try - { - MetricsFacade.getInstance().addRawData(data.length); - GenericRecord avroRecord = avroDeserializer.deserialize(topic, data); - Schema avroSchema = avroRecord.getSchema(); - RowChangeEvent rowChangeEvent = convertToRowChangeEvent(avroRecord, avroSchema); - return rowChangeEvent; - } catch (Exception e) - { - e.printStackTrace(); - return null; - // throw new SerializationException("Avro deserialization failed", e); - // TODO: 这里有些Row Change Event 不是完整的结构。 - } - } - - private void registerSchema(String topic, Schema avroSchema) - { - - } - - private RowChangeEvent convertToRowChangeEvent(GenericRecord avroRecord, Schema schema) throws SinkException - { - SinkProto.OperationType op = parseOperationType(avroRecord); - SinkProto.RowRecord.Builder recordBuilder = SinkProto.RowRecord.newBuilder() - .setOp(op) -// .setTsMs(DeserializerUtil.getLongSafely(avroRecord, "ts_ms")); - ; - if (avroRecord.get("source") != null) - { - //TODO: 这里看下怎么处理，如果没有source信息，其实可以通过topic推出schema和table信息。 - parseSourceInfo((GenericRecord) avroRecord.get("source"), recordBuilder.getSourceBuilder()); - } - - String sourceSchema = recordBuilder.getSource().getDb(); - String sourceTable = recordBuilder.getSource().getTable(); - TypeDescription typeDescription = null; - try - { - typeDescription = tableMetadataRegistry.getTypeDescription(sourceSchema, sourceTable); - } catch (SinkException e) - { - throw new RuntimeException(e); - } - // TableMetadata tableMetadata = tableMetadataRegistry.loadTableMetadata(sourceSchema, sourceTable); - - recordBuilder.setBefore(parseRowData(avroRecord.get("before"), typeDescription)); - recordBuilder.setAfter(parseRowData(avroRecord.get("after"), typeDescription)); - - if (avroRecord.get("transaction") != null) - { - parseTransactionInfo((GenericRecord) avroRecord.get("transaction"), - recordBuilder.getTransactionBuilder()); - } - - return new RowChangeEvent(recordBuilder.build(), typeDescription); - } - - private SinkProto.OperationType parseOperationType(GenericRecord record) - { - String op = DeserializerUtil.getStringSafely(record, "op"); - try - { - return DeserializerUtil.getOperationType(op); - } catch (IllegalArgumentException e) - { - return SinkProto.OperationType.UNRECOGNIZED; - } - } - - private SinkProto.RowValue.Builder parseRowData(Object data, TypeDescription typeDescription) - { - SinkProto.RowValue.Builder builder = SinkProto.RowValue.newBuilder(); - if (data instanceof GenericRecord rowData) - { - RowDataParser rowDataParser = new RowDataParser(typeDescription); // TODO make it static? - rowDataParser.parse(rowData, builder); - } - return builder; - } - - private void parseSourceInfo(GenericRecord source, SinkProto.SourceInfo.Builder builder) - { - - builder - .setDb(DeserializerUtil.getStringSafely(source, "db")) - .setSchema(DeserializerUtil.getStringSafely(source, "schema")) - .setTable(DeserializerUtil.getStringSafely(source, "table")) -// .setVersion(DeserializerUtil.getStringSafely(source, "version")) -// .setConnector(DeserializerUtil.getStringSafely(source, "connector")) -// .setName(DeserializerUtil.getStringSafely(source, "name")) -// .setTsMs(DeserializerUtil.getLongSafely(source, "ts_ms")) -// .setSnapshot(DeserializerUtil.getStringSafely(source, "snapshot")) -// -// .setSequence(DeserializerUtil.getStringSafely(source, "sequence")) - -// .setTxId(DeserializerUtil.getLongSafely(source, "tx_id")) -// .setLsn(DeserializerUtil.getLongSafely(source, "lsn")) -// .setXmin(DeserializerUtil.getLongSafely(source, "xmin")) - ; - } - - private void parseTransactionInfo(GenericRecord transaction, - SinkProto.TransactionInfo.Builder builder) - { - builder.setId(DeserializerUtil.getTransIdPrefix(DeserializerUtil.getStringSafely(transaction, "id"))) - .setTotalOrder(DeserializerUtil.getLongSafely(transaction, "total_order")) - .setDataCollectionOrder(DeserializerUtil.getLongSafely(transaction, "data_collection_order")); - } - -} \ No newline at end of file diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventJsonDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventJsonDeserializer.java deleted file mode 100644 index 6d55297..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventJsonDeserializer.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.pixelsdb.pixels.core.TypeDescription; -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.exception.SinkException; -import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; -import io.pixelsdb.pixels.sink.util.MetricsFacade; -import org.apache.kafka.common.serialization.Deserializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -public class RowChangeEventJsonDeserializer implements Deserializer -{ - private static final Logger logger = LoggerFactory.getLogger(RowChangeEventJsonDeserializer.class); - private static final ObjectMapper objectMapper = new ObjectMapper(); - private final TableMetadataRegistry tableMetadataRegistry = TableMetadataRegistry.Instance(); - - @Override - public RowChangeEvent deserialize(String topic, byte[] data) - { - if (data == null || data.length == 0) - { - logger.debug("Received empty message from topic: {}", topic); - return null; - } - MetricsFacade.getInstance().addRawData(data.length); - try - { - JsonNode rootNode = objectMapper.readTree(data); - JsonNode payloadNode = rootNode.path("payload"); - - SinkProto.OperationType opType = parseOperationType(payloadNode); - - return buildRowRecord(payloadNode, opType); - } catch (Exception e) - { - logger.error("Failed to deserialize message from topic {}: {}", topic, e.getMessage()); - return null; - } - } - - private SinkProto.OperationType parseOperationType(JsonNode payloadNode) - { - String opCode = payloadNode.path("op").asText(""); - return DeserializerUtil.getOperationType(opCode); - } - - @Deprecated - private TypeDescription getSchema(JsonNode schemaNode, SinkProto.OperationType opType) - { - return switch (opType) - { - case DELETE -> SchemaDeserializer.parseFromBeforeOrAfter(schemaNode, "before"); - case INSERT, UPDATE, SNAPSHOT -> SchemaDeserializer.parseFromBeforeOrAfter(schemaNode, "after"); - case UNRECOGNIZED -> throw new IllegalArgumentException("Operation type is unknown. Check op"); - }; - } - - private RowChangeEvent buildRowRecord(JsonNode payloadNode, - SinkProto.OperationType opType) throws SinkException - { - - SinkProto.RowRecord.Builder builder = SinkProto.RowRecord.newBuilder(); - - builder.setOp(parseOperationType(payloadNode)); -// .setTsMs(payloadNode.path("ts_ms").asLong()) -// .setTsUs(payloadNode.path("ts_us").asLong()) -// .setTsNs(payloadNode.path("ts_ns").asLong()); - - String schemaName; - String tableName; - if (payloadNode.has("source")) - { - SinkProto.SourceInfo.Builder sourceInfoBuilder = parseSourceInfo(payloadNode.get("source")); - schemaName = sourceInfoBuilder.getDb(); // Notice we use the schema - tableName = sourceInfoBuilder.getTable(); - builder.setSource(sourceInfoBuilder); - } else - { - throw new IllegalArgumentException("Missing source field in row record"); - } - - TypeDescription typeDescription = tableMetadataRegistry.getTypeDescription(schemaName, tableName); - RowDataParser rowDataParser = new RowDataParser(typeDescription); - if (payloadNode.hasNonNull("transaction")) - { - builder.setTransaction(parseTransactionInfo(payloadNode.get("transaction"))); - } - - if (DeserializerUtil.hasBeforeValue(opType)) - { - SinkProto.RowValue.Builder beforeBuilder = builder.getBeforeBuilder(); - rowDataParser.parse(payloadNode.get("before"), beforeBuilder); - builder.setBefore(beforeBuilder); - } - - if (DeserializerUtil.hasAfterValue(opType)) - { - - SinkProto.RowValue.Builder afterBuilder = builder.getAfterBuilder(); - rowDataParser.parse(payloadNode.get("after"), afterBuilder); - builder.setAfter(afterBuilder); - } - - RowChangeEvent event = new RowChangeEvent(builder.build(), typeDescription); - try - { - event.initIndexKey(); - } catch (SinkException e) - { - logger.warn("Row change event {}: Init index key failed", event); - } - - return event; - } - - private SinkProto.SourceInfo.Builder parseSourceInfo(JsonNode sourceNode) - { - return SinkProto.SourceInfo.newBuilder() - .setDb(sourceNode.path("db").asText()) - .setSchema(sourceNode.path("schema").asText()) - .setTable(sourceNode.path("table").asText()) -// .setVersion(sourceNode.path("version").asText()) -// .setConnector(sourceNode.path("connector").asText()) -// .setName(sourceNode.path("name").asText()) -// .setTsMs(sourceNode.path("ts_ms").asLong()) -// .setSnapshot(sourceNode.path("snapshot").asText()) - -// .setSequence(sourceNode.path("sequence").asText()) -// .setTsUs(sourceNode.path("ts_us").asLong()) -// .setTsNs(sourceNode.path("ts_ns").asLong()) - -// .setTxId(sourceNode.path("txId").asLong()) -// .setLsn(sourceNode.path("lsn").asLong()) -// .setXmin(sourceNode.path("xmin").asLong()) - ; - } - - private SinkProto.TransactionInfo parseTransactionInfo(JsonNode txNode) - { - return SinkProto.TransactionInfo.newBuilder() - .setId(DeserializerUtil.getTransIdPrefix(txNode.path("id").asText())) - .setTotalOrder(txNode.path("total_order").asLong()) - .setDataCollectionOrder(txNode.path("data_collection_order").asLong()) - .build(); - } - -} - diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventStructDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventStructDeserializer.java deleted file mode 100644 index 498d382..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventStructDeserializer.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - - -import io.pixelsdb.pixels.core.TypeDescription; -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.exception.SinkException; -import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; -import org.apache.kafka.connect.data.Struct; -import org.apache.kafka.connect.errors.DataException; -import org.apache.kafka.connect.source.SourceRecord; - -import java.util.logging.Logger; - -/** - * @package: io.pixelsdb.pixels.sink.event.deserializer - * @className: RowChangeEventStructDeserializer - * @author: AntiO2 - * @date: 2025/9/26 12:00 - */ -public class RowChangeEventStructDeserializer -{ - private static final Logger LOGGER = Logger.getLogger(RowChangeEventStructDeserializer.class.getName()); - private static final TableMetadataRegistry tableMetadataRegistry = TableMetadataRegistry.Instance(); - - public static RowChangeEvent convertToRowChangeEvent(SourceRecord sourceRecord) throws SinkException - { - Struct value = (Struct) sourceRecord.value(); - String op = value.getString("op"); - SinkProto.OperationType operationType = DeserializerUtil.getOperationType(op); - return buildRowRecord(value, operationType); - } - - public static RowChangeEvent convertToRowChangeEvent(SinkProto.RowRecord rowRecord) throws SinkException - { - String schemaName = rowRecord.getSource().getDb(); - String tableName = rowRecord.getSource().getTable(); - TypeDescription typeDescription = tableMetadataRegistry.getTypeDescription(schemaName, tableName); - return new RowChangeEvent(rowRecord, typeDescription); - } - - private static RowChangeEvent buildRowRecord(Struct value, - SinkProto.OperationType opType) throws SinkException - { - - SinkProto.RowRecord.Builder builder = SinkProto.RowRecord.newBuilder(); - - builder.setOp(opType); - - String schemaName; - String tableName; - try - { - Struct source = value.getStruct("source"); - SinkProto.SourceInfo.Builder sourceInfoBuilder = parseSourceInfo(source); - schemaName = sourceInfoBuilder.getDb(); // Notice we use the schema - tableName = sourceInfoBuilder.getTable(); - builder.setSource(sourceInfoBuilder); - } catch (DataException e) - { - LOGGER.warning("Missing source field in row record"); - throw new SinkException(e); - } - - TypeDescription typeDescription = tableMetadataRegistry.getTypeDescription(schemaName, tableName); - RowDataParser rowDataParser = new RowDataParser(typeDescription); - - try - { - Struct transaction = value.getStruct("transaction"); - SinkProto.TransactionInfo transactionInfo = parseTransactionInfo(transaction); - builder.setTransaction(transactionInfo); - } catch (DataException e) - { - LOGGER.warning("Missing transaction field in row record"); - } - - if (DeserializerUtil.hasBeforeValue(opType)) - { - SinkProto.RowValue.Builder beforeBuilder = builder.getBeforeBuilder(); - rowDataParser.parse(value.getStruct("before"), beforeBuilder); - builder.setBefore(beforeBuilder); - } - - if (DeserializerUtil.hasAfterValue(opType)) - { - - SinkProto.RowValue.Builder afterBuilder = builder.getAfterBuilder(); - rowDataParser.parse(value.getStruct("after"), afterBuilder); - builder.setAfter(afterBuilder); - } - - RowChangeEvent event = new RowChangeEvent(builder.build(), typeDescription); - return event; - } - - private static SinkProto.SourceInfo.Builder parseSourceInfo(T source) - { - return SinkProto.SourceInfo.newBuilder() - // .setVersion(DeserializerUtil.getStringSafely(source, "version")) - // .setConnector(DeserializerUtil.getStringSafely(source, "connector")) -// .setName(DeserializerUtil.getStringSafely(source, "name")) -// .setTsMs(DeserializerUtil.getLongSafely(source, "ts_ms")) -// .setSnapshot(DeserializerUtil.getStringSafely(source, "snapshot")) - .setDb(DeserializerUtil.getStringSafely(source, "db")) -// .setSequence(DeserializerUtil.getStringSafely(source, "sequence")) -// .setTsUs(DeserializerUtil.getLongSafely(source, "ts_us")) -// .setTsNs(DeserializerUtil.getLongSafely(source, "ts_ns")) - .setSchema(DeserializerUtil.getStringSafely(source, "schema")) - .setTable(DeserializerUtil.getStringSafely(source, "table")); -// .setTxId(DeserializerUtil.getLongSafely(source, "txId")) -// .setLsn(DeserializerUtil.getLongSafely(source, "lsn")) -// .setXmin(DeserializerUtil.getLongSafely(source, "xmin")); - } - - private static SinkProto.TransactionInfo parseTransactionInfo(T txNode) - { - return SinkProto.TransactionInfo.newBuilder() - .setId(DeserializerUtil.getTransIdPrefix( - DeserializerUtil.getStringSafely(txNode, "id"))) - .setTotalOrder(DeserializerUtil.getLongSafely(txNode, "total_order")) - .setDataCollectionOrder(DeserializerUtil.getLongSafely(txNode, "data_collection_order")) - .build(); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/SchemaDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/SchemaDeserializer.java deleted file mode 100644 index d33f3fe..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/SchemaDeserializer.java +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - -import com.fasterxml.jackson.databind.JsonNode; -import io.pixelsdb.pixels.core.TypeDescription; -import org.apache.avro.Schema; - -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -public class SchemaDeserializer -{ - public static TypeDescription parseFromBeforeOrAfter(JsonNode schemaNode, String fieldName) - { - JsonNode beforeAfterSchema = findSchemaField(schemaNode, fieldName); - if (beforeAfterSchema == null) - { - throw new IllegalArgumentException("Field '" + fieldName + "' not found in schema"); - } - return parseStruct(beforeAfterSchema.get("fields")); - } - - private static JsonNode findSchemaField(JsonNode schemaNode, String targetField) - { - Iterator fields = schemaNode.get("fields").elements(); - while (fields.hasNext()) - { - JsonNode field = fields.next(); - if (targetField.equals(field.get("field").asText())) - { - return field; - } - } - return null; - } - - public static TypeDescription parseStruct(JsonNode fields) - { - TypeDescription structType = TypeDescription.createStruct(); - fields.forEach(field -> - { - String name = field.get("field").asText(); - TypeDescription fieldType = parseFieldType(field); - structType.addField(name, fieldType); - }); - return structType; - } - - static TypeDescription parseFieldType(JsonNode fieldNode) - { - if (!fieldNode.has("type")) - { - throw new IllegalArgumentException("Field is missing required 'type' property"); - } - String typeName = fieldNode.get("type").asText(); - String logicalType = fieldNode.has("name") ? fieldNode.get("name").asText() : null; - - if (logicalType != null) - { - switch (logicalType) - { - case "org.apache.kafka.connect.data.Decimal": - int precision = Integer.parseInt(fieldNode.get("parameters").get("connect.decimal.precision").asText()); - int scale = Integer.parseInt(fieldNode.get("parameters").get("scale").asText()); - return TypeDescription.createDecimal(precision, scale); - case "io.debezium.time.Date": - return TypeDescription.createDate(); - } - } - - switch (typeName) - { - case "int64": - return TypeDescription.createLong(); - case "int32": - return TypeDescription.createInt(); - case "string": - return TypeDescription.createString(); - case "struct": - return parseStruct(fieldNode.get("fields")); - default: - throw new IllegalArgumentException("Unsupported type: " + typeName); - } - } - - public static TypeDescription parseFromBeforeOrAfter(Schema schemaNode, String fieldName) - { - Schema.Field filed = schemaNode.getField(fieldName); - if (filed == null) - { - throw new IllegalArgumentException("Can't find field in avro schema: " + fieldName); - } - - Schema valueSchema = filed.schema(); - return parseFromAvroSchema(valueSchema); - } - - - public static TypeDescription parseFromAvroSchema(Schema avroSchema) - { - return parseAvroType(avroSchema, new HashMap<>()); - } - - private static TypeDescription parseAvroType(Schema schema, Map cache) - { - String schemaKey = schema.getFullName() + ":" + schema.hashCode(); - if (cache.containsKey(schemaKey)) - { - return cache.get(schemaKey); - } - - TypeDescription typeDesc; - switch (schema.getType()) - { - case RECORD: - typeDesc = parseAvroRecord(schema, cache); - break; - case UNION: - typeDesc = parseAvroUnion(schema, cache); - break; - case ARRAY: - typeDesc = parseAvroArray(schema, cache); - break; - case MAP: - typeDesc = parseAvroMap(schema, cache); - break; - default: - typeDesc = parseAvroPrimitive(schema); - } - - cache.put(schemaKey, typeDesc); - return typeDesc; - } - - private static TypeDescription parseAvroRecord(Schema schema, Map cache) - { - TypeDescription structType = TypeDescription.createStruct(); - for (Schema.Field field : schema.getFields()) - { - TypeDescription fieldType = parseAvroType(field.schema(), cache); - structType.addField(field.name(), fieldType); - } - return structType; - } - - private static TypeDescription parseAvroUnion(Schema schema, Map cache) - { - for (Schema type : schema.getTypes()) - { - if (type.getType() != Schema.Type.NULL) - { - return parseAvroType(type, cache); - } - } - throw new IllegalArgumentException("Invalid union type: " + schema); - } - - private static TypeDescription parseAvroArray(Schema schema, Map cache) - { - throw new RuntimeException("Doesn't support Array"); - } - - private static TypeDescription parseAvroMap(Schema schema, Map cache) - { - throw new RuntimeException("Doesn't support Map"); - } - - private static TypeDescription parseAvroPrimitive(Schema schema) - { - String logicalType = schema.getLogicalType() != null ? - schema.getLogicalType().getName() : null; - - if (logicalType != null) - { - switch (logicalType) - { - case "decimal": - return TypeDescription.createDecimal( - (Integer) (schema.getObjectProp("precision")), - (Integer) (schema.getObjectProp("scale")) - ); - case "date": - return TypeDescription.createDate(); - case "timestamp-millis": - return TypeDescription.createTimestamp((Integer) (schema.getObjectProp("precision"))); - case "uuid": - return TypeDescription.createString(); - } - } - - switch (schema.getType()) - { - case LONG: - return TypeDescription.createLong(); - case INT: - return TypeDescription.createInt(); - case STRING: - return TypeDescription.createString(); - case BOOLEAN: - return TypeDescription.createBoolean(); - case FLOAT: - return TypeDescription.createFloat(); - case DOUBLE: - return TypeDescription.createDouble(); - case BYTES: - // return TypeDescription.createBinary(); - default: - throw new IllegalArgumentException("Unsupported Avro type: " + schema); - } - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionAvroMessageDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionAvroMessageDeserializer.java deleted file mode 100644 index ffdf3f6..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionAvroMessageDeserializer.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - -import io.apicurio.registry.serde.SerdeConfig; -import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import io.pixelsdb.pixels.sink.util.MetricsFacade; -import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.common.errors.SerializationException; -import org.apache.kafka.common.serialization.Deserializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashMap; -import java.util.Map; - -public class TransactionAvroMessageDeserializer implements Deserializer -{ - private static final Logger logger = LoggerFactory.getLogger(TransactionAvroMessageDeserializer.class); - private final AvroKafkaDeserializer avroDeserializer = new AvroKafkaDeserializer<>(); - private final PixelsSinkConfig config = PixelsSinkConfigFactory.getInstance(); - - @Override - public void configure(Map configs, boolean isKey) - { - Map enrichedConfig = new HashMap<>(configs); - enrichedConfig.put(SerdeConfig.REGISTRY_URL, config.getRegistryUrl()); - enrichedConfig.put(SerdeConfig.CHECK_PERIOD_MS, SerdeConfig.CHECK_PERIOD_MS_DEFAULT); - avroDeserializer.configure(enrichedConfig, isKey); - } - - @Override - public SinkProto.TransactionMetadata deserialize(String topic, byte[] bytes) - { - if (bytes == null || bytes.length == 0) - { - return null; - } - try - { - MetricsFacade.getInstance().addRawData(bytes.length); - GenericRecord avroRecord = avroDeserializer.deserialize(topic, bytes); - return TransactionStructMessageDeserializer.convertToTransactionMetadata(avroRecord); - } catch (Exception e) - { - logger.error("Avro deserialization failed for topic {}: {}", topic, e.getMessage()); - throw new SerializationException("Failed to deserialize Avro message", e); - } - } - - @Override - public void close() - { - Deserializer.super.close(); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionJsonMessageDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionJsonMessageDeserializer.java deleted file mode 100644 index c09ba3e..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionJsonMessageDeserializer.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.protobuf.util.JsonFormat; -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.util.MetricsFacade; -import org.apache.kafka.common.serialization.Deserializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Map; - -public class TransactionJsonMessageDeserializer implements Deserializer -{ - - private static final Logger LOGGER = LoggerFactory.getLogger(TransactionJsonMessageDeserializer.class); - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static final JsonFormat.Parser PROTO_PARSER = JsonFormat.parser().ignoringUnknownFields(); - - @Override - public SinkProto.TransactionMetadata deserialize(String topic, byte[] data) - { - if (data == null || data.length == 0) - { - return null; - } - MetricsFacade.getInstance().addRawData(data.length); - try - { - Map rawMessage = OBJECT_MAPPER.readValue(data, Map.class); - return parseTransactionMetadata(rawMessage); - } catch (IOException e) - { - LOGGER.error("Failed to deserialize transaction message", e); - throw new RuntimeException("Deserialization error", e); - } - } - - private SinkProto.TransactionMetadata parseTransactionMetadata(Map rawMessage) throws IOException - { - SinkProto.TransactionMetadata.Builder builder = SinkProto.TransactionMetadata.newBuilder(); - String json = OBJECT_MAPPER.writeValueAsString(rawMessage.get("payload")); - PROTO_PARSER.merge(json, builder); - - builder.setId(DeserializerUtil.getTransIdPrefix(builder.getId())); - - return builder.build(); - } -} \ No newline at end of file diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionStructMessageDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionStructMessageDeserializer.java deleted file mode 100644 index dd521cb..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionStructMessageDeserializer.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - - -import io.pixelsdb.pixels.sink.SinkProto; -import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.connect.data.Struct; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @package: io.pixelsdb.pixels.sink.event.deserializer - * @className: TransactionStructMessageDeserializer - * @author: AntiO2 - * @date: 2025/9/26 12:42 - */ -public class TransactionStructMessageDeserializer -{ - private static final Logger LOGGER = LoggerFactory.getLogger(TransactionStructMessageDeserializer.class); - - @SuppressWarnings("unchecked") - public static SinkProto.TransactionMetadata convertToTransactionMetadata(T record) - { - SinkProto.TransactionMetadata.Builder builder = SinkProto.TransactionMetadata.newBuilder(); - - builder.setStatus(DeserializerUtil.getStatusSafely(record, "status")) - .setId(DeserializerUtil.getTransIdPrefix( - DeserializerUtil.getStringSafely(record, "id"))) - .setEventCount(DeserializerUtil.getLongSafely(record, "event_count")) - .setTimestamp(DeserializerUtil.getLongSafely(record, "ts_ms")); - - Object collections = DeserializerUtil.getFieldSafely(record, "data_collections"); - if (collections instanceof Iterable) - { - for (Object item : (Iterable) collections) - { - if (item instanceof GenericRecord collectionRecord) - { - SinkProto.DataCollection.Builder collectionBuilder = SinkProto.DataCollection.newBuilder(); - collectionBuilder.setDataCollection( - DeserializerUtil.getStringSafely(collectionRecord, "data_collection")); - collectionBuilder.setEventCount( - DeserializerUtil.getLongSafely(collectionRecord, "event_count")); - builder.addDataCollections(collectionBuilder); - } else if (item instanceof Struct collectionRecord) - { - SinkProto.DataCollection.Builder collectionBuilder = SinkProto.DataCollection.newBuilder(); - collectionBuilder.setDataCollection( - DeserializerUtil.getStringSafely(collectionRecord, "data_collection")); - collectionBuilder.setEventCount( - DeserializerUtil.getLongSafely(collectionRecord, "event_count")); - builder.addDataCollections(collectionBuilder); - } - } - } - - return builder.build(); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/exception/SinkException.java b/src/main/java/io/pixelsdb/pixels/sink/exception/SinkException.java index 5ad1c7a..ce208be 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/exception/SinkException.java +++ b/src/main/java/io/pixelsdb/pixels/sink/exception/SinkException.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.exception; public class SinkException extends Exception diff --git a/src/main/java/io/pixelsdb/pixels/sink/freshness/FreshnessClient.java b/src/main/java/io/pixelsdb/pixels/sink/freshness/FreshnessClient.java index da35c54..b94ad43 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/freshness/FreshnessClient.java +++ b/src/main/java/io/pixelsdb/pixels/sink/freshness/FreshnessClient.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.freshness; import io.pixelsdb.pixels.common.exception.TransException; @@ -36,8 +31,8 @@ import java.util.concurrent.*; /** - * FreshnessClient is responsible for monitoring data freshness by periodically - * querying the maximum timestamp from a set of dynamically configured tables via Trino JDBC. + * FreshnessClient monitors data freshness by periodically querying max(freshness_ts) + * via JDBC (Trino by default, or HiveServer2 for Hudi when built with {@code -Phudi-hive}). */ public class FreshnessClient { @@ -45,9 +40,10 @@ public class FreshnessClient private static final int QUERY_INTERVAL_SECONDS = 1; private static volatile FreshnessClient instance; // Configuration parameters (should ideally be loaded from a config file) - private final String trinoJdbcUrl; - private final String trinoUser; - private final String trinoPassword; + private final String queryJdbcUrl; + private final String queryUser; + private final String queryPassword; + private final QueryEngine queryEngine; private final int maxConcurrentQueries; private final Semaphore queryPermits; private final ThreadPoolExecutor connectionExecutor; @@ -58,17 +54,35 @@ public class FreshnessClient private final int warmUpSeconds; private final PixelsSinkConfig config; + private enum QueryEngine + { + TRINO("io.trino.jdbc.TrinoDriver", + "Trino JDBC driver not found on classpath."), + HIVE("org.apache.hive.jdbc.HiveDriver", + "Hive JDBC driver not found. Build with -Phudi-hive to enable Hudi freshness via HiveServer2."); + + private final String driverClass; + private final String missingHint; + + QueryEngine(String driverClass, String missingHint) + { + this.driverClass = driverClass; + this.missingHint = missingHint; + } + } + private FreshnessClient() { // Initializes the set with thread safety wrapper this.monitoredTables = Collections.synchronizedSet(new HashSet<>()); this.config = PixelsSinkConfigFactory.getInstance(); - this.trinoUser = config.getTrinoUser(); - this.trinoJdbcUrl = config.getTrinoUrl(); - this.trinoPassword = config.getTrinoPassword(); + this.queryUser = config.getSinkQueryUser(); + this.queryJdbcUrl = config.getSinkQueryUrl(); + this.queryPassword = config.getSinkQueryPassword(); + this.queryEngine = resolveQueryEngine(queryJdbcUrl); this.warmUpSeconds = config.getSinkMonitorFreshnessEmbedWarmupSeconds(); - this.maxConcurrentQueries = config.getTrinoParallel(); + this.maxConcurrentQueries = config.getSinkQueryParallel(); this.queryPermits = new Semaphore(maxConcurrentQueries); // Initializes a single-threaded scheduler for executing freshness queries this.scheduler = Executors.newSingleThreadScheduledExecutor(r -> @@ -113,42 +127,59 @@ public static FreshnessClient getInstance() return instance; } - @Deprecated - protected Connection createNewConnection() throws SQLException + /** + * Unified JDBC entry: resolve engine from URL at construction, load driver, + * apply engine-specific connection properties, then return a ready Connection. + * + * @param queryTimestamp Trino snapshot timestamp when embed.snapshot is enabled; ignored otherwise / for Hive + */ + protected Connection openConnection(Long queryTimestamp) throws SQLException { try { - Class.forName("io.trino.jdbc.TrinoDriver"); - } catch (ClassNotFoundException e) + Class.forName(queryEngine.driverClass); + } + catch (ClassNotFoundException e) { - throw new SQLException(e); + throw new SQLException(queryEngine.missingHint, e); } Properties properties = new Properties(); + properties.setProperty("user", queryUser); + if (queryEngine == QueryEngine.TRINO + && queryTimestamp != null + && config.isSinkMonitorFreshnessEmbedSnapshot()) + { + String sessionPropValue = String.format( + "pixels.query_snapshot_timestamp:%d", queryTimestamp); + properties.setProperty("sessionProperties", sessionPropValue); + } + return DriverManager.getConnection(queryJdbcUrl, properties); + } - - return DriverManager.getConnection(trinoJdbcUrl, trinoUser, null); + @Deprecated + protected Connection createNewConnection() throws SQLException + { + return openConnection(null); } protected Connection createNewConnection(long queryTimestamp) throws SQLException { - try - { - Class.forName("io.trino.jdbc.TrinoDriver"); - } catch (ClassNotFoundException e) + return openConnection(queryTimestamp); + } + + private static QueryEngine resolveQueryEngine(String jdbcUrl) + { + if (jdbcUrl != null && jdbcUrl.startsWith("jdbc:hive2:")) { - throw new SQLException(e); + return QueryEngine.HIVE; } - - Properties properties = new Properties(); - properties.setProperty("user", trinoUser); - if (config.isSinkMonitorFreshnessEmbedSnapshot()) + if (jdbcUrl != null && jdbcUrl.startsWith("jdbc:trino:")) { - String catalogName = "pixels"; - String sessionPropValue = String.format("%s.query_snapshot_timestamp:%d", catalogName, queryTimestamp); - properties.setProperty("sessionProperties", sessionPropValue); + return QueryEngine.TRINO; } - return DriverManager.getConnection(trinoJdbcUrl, properties); + throw new IllegalArgumentException( + "Unsupported freshness JDBC URL (expect jdbc:trino:// or jdbc:hive2://): " + jdbcUrl); } private void closeConnection(Connection conn) @@ -160,7 +191,7 @@ private void closeConnection(Connection conn) conn.close(); } catch (SQLException e) { - LOGGER.warn("Error closing Trino connection.", e); + LOGGER.warn("Error closing freshness JDBC connection.", e); } } } @@ -282,14 +313,13 @@ void queryAndCalculateFreshness() } // Timestamp when the query is sent (t_send) long tSendMillis = System.currentTimeMillis(); + Long snapshotTimestamp = null; if (config.isSinkMonitorFreshnessEmbedSnapshot()) { transContext = TransService.Instance().beginTrans(true); - conn = createNewConnection(transContext.getTimestamp()); - } else - { - conn = createNewConnection(); + snapshotTimestamp = transContext.getTimestamp(); } + conn = openConnection(snapshotTimestamp); String tSendMillisStr = DateUtil.convertDateToString(new Date(tSendMillis)); // Query to find the latest timestamp in the table diff --git a/src/main/java/io/pixelsdb/pixels/sink/freshness/FreshnessHistory.java b/src/main/java/io/pixelsdb/pixels/sink/freshness/FreshnessHistory.java index 1037f2e..2843b2c 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/freshness/FreshnessHistory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/freshness/FreshnessHistory.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.freshness; diff --git a/src/main/java/io/pixelsdb/pixels/sink/freshness/OneSecondAverage.java b/src/main/java/io/pixelsdb/pixels/sink/freshness/OneSecondAverage.java index cd4f3c2..b017536 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/freshness/OneSecondAverage.java +++ b/src/main/java/io/pixelsdb/pixels/sink/freshness/OneSecondAverage.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.freshness; import java.util.ArrayDeque; diff --git a/src/main/java/io/pixelsdb/pixels/sink/metadata/TableMetadata.java b/src/main/java/io/pixelsdb/pixels/sink/metadata/TableMetadata.java index 0935d7a..63e1d16 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/metadata/TableMetadata.java +++ b/src/main/java/io/pixelsdb/pixels/sink/metadata/TableMetadata.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.metadata; import io.pixelsdb.pixels.common.exception.MetadataException; diff --git a/src/main/java/io/pixelsdb/pixels/sink/metadata/TableMetadataRegistry.java b/src/main/java/io/pixelsdb/pixels/sink/metadata/TableMetadataRegistry.java index c776e71..9b6121e 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/metadata/TableMetadataRegistry.java +++ b/src/main/java/io/pixelsdb/pixels/sink/metadata/TableMetadataRegistry.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.metadata; import io.pixelsdb.pixels.common.exception.MetadataException; diff --git a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipeline.java b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipeline.java new file mode 100644 index 0000000..4580679 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipeline.java @@ -0,0 +1,74 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.pipeline; + +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.processor.TableProcessor; +import io.pixelsdb.pixels.sink.util.BlockingBoundedQueue; +import io.pixelsdb.pixels.sink.writer.PixelsSinkWriter; +import io.pixelsdb.pixels.sink.writer.PixelsSinkWriterFactory; + +public final class TablePipeline implements AutoCloseable +{ + private final BlockingBoundedQueue eventQueue; + private final PixelsSinkWriter writer; + private final TableProcessor processor; + + public TablePipeline() + { + this.eventQueue = new BlockingBoundedQueue<>(PixelsSinkConstants.MAX_QUEUE_SIZE); + this.writer = PixelsSinkWriterFactory.getWriter(); + this.processor = new TableProcessor(eventQueue, writer); + } + + public void start() + { + processor.run(); + } + + public void publish(RowChangeEvent event) + { + eventQueue.put(event); + } + + /** + * Stops accepting events and waits for all pending events to be written. + */ + @Override + public void close() + { + eventQueue.close(); + try + { + processor.awaitTermination(); + } catch (InterruptedException e) + { + abort(); + Thread.currentThread().interrupt(); + } + } + + /** + * Discards pending events and interrupts processing. Already written events + * are not rolled back. + */ + public void abort() + { + processor.abort(); + eventQueue.abort(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipelineManager.java b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipelineManager.java new file mode 100644 index 0000000..bec3ace --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipelineManager.java @@ -0,0 +1,91 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.pipeline; + +import io.pixelsdb.pixels.common.metadata.SchemaTableName; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public final class TablePipelineManager implements AutoCloseable +{ + private final Map pipelines = new ConcurrentHashMap<>(); + private final Object lifecycleLock = new Object(); + private volatile boolean closed; + + public void route(RowChangeEvent event) + { + if (event == null || closed) + { + return; + } + SchemaTableName table = new SchemaTableName(event.getSchemaName(), event.getTable()); + TablePipeline pipeline = pipelines.get(table); + if (pipeline == null) + { + synchronized (lifecycleLock) + { + if (closed) + { + return; + } + pipeline = pipelines.computeIfAbsent(table, ignored -> + { + TablePipeline newPipeline = new TablePipeline(); + newPipeline.start(); + return newPipeline; + }); + } + } + pipeline.publish(event); + } + + @Override + public void close() + { + List pipelinesToClose; + synchronized (lifecycleLock) + { + if (closed) + { + return; + } + closed = true; + pipelinesToClose = new ArrayList<>(pipelines.values()); + pipelines.clear(); + } + pipelinesToClose.forEach(TablePipeline::close); + } + + public void abort() + { + List pipelinesToAbort; + synchronized (lifecycleLock) + { + if (closed) + { + return; + } + closed = true; + pipelinesToAbort = new ArrayList<>(pipelines.values()); + pipelines.clear(); + } + pipelinesToAbort.forEach(TablePipeline::abort); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TransactionPipeline.java b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TransactionPipeline.java new file mode 100644 index 0000000..696c002 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TransactionPipeline.java @@ -0,0 +1,81 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.pipeline; + +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; +import io.pixelsdb.pixels.sink.processor.TransactionProcessor; +import io.pixelsdb.pixels.sink.util.BlockingBoundedQueue; +import io.pixelsdb.pixels.sink.writer.PixelsSinkWriter; +import io.pixelsdb.pixels.sink.writer.PixelsSinkWriterFactory; + +public final class TransactionPipeline implements AutoCloseable +{ + private final BlockingBoundedQueue eventQueue; + private final PixelsSinkWriter writer; + private final TransactionProcessor processor; + private final Thread processorThread; + + public TransactionPipeline() + { + this.eventQueue = new BlockingBoundedQueue<>(PixelsSinkConstants.MAX_QUEUE_SIZE); + this.writer = PixelsSinkWriterFactory.getWriter(); + this.processor = new TransactionProcessor(eventQueue, writer); + this.processorThread = new Thread(processor, "transaction-processor"); + } + + public void start() + { + if (!processorThread.isAlive()) + { + processorThread.start(); + } + } + + public void publish(SinkProto.TransactionMetadata transaction) + { + eventQueue.put(transaction); + } + + /** + * Stops accepting transactions and waits for all pending transactions to + * be written. + */ + @Override + public void close() + { + eventQueue.close(); + try + { + processorThread.join(); + } catch (InterruptedException e) + { + abort(); + Thread.currentThread().interrupt(); + } + } + + /** + * Discards pending transactions and interrupts processing. Already written + * transactions are not rolled back. + */ + public void abort() + { + processor.abort(); + eventQueue.abort(); + processorThread.interrupt(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/processor/MonitorThreadManager.java b/src/main/java/io/pixelsdb/pixels/sink/processor/MonitorThreadManager.java deleted file mode 100644 index a3686a4..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/processor/MonitorThreadManager.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.processor; - -import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; - -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -public class MonitorThreadManager -{ - private final List monitors = new CopyOnWriteArrayList<>(); - private final ExecutorService executor = Executors.newFixedThreadPool(PixelsSinkConstants.MONITOR_NUM); - - public void startMonitor(Runnable monitor) - { - monitors.add(monitor); - executor.submit(monitor); - } - - public void shutdown() - { - stopMonitors(); - shutdownExecutor(); - awaitTermination(); - } - - private void stopMonitors() - { - monitors.forEach(monitor -> - { - if (monitor instanceof StoppableProcessor) - { - ((StoppableProcessor) monitor).stopProcessor(); - } - }); - } - - private void shutdownExecutor() - { - executor.shutdown(); - } - - private void awaitTermination() - { - try - { - if (!executor.awaitTermination(10, TimeUnit.SECONDS)) - { - executor.shutdownNow(); - } - } catch (InterruptedException e) - { - executor.shutdownNow(); - } - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/processor/StoppableProcessor.java b/src/main/java/io/pixelsdb/pixels/sink/processor/StoppableProcessor.java deleted file mode 100644 index 9c42a8d..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/processor/StoppableProcessor.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - - -package io.pixelsdb.pixels.sink.processor; - -public interface StoppableProcessor -{ - void stopProcessor(); -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/processor/TableProcessor.java b/src/main/java/io/pixelsdb/pixels/sink/processor/TableProcessor.java index 134dab7..98698fc 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/processor/TableProcessor.java +++ b/src/main/java/io/pixelsdb/pixels/sink/processor/TableProcessor.java @@ -1,28 +1,23 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.processor; import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.provider.TableEventProvider; +import io.pixelsdb.pixels.sink.util.BlockingBoundedQueue; import io.pixelsdb.pixels.sink.util.MetricsFacade; import io.pixelsdb.pixels.sink.writer.PixelsSinkWriter; import io.pixelsdb.pixels.sink.writer.PixelsSinkWriterFactory; @@ -37,20 +32,27 @@ * @author: AntiO2 * @date: 2025/9/26 11:01 */ -public class TableProcessor implements StoppableProcessor, Runnable +public class TableProcessor implements Runnable { private static final Logger LOGGER = LoggerFactory.getLogger(TableProcessor.class); private final AtomicBoolean running = new AtomicBoolean(true); private final PixelsSinkWriter pixelsSinkWriter; - private final TableEventProvider tableEventProvider; + private final BlockingBoundedQueue eventQueue; private final MetricsFacade metricsFacade = MetricsFacade.getInstance(); private Thread processorThread; private boolean tableAdded = false; - public TableProcessor(TableEventProvider tableEventProvider) + public TableProcessor(BlockingBoundedQueue eventQueue) { - this.pixelsSinkWriter = PixelsSinkWriterFactory.getWriter(); - this.tableEventProvider = tableEventProvider; + this(eventQueue, PixelsSinkWriterFactory.getWriter()); + } + + public TableProcessor( + BlockingBoundedQueue eventQueue, + PixelsSinkWriter pixelsSinkWriter) + { + this.pixelsSinkWriter = pixelsSinkWriter; + this.eventQueue = eventQueue; } @Override @@ -64,21 +66,32 @@ private void processLoop() { while (running.get()) { - RowChangeEvent event = tableEventProvider.getRowChangeEvent(); + RowChangeEvent event = eventQueue.take(); if (event == null) { - continue; + break; } pixelsSinkWriter.writeRow(event); } + running.set(false); LOGGER.info("Processor thread exited"); } - @Override - public void stopProcessor() + public void awaitTermination() throws InterruptedException { - LOGGER.info("Stopping transaction monitor"); + if (processorThread != null) + { + processorThread.join(); + } + } + + public void abort() + { + LOGGER.info("Aborting table processor"); running.set(false); - processorThread.interrupt(); + if (processorThread != null) + { + processorThread.interrupt(); + } } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/processor/TopicProcessor.java b/src/main/java/io/pixelsdb/pixels/sink/processor/TopicProcessor.java deleted file mode 100644 index 905c87e..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/processor/TopicProcessor.java +++ /dev/null @@ -1,285 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.processor; - -import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; -import io.pixelsdb.pixels.sink.provider.TableEventKafkaProvider; -import org.apache.kafka.clients.admin.AdminClient; -import org.apache.kafka.clients.admin.AdminClientConfig; -import org.apache.kafka.clients.admin.ListTopicsResult; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.time.Duration; -import java.util.*; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - -public class TopicProcessor implements StoppableProcessor, Runnable -{ - - private static final Logger log = LoggerFactory.getLogger(TopicProcessor.class); - private final Properties kafkaProperties; - private final PixelsSinkConfig pixelsSinkConfig; - private final String[] includeTables; - private final Set subscribedTopics = ConcurrentHashMap.newKeySet(); - private final String bootstrapServers; - private final String baseTopic; - - private final AtomicBoolean running = new AtomicBoolean(true); - private final Map activeTasks = new ConcurrentHashMap<>(); // track row event consumer - private final ExecutorService executorService = Executors.newCachedThreadPool(); - private AdminClient adminClient; - private Timer timer; - - public TopicProcessor(PixelsSinkConfig pixelsSinkConfig, Properties kafkaProperties) - { - this.pixelsSinkConfig = pixelsSinkConfig; - this.kafkaProperties = kafkaProperties; - this.baseTopic = pixelsSinkConfig.getTopicPrefix() + "." + pixelsSinkConfig.getCaptureDatabase(); - this.includeTables = pixelsSinkConfig.getIncludeTables(); - this.bootstrapServers = pixelsSinkConfig.getBootstrapServers(); - } - - private static Set filterTopics(Set topics, String prefix) - { - return topics.stream() - .filter(t -> t.startsWith(prefix)) - .collect(Collectors.toSet()); - } - - @Override - public void run() - { - try - { - initializeResources(); - startMonitoringCycle(); - } finally - { - cleanupResources(); - log.info("Topic monitor stopped"); - } - } - - @Override - public void stopProcessor() - { - log.info("Initiating topic monitor shutdown..."); - running.set(false); - interruptMonitoring(); - shutdownConsumerTasks(); - awaitTermination(); - } - - private void shutdownConsumerTasks() - { - log.info("Shutting down {} active consumer tasks", activeTasks.size()); - activeTasks.forEach((topic, task) -> - { - log.info("Stopping consumer for topic: {}", topic); - task.close(); - }); - activeTasks.clear(); - } - - private void awaitTermination() - { - try - { - if (executorService != null && !executorService.awaitTermination(30, TimeUnit.SECONDS)) - { - log.warn("Forcing shutdown of remaining tasks"); - executorService.shutdownNow(); - } - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - } - } - - private void initializeResources() - { - Properties props = new Properties(); - props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, this.bootstrapServers); - this.adminClient = AdminClient.create(props); - this.timer = new Timer("TopicProcessor-Timer", true); - log.info("Started topic monitor for base topic: {}", baseTopic); - } - - private void startMonitoringCycle() - { - String topicPrefix = baseTopic + "."; - timer.scheduleAtFixedRate(new TopicMonitorTask(), 0, 5000); - - while (running.get()) - { - try - { - TimeUnit.SECONDS.sleep(1); - } catch (InterruptedException e) - { - if (running.get()) - { - log.warn("Monitoring thread interrupted unexpectedly", e); - } - Thread.currentThread().interrupt(); - } - } - } - - private void interruptMonitoring() - { - if (timer != null) - { - timer.cancel(); - timer.purge(); - } - if (adminClient != null) - { - adminClient.close(Duration.ofSeconds(5)); - } - shutdownExecutorService(); - } - - private void shutdownExecutorService() - { - executorService.shutdown(); - try - { - if (!executorService.awaitTermination(10, TimeUnit.SECONDS)) - { - executorService.shutdownNow(); - } - } catch (InterruptedException e) - { - executorService.shutdownNow(); - Thread.currentThread().interrupt(); - } - } - - private void cleanupResources() - { - try - { - if (adminClient != null) - { - adminClient.close(Duration.ofSeconds(5)); - } - } catch (Exception e) - { - log.warn("Error closing admin client", e); - } - } - - private Set detectNewTopics(Set currentTopics) - { - return currentTopics.stream() - .filter(t -> !subscribedTopics.contains(t)) - .collect(Collectors.toSet()); - } - - private String extractTableName(String topic) - { - int lastDotIndex = topic.lastIndexOf('.'); - return lastDotIndex != -1 ? topic.substring(lastDotIndex + 1) : topic; - } - - private void launchConsumerTask(String topic) - { - try - { - TableEventKafkaProvider task = new TableEventKafkaProvider(kafkaProperties, topic); - executorService.submit(task); - } catch (Exception e) - { - log.error("Failed to start consumer for topic {}: {}", topic, e.getMessage()); - } - } - - private class TopicMonitorTask extends TimerTask - { - @Override - public void run() - { - if (!running.get()) - { - cancel(); - return; - } - - try - { - processTopicChanges(); - } catch (Exception e) - { - e.printStackTrace(); - log.error("Error processing topic changes: {}", e.getMessage()); - } - } - - private void processTopicChanges() - { - try - { - ListTopicsResult listTopicsResult = adminClient.listTopics(); - Set currentTopics = listTopicsResult.names().get(5, TimeUnit.SECONDS); - Set filteredTopics = filterTopics(currentTopics, baseTopic + "."); - - Set newTopics = detectNewTopics(filteredTopics); - handleNewTopics(newTopics); - } catch (TimeoutException | ExecutionException | InterruptedException ignored) - { - - } - } - - private void handleNewTopics(Set newTopics) - { - newTopics.stream() - .filter(this::shouldProcessTable) - .forEach(topic -> - { - try - { - TableEventKafkaProvider task = new TableEventKafkaProvider(kafkaProperties, topic); - executorService.submit(task); - activeTasks.put(topic, task); - subscribedTopics.add(topic); - } catch (IOException e) - { - log.error("Failed to create consumer for {}: {}", topic, e.getMessage()); - } - }); - } - - private boolean shouldProcessTable(String topic) - { - String tableName = extractTableName(topic); - return includeTables.length == 0 || - Arrays.stream(includeTables).anyMatch(t -> t.equals(tableName)); - } - } - -} - diff --git a/src/main/java/io/pixelsdb/pixels/sink/processor/TransactionProcessor.java b/src/main/java/io/pixelsdb/pixels/sink/processor/TransactionProcessor.java index 1d28fb1..3a298ff 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/processor/TransactionProcessor.java +++ b/src/main/java/io/pixelsdb/pixels/sink/processor/TransactionProcessor.java @@ -1,27 +1,22 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.processor; import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.provider.TransactionEventProvider; +import io.pixelsdb.pixels.sink.util.BlockingBoundedQueue; import io.pixelsdb.pixels.sink.writer.PixelsSinkWriter; import io.pixelsdb.pixels.sink.writer.PixelsSinkWriterFactory; import org.slf4j.Logger; @@ -29,17 +24,25 @@ import java.util.concurrent.atomic.AtomicBoolean; -public class TransactionProcessor implements Runnable, StoppableProcessor +public class TransactionProcessor implements Runnable { private static final Logger LOGGER = LoggerFactory.getLogger(TransactionProcessor.class); private final PixelsSinkWriter sinkWriter; private final AtomicBoolean running = new AtomicBoolean(true); - private final TransactionEventProvider transactionEventProvider; + private final BlockingBoundedQueue eventQueue; + + public TransactionProcessor( + BlockingBoundedQueue eventQueue) + { + this(eventQueue, PixelsSinkWriterFactory.getWriter()); + } - public TransactionProcessor(TransactionEventProvider transactionEventProvider) + public TransactionProcessor( + BlockingBoundedQueue eventQueue, + PixelsSinkWriter sinkWriter) { - this.transactionEventProvider = transactionEventProvider; - this.sinkWriter = PixelsSinkWriterFactory.getWriter(); + this.eventQueue = eventQueue; + this.sinkWriter = sinkWriter; } @Override @@ -47,10 +50,9 @@ public void run() { while (running.get()) { - SinkProto.TransactionMetadata transaction = transactionEventProvider.getTransaction(); + SinkProto.TransactionMetadata transaction = eventQueue.take(); if (transaction == null) { - LOGGER.warn("Received null transaction"); running.set(false); break; } @@ -59,10 +61,9 @@ public void run() LOGGER.info("Processor thread exited for transaction"); } - @Override - public void stopProcessor() + public void abort() { - LOGGER.info("Stopping transaction monitor"); + LOGGER.info("Aborting transaction processor"); running.set(false); } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/EventProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/EventProvider.java deleted file mode 100644 index 8039405..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/EventProvider.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - -import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; -import io.pixelsdb.pixels.sink.util.MetricsFacade; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.*; - -public abstract class EventProvider implements Runnable, Closeable -{ - private static final Logger LOGGER = LoggerFactory.getLogger(EventProvider.class); - - private static final int BATCH_SIZE = 64; - private static final int THREAD_NUM = 4; - private static final long MAX_WAIT_MS = 5; // configurable - protected final MetricsFacade metricsFacade = MetricsFacade.getInstance(); - private final BlockingQueue rawEventQueue = new LinkedBlockingQueue<>(PixelsSinkConstants.MAX_QUEUE_SIZE); - private final BlockingQueue eventQueue = new LinkedBlockingQueue<>(PixelsSinkConstants.MAX_QUEUE_SIZE); - private final ExecutorService decodeExecutor = Executors.newFixedThreadPool(THREAD_NUM); - - private Thread providerThread; - - - @Override - public void run() - { - providerThread = new Thread(this::processLoop); - providerThread.start(); - } - - @Override - public void close() - { - this.providerThread.interrupt(); - decodeExecutor.shutdown(); - } - - protected void processLoop() - { - List sourceBatch = new ArrayList<>(BATCH_SIZE); - while (true) - { - try - { - sourceBatch.clear(); - // take first element (blocking) - SOURCE_RECORD_T first = getRawEvent(); - sourceBatch.add(first); - long startTime = System.nanoTime(); - - // keep polling until sourceBatch full or timeout - while (sourceBatch.size() < BATCH_SIZE) - { - long elapsedMs = (System.nanoTime() - startTime) / 1_000_000; - long remainingMs = MAX_WAIT_MS - elapsedMs; - if (remainingMs <= 0) - { - break; - } - - SOURCE_RECORD_T next = pollRawEvent(remainingMs); - if (next == null) - { - break; - } - sourceBatch.add(next); - } - - // parallel decode - List> futures = new ArrayList<>(sourceBatch.size()); - for (SOURCE_RECORD_T data : sourceBatch) - { - futures.add(decodeExecutor.submit(() -> - convertToTargetRecord(data))); - } - - // ordered put into queue - for (Future future : futures) - { - try - { - TARGET_RECORD_T event = future.get(); - if (event != null) - { - recordSerdEvent(); - putTargetEvent(event); - } - } catch (ExecutionException e) - { - LOGGER.warn("Decode failed: {}", String.valueOf(e.getCause())); - } - } - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - break; - } - } - } - - abstract TARGET_RECORD_T convertToTargetRecord(SOURCE_RECORD_T record); - - protected TARGET_RECORD_T getTargetEvent() - { - try - { - return eventQueue.take(); - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - } - return null; - } - - protected void putTargetEvent(TARGET_RECORD_T event) - { - try - { - eventQueue.put(event); - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - } - } - - protected void putRawEvent(SOURCE_RECORD_T record) - { - try - { - rawEventQueue.put(record); - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - } - } - - protected SOURCE_RECORD_T getRawEvent() - { - try - { - return rawEventQueue.take(); - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - return null; - } - } - - protected SOURCE_RECORD_T pollRawEvent(long remainingMs) - { - try - { - return rawEventQueue.poll(remainingMs, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) - { - return null; - } - } - - abstract protected void recordSerdEvent(); -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/ProtoType.java b/src/main/java/io/pixelsdb/pixels/sink/provider/ProtoType.java index ea7a4bf..5eaa4c1 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/ProtoType.java +++ b/src/main/java/io/pixelsdb/pixels/sink/provider/ProtoType.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.provider; diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventEngineProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventEngineProvider.java deleted file mode 100644 index 2a3f0da..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventEngineProvider.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - - -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventStructDeserializer; -import io.pixelsdb.pixels.sink.exception.SinkException; -import org.apache.kafka.connect.source.SourceRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -/** - * @package: io.pixelsdb.pixels.sink.provider - * @className: TableEventEngineProvider - * @author: AntiO2 - * @date: 2025/9/26 10:45 - */ -public class TableEventEngineProvider extends TableEventProvider -{ - private final Logger LOGGER = LoggerFactory.getLogger(TableEventEngineProvider.class.getName()); - - @Override - RowChangeEvent convertToTargetRecord(T record) - { - SourceRecord sourceRecord = (SourceRecord) record; - try - { - return RowChangeEventStructDeserializer.convertToRowChangeEvent(sourceRecord); - } catch (SinkException e) - { - LOGGER.warn("Failed to convert RowChangeEvent to RowChangeEventStruct {}", e.getMessage()); - return null; - } - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventKafkaProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventKafkaProvider.java deleted file mode 100644 index b582b2c..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventKafkaProvider.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - -import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.util.DataTransform; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.errors.WakeupException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.time.Duration; -import java.util.Collections; -import java.util.Properties; -import java.util.concurrent.atomic.AtomicBoolean; - -public class TableEventKafkaProvider extends TableEventProvider -{ - private static final Logger log = LoggerFactory.getLogger(TableEventKafkaProvider.class); - private final Properties kafkaProperties; - private final String topic; - private final AtomicBoolean running = new AtomicBoolean(true); - private final String tableName; - private KafkaConsumer consumer; - - public TableEventKafkaProvider(Properties kafkaProperties, String topic) throws IOException - { - PixelsSinkConfig config = PixelsSinkConfigFactory.getInstance(); - this.kafkaProperties = kafkaProperties; - this.topic = topic; - this.kafkaProperties.put(ConsumerConfig.GROUP_ID_CONFIG, config.getGroupId() + "-" + topic); - this.kafkaProperties.put(ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, "false"); - this.kafkaProperties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 500); - this.tableName = DataTransform.extractTableName(topic); - } - - @Override - protected void processLoop() - { - try - { - consumer = new KafkaConsumer<>(kafkaProperties); - consumer.subscribe(Collections.singleton(topic)); - - while (running.get()) - { - try - { - ConsumerRecords records = consumer.poll(Duration.ofSeconds(5)); - if (!records.isEmpty()) - { - log.info("{} Consumer poll returned {} records", tableName, records.count()); - records.forEach(record -> - { - if (record.value() == null) - { - return; - } - metricsFacade.recordSerdRowChange(); - putRowChangeEvent(record.value()); - }); - } - } catch (InterruptException ignored) - { - Thread.currentThread().interrupt(); - break; - } - } - } catch (WakeupException e) - { - log.info("Consumer wakeup triggered for {}", tableName); - } catch (Exception e) - { - log.info("Exception: {}", e.getMessage()); - } finally - { - if (consumer != null) - { - consumer.close(Duration.ofSeconds(5)); - log.info("Kafka consumer closed for {}", tableName); - } - } - } - - @Override - RowChangeEvent convertToTargetRecord(Void record) - { - throw new UnsupportedOperationException(); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventProvider.java deleted file mode 100644 index 1dc5111..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventProvider.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - - -import io.pixelsdb.pixels.sink.event.RowChangeEvent; - -/** - * @package: io.pixelsdb.pixels.sink.provider - * @className: TableEventProvider - * @author: AntiO2 - * @date: 2025/9/26 07:47 - */ -public abstract class TableEventProvider extends EventProvider -{ - - protected void putRowChangeEvent(RowChangeEvent rowChangeEvent) - { - putTargetEvent(rowChangeEvent); - } - - public RowChangeEvent getRowChangeEvent() - { - return getTargetEvent(); - } - - protected void putRawRowChangeEvent(SOURCE_RECORD_T record) - { - putRawEvent(record); - } - - final protected void recordSerdEvent() - { - metricsFacade.recordSerdRowChange(); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventStorageLoopProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventStorageLoopProvider.java deleted file mode 100644 index b628876..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventStorageLoopProvider.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - -import com.google.protobuf.InvalidProtocolBufferException; -import io.pixelsdb.pixels.core.utils.Pair; -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventStructDeserializer; -import io.pixelsdb.pixels.sink.exception.SinkException; -import io.pixelsdb.pixels.sink.util.DataTransform; - -import java.nio.ByteBuffer; -import java.util.logging.Logger; - -public class TableEventStorageLoopProvider extends TableEventProvider -{ - private final Logger LOGGER = Logger.getLogger(TableEventStorageProvider.class.getName()); - private final boolean freshness_embed; - private final boolean freshness_timestamp; - - protected TableEventStorageLoopProvider() - { - super(); - PixelsSinkConfig config = PixelsSinkConfigFactory.getInstance(); - String sinkMonitorFreshnessLevel = config.getSinkMonitorFreshnessLevel(); - if (sinkMonitorFreshnessLevel.equals("embed")) - { - freshness_embed = true; - } else - { - freshness_embed = false; - } - freshness_timestamp = config.isSinkMonitorFreshnessTimestamp(); - } - - @Override - RowChangeEvent convertToTargetRecord(T record) - { - Pair pairRecord = (Pair) record; - ByteBuffer sourceRecord = pairRecord.getLeft(); - sourceRecord.rewind(); - Integer loopId = pairRecord.getRight(); - try - { - SinkProto.RowRecord rowRecord = SinkProto.RowRecord.parseFrom(sourceRecord); - - SinkProto.RowRecord.Builder rowRecordBuilder = rowRecord.toBuilder(); - if (freshness_timestamp) - { - DataTransform.updateRecordTimestamp(rowRecordBuilder, System.currentTimeMillis() * 1000); - } - -// if(rowRecord.getSource().getTable().equals("transfer")) -// { -// DataTransform.transIdToBigint(rowRecordBuilder); -// } - - SinkProto.TransactionInfo.Builder transactionBuilder = rowRecordBuilder.getTransactionBuilder(); - String id = transactionBuilder.getId(); - transactionBuilder.setId(id + "_" + loopId); - rowRecordBuilder.setTransaction(transactionBuilder); - return RowChangeEventStructDeserializer.convertToRowChangeEvent(rowRecordBuilder.build()); - } catch (InvalidProtocolBufferException | SinkException e) - { - LOGGER.warning(e.getMessage()); - return null; - } - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventStorageProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventStorageProvider.java deleted file mode 100644 index 8ad49d3..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventStorageProvider.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - - -import com.google.protobuf.InvalidProtocolBufferException; -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventStructDeserializer; -import io.pixelsdb.pixels.sink.exception.SinkException; - -import java.nio.ByteBuffer; -import java.util.logging.Logger; - -/** - * @package: io.pixelsdb.pixels.sink.event - * @className: TableEventStorageProvider - * @author: AntiO2 - * @date: 2025/9/26 10:45 - */ -public class TableEventStorageProvider extends TableEventProvider -{ - private final Logger LOGGER = Logger.getLogger(TableEventStorageProvider.class.getName()); - - protected TableEventStorageProvider() - { - super(); - } - - @Override - RowChangeEvent convertToTargetRecord(T record) - { - ByteBuffer sourceRecord = (ByteBuffer) record; - try - { - SinkProto.RowRecord rowRecord = SinkProto.RowRecord.parseFrom(sourceRecord); - return RowChangeEventStructDeserializer.convertToRowChangeEvent(rowRecord); - } catch (InvalidProtocolBufferException | SinkException e) - { - LOGGER.warning(e.getMessage()); - return null; - } - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TableProviderAndProcessorPipelineManager.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TableProviderAndProcessorPipelineManager.java deleted file mode 100644 index 128f70b..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TableProviderAndProcessorPipelineManager.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - - -import io.pixelsdb.pixels.common.metadata.SchemaTableName; -import io.pixelsdb.pixels.core.utils.Pair; -import io.pixelsdb.pixels.sink.processor.TableProcessor; -import org.apache.kafka.connect.source.SourceRecord; - -import java.nio.ByteBuffer; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * @package: io.pixelsdb.pixels.sink.provider - * @className: TableProviderAndProcessorPipelineManager - * @author: AntiO2 - * @date: 2025/9/26 10:44 - */ -public class TableProviderAndProcessorPipelineManager -{ - protected final Map activeTableProcessors = new ConcurrentHashMap<>(); - protected final Map tableIds = new ConcurrentHashMap<>(); - private final Map> tableProviders = new ConcurrentHashMap<>(); - private final AtomicInteger nextTableId = new AtomicInteger(); - - - public void routeRecord(SchemaTableName schemaTableName, SOURCE_RECORD_T record) - { - routeRecord(getTableId(schemaTableName), record); - } - - public void routeRecord(Integer tableId, SOURCE_RECORD_T record) - { - TableEventProvider pipeline = tableProviders.computeIfAbsent(tableId, k -> - { - TableEventProvider newPipeline = createProvider(record); - TableProcessor tableProcessor = activeTableProcessors.computeIfAbsent(tableId, k2 -> - new TableProcessor(newPipeline) - ); - tableProcessor.run(); - newPipeline.run(); - return newPipeline; - }); - pipeline.putRawEvent(record); - } - - private TableEventProvider createProvider(SOURCE_RECORD_T record) - { - Class recordType = record.getClass(); - if (recordType == Pair.class) - { - return new TableEventStorageLoopProvider<>(); - } - if (recordType == SourceRecord.class) - { - return new TableEventEngineProvider<>(); - } else if (ByteBuffer.class.isAssignableFrom(recordType)) - { - return new TableEventStorageProvider<>(); - } else - { - throw new IllegalArgumentException("Unsupported record type: " + recordType.getName()); - } - } - - private Integer getTableId(SchemaTableName schemaTableName) - { - return tableIds.computeIfAbsent(schemaTableName, k -> allocateTableId()); - } - - private Integer allocateTableId() - { - return nextTableId.getAndIncrement(); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventEngineProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventEngineProvider.java deleted file mode 100644 index e39d853..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventEngineProvider.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - - -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.event.deserializer.TransactionStructMessageDeserializer; -import org.apache.kafka.connect.data.Struct; -import org.apache.kafka.connect.source.SourceRecord; - -/** - * @package: io.pixelsdb.pixels.sink.provider - * @className: TransactionEventEngineProvider - * @author: AntiO2 - * @date: 2025/9/25 13:20 - */ -public class TransactionEventEngineProvider extends TransactionEventProvider -{ - - public static final TransactionEventEngineProvider INSTANCE = new TransactionEventEngineProvider<>(); - - public static TransactionEventEngineProvider getInstance() - { - return INSTANCE; - } - - @Override - SinkProto.TransactionMetadata convertToTargetRecord(T record) - { - SourceRecord sourceRecord = (SourceRecord) record; - Struct value = (Struct) sourceRecord.value(); - return TransactionStructMessageDeserializer.convertToTransactionMetadata(value); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventKafkaProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventKafkaProvider.java deleted file mode 100644 index 081e0cf..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventKafkaProvider.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - - -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.errors.WakeupException; - -import java.time.Duration; -import java.util.Collections; -import java.util.Properties; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * @package: io.pixelsdb.pixels.sink.provider - * @className: TransactionEventKafkaProvider - * @author: AntiO2 - * @date: 2025/9/25 13:40 - */ -public class TransactionEventKafkaProvider extends TransactionEventProvider -{ - private final AtomicBoolean running = new AtomicBoolean(true); - private final String transactionTopic; - private final KafkaConsumer consumer; - - private TransactionEventKafkaProvider() - { - Properties kafkaProperties = new Properties(); - PixelsSinkConfig pixelsSinkConfig = PixelsSinkConfigFactory.getInstance(); - this.transactionTopic = pixelsSinkConfig.getTopicPrefix() + "." + pixelsSinkConfig.getTransactionTopicSuffix(); - this.consumer = new KafkaConsumer<>(kafkaProperties); - } - - - @Override - public void processLoop() - { - consumer.subscribe(Collections.singletonList(transactionTopic)); - while (running.get()) - { - try - { - - ConsumerRecords records = - consumer.poll(Duration.ofMillis(1000)); - - for (ConsumerRecord record : records) - { - if (record.value() == null) - { - continue; - } - putTargetEvent(record.value()); - } - } catch (WakeupException e) - { - if (running.get()) - { - // LOGGER.warn("Consumer wakeup unexpectedly", e); - } - } catch (Exception e) - { - e.printStackTrace(); - throw new RuntimeException(e); - } - } - } - - @Override - SinkProto.TransactionMetadata convertToTargetRecord(T record) - { - throw new UnsupportedOperationException(); - } - -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventProvider.java deleted file mode 100644 index 9239ba1..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventProvider.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - -import io.pixelsdb.pixels.sink.SinkProto; - -public abstract class TransactionEventProvider extends EventProvider -{ - public void putTransRawEvent(SOURCE_RECORD_T record) - { - putRawEvent(record); - } - - public SinkProto.TransactionMetadata getTransaction() - { - return getTargetEvent(); - } - - final protected void recordSerdEvent() - { - metricsFacade.recordSerdTxChange(); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventStorageLoopProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventStorageLoopProvider.java deleted file mode 100644 index 956de39..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventStorageLoopProvider.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - -import com.google.protobuf.InvalidProtocolBufferException; -import io.pixelsdb.pixels.core.utils.Pair; -import io.pixelsdb.pixels.sink.SinkProto; - -import java.nio.ByteBuffer; - -public class TransactionEventStorageLoopProvider extends TransactionEventProvider -{ - @Override - SinkProto.TransactionMetadata convertToTargetRecord(T record) - { - Pair buffer = (Pair) record; - try - { - SinkProto.TransactionMetadata tx = SinkProto.TransactionMetadata.parseFrom(buffer.getLeft()); - Integer loopId = buffer.getRight(); - SinkProto.TransactionMetadata.Builder builder = tx.toBuilder(); - builder.setId(builder.getId() + "_" + loopId); - return builder.build(); - } catch (InvalidProtocolBufferException e) - { - throw new RuntimeException(e); - } - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventStorageProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventStorageProvider.java deleted file mode 100644 index 59198c3..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventStorageProvider.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - -import com.google.protobuf.InvalidProtocolBufferException; -import io.pixelsdb.pixels.sink.SinkProto; - -import java.nio.ByteBuffer; - -public class TransactionEventStorageProvider extends TransactionEventProvider -{ - @Override - SinkProto.TransactionMetadata convertToTargetRecord(T record) - { - ByteBuffer buffer = (ByteBuffer) record; - try - { - SinkProto.TransactionMetadata tx = SinkProto.TransactionMetadata.parseFrom(buffer); - return tx; - } catch (InvalidProtocolBufferException e) - { - throw new RuntimeException(e); - } - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/SinkSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/SinkSource.java index bf2bb4b..50b1014 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/SinkSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/SinkSource.java @@ -1,38 +1,40 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - - package io.pixelsdb.pixels.sink.source; - -import io.pixelsdb.pixels.sink.processor.StoppableProcessor; - /** * @package: io.pixelsdb.pixels.sink.source * @className: SinkSource * @author: AntiO2 * @date: 2025/9/26 13:45 */ -public interface SinkSource extends StoppableProcessor +public interface SinkSource extends AutoCloseable { void start(); boolean isRunning(); + + /** + * Stops producing new events and waits for pending events to be processed. + */ + @Override + void close(); + + /** + * Stops immediately and discards events that have not been processed. + */ + void abort(); } diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/SinkSourceFactory.java b/src/main/java/io/pixelsdb/pixels/sink/source/SinkSourceFactory.java index c0463d0..2f7ca8a 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/SinkSourceFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/SinkSourceFactory.java @@ -1,31 +1,28 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - - package io.pixelsdb.pixels.sink.source; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; import io.pixelsdb.pixels.sink.source.engine.SinkEngineSource; import io.pixelsdb.pixels.sink.source.kafka.SinkKafkaSource; -import io.pixelsdb.pixels.sink.source.storage.FasterSinkStorageSource; +import io.pixelsdb.pixels.sink.source.storage.MemorySinkStorageSource; +import io.pixelsdb.pixels.sink.source.storage.StreamingSinkStorageSource; + +import java.util.Locale; public class SinkSourceFactory { @@ -36,8 +33,20 @@ public static SinkSource createSinkSource() { case "kafka" -> new SinkKafkaSource(); case "engine" -> new SinkEngineSource(); - case "storage" -> new FasterSinkStorageSource(); + case "storage" -> createStorageSource(config); default -> throw new IllegalStateException("Unsupported data source type: " + config.getDataSource()); }; } + + private static SinkSource createStorageSource(PixelsSinkConfig config) + { + String storageMode = config.getSinkStorageMode().trim().toLowerCase(Locale.ROOT); + return switch (storageMode) + { + case "stream" -> new StreamingSinkStorageSource(); + case "memory" -> new MemorySinkStorageSource(); + default -> throw new IllegalStateException( + "Unsupported storage source mode: " + config.getSinkStorageMode()); + }; + } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifier.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifier.java new file mode 100644 index 0000000..ad53b97 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifier.java @@ -0,0 +1,105 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.source.engine; + +import io.pixelsdb.pixels.common.metadata.SchemaTableName; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRecordUtil; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.source.SourceRecord; + +import java.util.Locale; + +/** + * Classifies Kafka Connect {@link SourceRecord} Debezium envelopes. + */ +public final class ConnectEventClassifier implements DebeziumEventClassifier +{ + @Override + public DebeziumRecordType classify( + SourceRecord sourceRecord, String transactionTopic) + { + if (sourceRecord == null || sourceRecord.value() == null) + { + return DebeziumRecordType.TOMBSTONE; + } + if (!(sourceRecord.value() instanceof Struct value)) + { + return DebeziumRecordType.UNKNOWN_CONTROL; + } + + if (transactionTopic != null && transactionTopic.equals(sourceRecord.topic())) + { + String status = DebeziumRecordUtil.getStringSafely(value, "status"); + String id = DebeziumRecordUtil.getStringSafely(value, "id"); + return (status.equals("BEGIN") || status.equals("END")) && !id.isBlank() + ? DebeziumRecordType.TRANSACTION + : DebeziumRecordType.UNKNOWN_CONTROL; + } + + return isRowChange(value) + ? DebeziumRecordType.ROW + : DebeziumRecordType.UNKNOWN_CONTROL; + } + + @Override + public SchemaTableName tableOf(SourceRecord record) + { + if (!(record.value() instanceof Struct value)) + { + throw new IllegalArgumentException("Debezium row value must be a Struct"); + } + Object sourceObject = DebeziumRecordUtil.getFieldSafely(value, "source"); + if (!(sourceObject instanceof Struct source)) + { + throw new IllegalArgumentException("Debezium row source must be a Struct"); + } + String database = DebeziumRecordUtil.getStringSafely(source, "db"); + String table = DebeziumRecordUtil.getStringSafely(source, "table"); + if (database.isBlank() || table.isBlank()) + { + throw new IllegalArgumentException( + "Debezium row source is missing db or table"); + } + return new SchemaTableName(database, table); + } + + private static boolean isRowChange(Struct value) + { + String op = DebeziumRecordUtil.getStringSafely(value, "op").toLowerCase(Locale.ROOT); + if (!(op.equals("c") || op.equals("u") || op.equals("d") || op.equals("r"))) + { + return false; + } + + Object sourceObject = DebeziumRecordUtil.getFieldSafely(value, "source"); + if (!(sourceObject instanceof Struct source) || + DebeziumRecordUtil.getStringSafely(source, "db").isBlank() || + DebeziumRecordUtil.getStringSafely(source, "table").isBlank()) + { + return false; + } + + Object before = DebeziumRecordUtil.getFieldSafely(value, "before"); + Object after = DebeziumRecordUtil.getFieldSafely(value, "after"); + return switch (op) + { + case "c", "r" -> after instanceof Struct; + case "u" -> before instanceof Struct && after instanceof Struct; + case "d" -> before instanceof Struct; + default -> false; + }; + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/TestUtils.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumEventClassifier.java similarity index 56% rename from src/test/java/io/pixelsdb/pixels/sink/TestUtils.java rename to src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumEventClassifier.java index 7713401..dab32a0 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/TestUtils.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumEventClassifier.java @@ -1,5 +1,5 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,23 +12,18 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ +package io.pixelsdb.pixels.sink.source.engine; -package io.pixelsdb.pixels.sink; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import io.pixelsdb.pixels.common.metadata.SchemaTableName; -public class TestUtils +/** + * Engine-side classification of Debezium transport records. + * Lives in {@code source.engine}, not conversion. + */ +public interface DebeziumEventClassifier { - public static ExecutorService synchronousExecutor() - { - return Executors.newSingleThreadExecutor(runnable -> - { - Thread thread = new Thread(runnable); - thread.setDaemon(true); - return thread; - }); - } + DebeziumRecordType classify(I input, String transactionTopic); + + SchemaTableName tableOf(I input); } diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumRecordType.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumRecordType.java new file mode 100644 index 0000000..d90cb8f --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumRecordType.java @@ -0,0 +1,27 @@ +/* + * Copyright 2023 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.source.engine; + +/** + * Classification of Debezium Engine {@code SourceRecord} envelopes. + */ +public enum DebeziumRecordType +{ + ROW, + TRANSACTION, + TOMBSTONE, + UNKNOWN_CONTROL +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumer.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumer.java index 96785a7..2c74e33 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumer.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumer.java @@ -1,40 +1,48 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.engine; import io.debezium.engine.DebeziumEngine; import io.debezium.engine.RecordChangeEvent; -import io.pixelsdb.pixels.common.metadata.SchemaTableName; +import io.pixelsdb.pixels.sink.SinkProto; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import io.pixelsdb.pixels.sink.processor.StoppableProcessor; -import io.pixelsdb.pixels.sink.processor.TransactionProcessor; -import io.pixelsdb.pixels.sink.provider.TableProviderAndProcessorPipelineManager; -import io.pixelsdb.pixels.sink.provider.TransactionEventEngineProvider; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumRowConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumTransactionConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.connect.DebeziumConnectRowConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.connect.DebeziumConnectTransactionConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRecordUtil; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import io.pixelsdb.pixels.sink.pipeline.TablePipelineManager; +import io.pixelsdb.pixels.sink.pipeline.TransactionPipeline; +import io.pixelsdb.pixels.sink.source.engine.adapter.DebeziumSourceAdapterSelector; import io.pixelsdb.pixels.sink.util.MetricsFacade; +import io.pixelsdb.pixels.sink.util.concurrent.StreamOrderedDecoder; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.source.SourceRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; /** * @package: io.pixelsdb.pixels.source @@ -42,88 +50,209 @@ * @author: AntiO2 * @date: 2025/9/25 12:51 */ -public class PixelsDebeziumConsumer implements DebeziumEngine.ChangeConsumer>, StoppableProcessor +public class PixelsDebeziumConsumer + implements DebeziumEngine.ChangeConsumer>, AutoCloseable { + private static final Logger LOGGER = LoggerFactory.getLogger(PixelsDebeziumConsumer.class); + private static final Object TRANSACTION_STREAM_KEY = new Object(); + private static final ConnectEventClassifier CLASSIFIER = new ConnectEventClassifier(); + private final String checkTransactionTopic; - private final TransactionEventEngineProvider transactionEventProvider = TransactionEventEngineProvider.INSTANCE; - private final TableProviderAndProcessorPipelineManager tableProvidersManagerImpl = new TableProviderAndProcessorPipelineManager<>(); - private final TransactionProcessor processor = new TransactionProcessor(transactionEventProvider); - private final Thread transactionProviderThread; - private final Thread transactionProcessorThread; + private final DebeziumSourceAdapter connectorAdapter; + private final DebeziumRowConverter rowConverter; + private final DebeziumTransactionConverter transactionConverter; + private final TransactionPipeline transactionPipeline; + private final TablePipelineManager tablePipelineManager; + private final StreamOrderedDecoder decodePipeline; private final MetricsFacade metricsFacade = MetricsFacade.getInstance(); - PixelsSinkConfig pixelsSinkConfig = PixelsSinkConfigFactory.getInstance(); + private final PixelsSinkConfig pixelsSinkConfig = PixelsSinkConfigFactory.getInstance(); + + private record PendingRecord( + RecordChangeEvent record, + CompletableFuture completion) + { + } public PixelsDebeziumConsumer() { this.checkTransactionTopic = pixelsSinkConfig.getDebeziumTopicPrefix() + ".transaction"; - this.transactionProviderThread = new Thread(this.transactionEventProvider, "transaction-adapter"); - this.transactionProcessorThread = new Thread(this.processor, "transaction-processor"); + this.connectorAdapter = DebeziumSourceAdapterSelector.configured(); + this.rowConverter = new DebeziumConnectRowConverter( + TableMetadataRegistry.Instance(), connectorAdapter); + this.transactionConverter = new DebeziumConnectTransactionConverter(connectorAdapter); + this.transactionPipeline = new TransactionPipeline(); + this.tablePipelineManager = new TablePipelineManager(); + this.decodePipeline = new StreamOrderedDecoder( + pixelsSinkConfig.getSourceDecodeThreads(), + "debezium-decoder"); + } - this.transactionProcessorThread.start(); - this.transactionProviderThread.start(); + public void start() + { + transactionPipeline.start(); + decodePipeline.start(); } public void handleBatch(List> event, DebeziumEngine.RecordCommitter> committer) throws InterruptedException { + List pendingRecords = new ArrayList<>(event.size()); for (RecordChangeEvent record : event) { - try + SourceRecord sourceRecord = record.record(); + if (sourceRecord == null) { - SourceRecord sourceRecord = record.record(); - if (sourceRecord == null) - { - continue; - } + pendingRecords.add(new PendingRecord( + record, CompletableFuture.completedFuture(null))); + continue; + } - metricsFacade.recordDebeziumEvent(); - if (isTransactionEvent(sourceRecord)) - { - handleTransactionSourceRecord(sourceRecord); - } else + metricsFacade.recordDebeziumEvent(); + DebeziumRecordType recordType = + CLASSIFIER.classify(sourceRecord, checkTransactionTopic); + logSourceRecord(sourceRecord, recordType); + CompletableFuture completion = switch (recordType) + { + case ROW -> submitRow(sourceRecord); + case TRANSACTION -> submitTransaction(sourceRecord); + case TOMBSTONE, UNKNOWN_CONTROL -> { - handleRowChangeSourceRecord(sourceRecord); + LOGGER.debug("Skipping Debezium {} event from topic {}", + recordType, sourceRecord.topic()); + yield CompletableFuture.completedFuture(null); } - } finally - { - committer.markProcessed(record); - } + }; + pendingRecords.add(new PendingRecord(record, completion)); + } + for (PendingRecord pending : pendingRecords) + { + awaitCompletion(pending.completion()); + committer.markProcessed(pending.record()); } committer.markBatchFinished(); } - private void handleTransactionSourceRecord(SourceRecord sourceRecord) throws InterruptedException + private CompletableFuture submitRow(SourceRecord record) + { + return decodePipeline.submit( + CLASSIFIER.tableOf(record), + record, + rowConverter::convert, + result -> publishRow(record, result)); + } + + private CompletableFuture submitTransaction(SourceRecord record) { - transactionEventProvider.putTransRawEvent(sourceRecord); + return decodePipeline.submit( + TRANSACTION_STREAM_KEY, + record, + transactionConverter::convert, + result -> publishTransaction(record, result)); } - private void handleRowChangeSourceRecord(SourceRecord sourceRecord) + private void publishRow( + SourceRecord record, + StreamOrderedDecoder.DecodeResult result) { - Struct value = (Struct) sourceRecord.value(); - if (value == null) + if (result.failure() != null) + { + LOGGER.warn("Skipping invalid Debezium ROW event from topic {}", + record.topic(), result.failure()); + return; + } + if (result.value() == null) { - return; // Delete Record, We will handle it in next record + return; } - Object sourceObject = value.get("source"); - Struct source = (Struct) sourceObject; - String schemaName = source.get("db").toString(); - String tableName = source.get("table").toString(); - SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName); - tableProvidersManagerImpl.routeRecord(schemaTableName, sourceRecord); + metricsFacade.recordSerdRowChange(); + tablePipelineManager.route(result.value()); } - private boolean isTransactionEvent(SourceRecord sourceRecord) + private void publishTransaction( + SourceRecord record, + StreamOrderedDecoder.DecodeResult result) { - return checkTransactionTopic.equals(sourceRecord.topic()); + if (result.failure() != null) + { + LOGGER.warn("Skipping invalid Debezium TRANSACTION event from topic {}", + record.topic(), result.failure()); + return; + } + if (result.value() == null) + { + return; + } + metricsFacade.recordSerdTxChange(); + transactionPipeline.publish(result.value()); + } + + private void awaitCompletion(CompletableFuture completion) + throws InterruptedException + { + try + { + completion.get(); + } catch (ExecutionException e) + { + throw new IllegalStateException( + "Debezium decode pipeline failed before publishing an event", + e.getCause()); + } + } + + private void logSourceRecord(SourceRecord record, DebeziumRecordType recordType) + { + if (!LOGGER.isDebugEnabled()) + { + return; + } + Struct value = record.value() instanceof Struct struct ? struct : null; + Struct source = value == null ? null : + asStruct(DebeziumRecordUtil.getFieldSafely(value, "source")); + Struct transaction = value == null ? null : + asStruct(DebeziumRecordUtil.getFieldSafely(value, "transaction")); + String rawTransactionId = recordType == DebeziumRecordType.TRANSACTION + ? DebeziumRecordUtil.getStringSafely(value, "id") + : DebeziumRecordUtil.getStringSafely(transaction, "id"); + String canonicalTransactionId = + connectorAdapter.normalizeTransactionId(rawTransactionId); + + LOGGER.debug("Debezium SourceRecord topic={}, sourcePartition={}, sourceOffset={}, " + + "keySchema={}, key={}, valueSchema={}, category={}, transaction.id={}, " + + "source.gtid={}, source.file={}, source.pos={}, source.row={}", + record.topic(), record.sourcePartition(), record.sourceOffset(), + schemaName(record.keySchema()), record.key(), schemaName(record.valueSchema()), + recordType, canonicalTransactionId, + DebeziumRecordUtil.getStringSafely(source, "gtid"), + DebeziumRecordUtil.getStringSafely(source, "file"), + DebeziumRecordUtil.getStringSafely(source, "pos"), + DebeziumRecordUtil.getStringSafely(source, "row")); + } + + private static Struct asStruct(Object value) + { + return value instanceof Struct struct ? struct : null; + } + + private static String schemaName(org.apache.kafka.connect.data.Schema schema) + { + return schema == null ? "" : String.valueOf(schema.name()); } @Override - public void stopProcessor() + public void close() + { + decodePipeline.close(); + tablePipelineManager.close(); + transactionPipeline.close(); + } + + public void abort() { - transactionProviderThread.interrupt(); - processor.stopProcessor(); - transactionProcessorThread.interrupt(); + decodePipeline.abort(); + tablePipelineManager.abort(); + transactionPipeline.abort(); } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/SinkEngineSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/SinkEngineSource.java index 47d2735..08f93f1 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/engine/SinkEngineSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/SinkEngineSource.java @@ -1,29 +1,25 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.engine; import io.debezium.embedded.Connect; import io.debezium.engine.DebeziumEngine; import io.debezium.engine.RecordChangeEvent; import io.debezium.engine.format.ChangeEventFormat; +import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; import io.pixelsdb.pixels.sink.source.SinkSource; import org.apache.kafka.connect.source.SourceRecord; @@ -31,23 +27,29 @@ import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; public class SinkEngineSource implements SinkSource { + private static final long SHUTDOWN_TIMEOUT_SECONDS = 10; + + private final PixelsSinkConfig pixelsSinkConfig; private final PixelsDebeziumConsumer consumer; private DebeziumEngine> engine; private ExecutorService executor; - private volatile boolean running = true; + private volatile boolean running; public SinkEngineSource() { + this.pixelsSinkConfig = PixelsSinkConfigFactory.getInstance(); this.consumer = new PixelsDebeziumConsumer(); } public void start() { - Properties debeziumProps = PixelsSinkConfigFactory.getInstance() - .getConfig().extractPropertiesByPrefix("debezium.", true); + consumer.start(); + Properties debeziumProps = pixelsSinkConfig.getConfig() + .extractPropertiesByPrefix("debezium.", true); this.engine = DebeziumEngine.create(ChangeEventFormat.of(Connect.class)) .using(debeziumProps) @@ -56,10 +58,11 @@ public void start() this.executor = Executors.newSingleThreadExecutor(); this.executor.execute(engine); + running = true; } @Override - public void stopProcessor() + public void close() { try { @@ -70,10 +73,30 @@ public void stopProcessor() if (executor != null) { executor.shutdown(); + if (!executor.awaitTermination( + SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + { + executor.shutdownNow(); + consumer.abort(); + return; + } + } + consumer.close(); + } catch (InterruptedException e) + { + if (executor != null) + { + executor.shutdownNow(); } - consumer.stopProcessor(); + consumer.abort(); + Thread.currentThread().interrupt(); } catch (Exception e) { + if (executor != null) + { + executor.shutdownNow(); + } + consumer.abort(); throw new RuntimeException("Failed to stop PixelsSinkEngine", e); } finally { @@ -81,6 +104,29 @@ public void stopProcessor() } } + @Override + public void abort() + { + running = false; + if (executor != null) + { + executor.shutdownNow(); + } + try + { + if (engine != null) + { + engine.close(); + } + } catch (Exception e) + { + throw new RuntimeException("Failed to abort PixelsSinkEngine", e); + } finally + { + consumer.abort(); + } + } + @Override public boolean isRunning() { diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumSourceAdapterSelector.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumSourceAdapterSelector.java new file mode 100644 index 0000000..2ac349f --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumSourceAdapterSelector.java @@ -0,0 +1,46 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.source.engine.adapter; + +import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapterRegistry; + +/** + * Wiring-side selector for sink CDC dialect ({@code sink.debezium.dialect}, + * with fallback to {@code debezium.connector.class}). + */ +public final class DebeziumSourceAdapterSelector +{ + private DebeziumSourceAdapterSelector() + { + } + + public static DebeziumSourceAdapter configured() + { + return DebeziumSourceAdapterRegistry.resolve( + PixelsSinkConfigFactory.getInstance().resolveDebeziumSourceDialect()); + } + + public static DebeziumSourceAdapter configuredIfPresent() + { + String dialect = + PixelsSinkConfigFactory.getInstance().resolveDebeziumSourceDialect(); + return dialect == null || dialect.isBlank() + ? null + : DebeziumSourceAdapterRegistry.resolve(dialect); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaRowSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaRowSource.java new file mode 100644 index 0000000..49fc886 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaRowSource.java @@ -0,0 +1,135 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.source.kafka; + +import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; +import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.pipeline.TablePipelineManager; +import io.pixelsdb.pixels.sink.source.kafka.serde.KafkaRecordConverter; +import io.pixelsdb.pixels.sink.util.DataTransform; +import io.pixelsdb.pixels.sink.util.MetricsFacade; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.errors.WakeupException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.Collections; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicBoolean; + +public final class KafkaRowSource implements Runnable +{ + private static final Logger LOGGER = LoggerFactory.getLogger(KafkaRowSource.class); + + private final Properties kafkaProperties; + private final String topic; + private final String tableName; + private final TablePipelineManager tablePipelineManager; + private final KafkaRecordConverter converter; + private final MetricsFacade metricsFacade = MetricsFacade.getInstance(); + private final AtomicBoolean running = new AtomicBoolean(true); + private KafkaConsumer consumer; + + public KafkaRowSource( + Properties kafkaProperties, + String topic, + TablePipelineManager tablePipelineManager) + { + PixelsSinkConfig config = PixelsSinkConfigFactory.getInstance(); + this.kafkaProperties = new Properties(); + this.kafkaProperties.putAll(kafkaProperties); + this.kafkaProperties.put( + ConsumerConfig.GROUP_ID_CONFIG, config.getGroupId() + "-" + topic); + this.kafkaProperties.put(ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, "false"); + this.kafkaProperties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 500); + this.topic = topic; + this.tableName = DataTransform.extractTableName(topic); + this.tablePipelineManager = tablePipelineManager; + this.converter = KafkaRecordConverter.forRow(this.kafkaProperties); + } + + @Override + public void run() + { + try + { + consumer = new KafkaConsumer<>(kafkaProperties); + consumer.subscribe(Collections.singleton(topic)); + while (running.get()) + { + try + { + ConsumerRecords records = + consumer.poll(Duration.ofSeconds(5)); + records.forEach(record -> + { + byte[] value = record.value(); + if (value == null) + { + return; + } + metricsFacade.addRawData(value.length); + try + { + RowChangeEvent event = converter.convert(topic, value); + if (event != null) + { + metricsFacade.recordSerdRowChange(); + tablePipelineManager.route(event); + } + } catch (RuntimeException e) + { + LOGGER.warn( + "Failed to convert Kafka record from topic {}: {}", + topic, e.getMessage()); + } + }); + } catch (InterruptException e) + { + Thread.currentThread().interrupt(); + break; + } + } + } catch (WakeupException e) + { + LOGGER.debug("Consumer wakeup triggered for {}", tableName); + } catch (Exception e) + { + LOGGER.error("Kafka source failed for {}", tableName, e); + } finally + { + converter.close(); + if (consumer != null) + { + consumer.close(Duration.ofSeconds(5)); + } + } + } + + void requestStop() + { + running.set(false); + if (consumer != null) + { + consumer.wakeup(); + } + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaTransactionSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaTransactionSource.java new file mode 100644 index 0000000..5624d12 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaTransactionSource.java @@ -0,0 +1,112 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.source.kafka; + +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.pipeline.TransactionPipeline; +import io.pixelsdb.pixels.sink.source.kafka.serde.KafkaRecordConverter; +import io.pixelsdb.pixels.sink.util.MetricsFacade; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.errors.WakeupException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.Collections; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicBoolean; + +public final class KafkaTransactionSource implements Runnable +{ + private static final Logger LOGGER = LoggerFactory.getLogger(KafkaTransactionSource.class); + + private final String transactionTopic; + private final KafkaConsumer consumer; + private final KafkaRecordConverter converter; + private final TransactionPipeline transactionPipeline; + private final MetricsFacade metricsFacade = MetricsFacade.getInstance(); + private final AtomicBoolean running = new AtomicBoolean(true); + + public KafkaTransactionSource( + Properties kafkaProperties, + String transactionTopic, + TransactionPipeline transactionPipeline) + { + this.transactionTopic = transactionTopic; + Properties consumerProperties = new Properties(); + consumerProperties.putAll(kafkaProperties); + this.converter = KafkaRecordConverter.forTransaction(consumerProperties); + this.consumer = new KafkaConsumer<>(consumerProperties); + this.transactionPipeline = transactionPipeline; + } + + @Override + public void run() + { + try + { + consumer.subscribe(Collections.singletonList(transactionTopic)); + while (running.get()) + { + try + { + ConsumerRecords records = + consumer.poll(Duration.ofMillis(1000)); + for (ConsumerRecord record : records) + { + byte[] value = record.value(); + if (value == null) + { + continue; + } + metricsFacade.addRawData(value.length); + SinkProto.TransactionMetadata transaction = + converter.convert(transactionTopic, value); + if (transaction != null) + { + metricsFacade.recordSerdTxChange(); + transactionPipeline.publish(transaction); + } + } + } catch (WakeupException e) + { + if (running.get()) + { + throw e; + } + } + } + } catch (Exception e) + { + if (running.get()) + { + LOGGER.error("Kafka transaction source failed for {}", transactionTopic, e); + } + } finally + { + converter.close(); + consumer.close(Duration.ofSeconds(5)); + } + } + + void requestStop() + { + running.set(false); + consumer.wakeup(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/SinkKafkaSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/SinkKafkaSource.java index 423e1d1..3e661b5 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/SinkKafkaSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/SinkKafkaSource.java @@ -1,40 +1,43 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.kafka; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; import io.pixelsdb.pixels.sink.config.factory.KafkaPropFactorySelector; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import io.pixelsdb.pixels.sink.processor.MonitorThreadManager; -import io.pixelsdb.pixels.sink.processor.TopicProcessor; -import io.pixelsdb.pixels.sink.processor.TransactionProcessor; +import io.pixelsdb.pixels.sink.pipeline.TablePipelineManager; +import io.pixelsdb.pixels.sink.pipeline.TransactionPipeline; import io.pixelsdb.pixels.sink.source.SinkSource; import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; public class SinkKafkaSource implements SinkSource { - private MonitorThreadManager manager; - private volatile boolean running = true; + private static final long SHUTDOWN_TIMEOUT_SECONDS = 10; + + private ExecutorService sourceExecutor; + private KafkaTransactionSource transactionSource; + private TopicProcessor topicProcessor; + private TablePipelineManager tablePipelineManager; + private TransactionPipeline transactionPipeline; + private volatile boolean running; @Override public void start() @@ -45,24 +48,107 @@ public void start() Properties transactionKafkaProperties = kafkaPropFactorySelector .getFactory(PixelsSinkConstants.TRANSACTION_KAFKA_PROP_FACTORY) .createKafkaProperties(pixelsSinkConfig); - TransactionProcessor transactionProcessor = null; // TODO: new TransactionProcessor(); + String transactionTopic = pixelsSinkConfig.getTopicPrefix() + "." + + pixelsSinkConfig.getTransactionTopicSuffix(); + tablePipelineManager = new TablePipelineManager(); + transactionPipeline = new TransactionPipeline(); + transactionSource = + new KafkaTransactionSource( + transactionKafkaProperties, transactionTopic, transactionPipeline); Properties topicKafkaProperties = kafkaPropFactorySelector .getFactory(PixelsSinkConstants.ROW_RECORD_KAFKA_PROP_FACTORY) .createKafkaProperties(pixelsSinkConfig); - TopicProcessor topicMonitor = new TopicProcessor(pixelsSinkConfig, topicKafkaProperties); + topicProcessor = new TopicProcessor( + pixelsSinkConfig, topicKafkaProperties, tablePipelineManager); - manager = new MonitorThreadManager(); - manager.startMonitor(transactionProcessor); - manager.startMonitor(topicMonitor); + transactionPipeline.start(); + sourceExecutor = Executors.newFixedThreadPool(2); + sourceExecutor.submit(transactionSource); + sourceExecutor.submit(topicProcessor); + running = true; } @Override - public void stopProcessor() + public void close() + { + if (transactionSource != null) + { + transactionSource.requestStop(); + } + if (topicProcessor != null) + { + topicProcessor.requestStop(); + } + + boolean sourcesStopped = true; + if (sourceExecutor != null) + { + sourceExecutor.shutdown(); + try + { + if (!sourceExecutor.awaitTermination( + SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + { + sourceExecutor.shutdownNow(); + sourcesStopped = false; + } + } catch (InterruptedException e) + { + sourceExecutor.shutdownNow(); + Thread.currentThread().interrupt(); + sourcesStopped = false; + } + } + + if (transactionPipeline != null) + { + if (sourcesStopped) + { + transactionPipeline.close(); + } else + { + transactionPipeline.abort(); + } + } + if (tablePipelineManager != null) + { + if (sourcesStopped) + { + tablePipelineManager.close(); + } else + { + tablePipelineManager.abort(); + } + } + running = false; + } + + @Override + public void abort() { - manager.shutdown(); running = false; + if (transactionSource != null) + { + transactionSource.requestStop(); + } + if (topicProcessor != null) + { + topicProcessor.abort(); + } + if (sourceExecutor != null) + { + sourceExecutor.shutdownNow(); + } + if (transactionPipeline != null) + { + transactionPipeline.abort(); + } + if (tablePipelineManager != null) + { + tablePipelineManager.abort(); + } } @Override diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/TopicProcessor.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/TopicProcessor.java new file mode 100644 index 0000000..3929fee --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/TopicProcessor.java @@ -0,0 +1,207 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.source.kafka; + +import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; +import io.pixelsdb.pixels.sink.pipeline.TablePipelineManager; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +public final class TopicProcessor implements Runnable +{ + private static final Logger LOGGER = LoggerFactory.getLogger(TopicProcessor.class); + + private final Properties kafkaProperties; + private final PixelsSinkConfig config; + private final String[] includeTables; + private final String bootstrapServers; + private final String baseTopic; + private final TablePipelineManager tablePipelineManager; + private final Set subscribedTopics = ConcurrentHashMap.newKeySet(); + private final Map activeSources = new ConcurrentHashMap<>(); + private final ExecutorService executor = Executors.newCachedThreadPool(); + private final AtomicBoolean running = new AtomicBoolean(true); + private final Object sourceLifecycleLock = new Object(); + private AdminClient adminClient; + private Timer timer; + + public TopicProcessor( + PixelsSinkConfig config, + Properties kafkaProperties, + TablePipelineManager tablePipelineManager) + { + this.config = config; + this.kafkaProperties = kafkaProperties; + this.includeTables = config.getIncludeTables(); + this.bootstrapServers = config.getBootstrapServers(); + this.baseTopic = config.getTopicPrefix() + "." + config.getCaptureDatabase(); + this.tablePipelineManager = tablePipelineManager; + } + + @Override + public void run() + { + try + { + Properties adminProperties = new Properties(); + adminProperties.put( + AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + adminClient = AdminClient.create(adminProperties); + timer = new Timer("TopicProcessor-Timer", true); + timer.scheduleAtFixedRate(new TopicMonitorTask(), 0, 5000); + while (running.get()) + { + try + { + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) + { + if (running.get()) + { + LOGGER.warn("Kafka topic monitor interrupted", e); + } + Thread.currentThread().interrupt(); + break; + } + } + } finally + { + requestStop(); + } + } + + void requestStop() + { + if (!running.compareAndSet(true, false)) + { + return; + } + if (timer != null) + { + timer.cancel(); + } + if (adminClient != null) + { + adminClient.close(Duration.ofSeconds(5)); + } + synchronized (sourceLifecycleLock) + { + activeSources.values().forEach(KafkaRowSource::requestStop); + activeSources.clear(); + executor.shutdown(); + } + try + { + if (!executor.awaitTermination(10, TimeUnit.SECONDS)) + { + executor.shutdownNow(); + } + } catch (InterruptedException e) + { + executor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + + void abort() + { + running.set(false); + if (timer != null) + { + timer.cancel(); + } + if (adminClient != null) + { + adminClient.close(Duration.ofSeconds(5)); + } + synchronized (sourceLifecycleLock) + { + activeSources.values().forEach(KafkaRowSource::requestStop); + activeSources.clear(); + executor.shutdownNow(); + } + } + + private class TopicMonitorTask extends TimerTask + { + @Override + public void run() + { + if (!running.get()) + { + cancel(); + return; + } + try + { + Set topics = adminClient.listTopics().names().get(5, TimeUnit.SECONDS) + .stream() + .filter(topic -> topic.startsWith(baseTopic + ".")) + .collect(Collectors.toSet()); + topics.stream() + .filter(topic -> !subscribedTopics.contains(topic)) + .filter(TopicProcessor.this::shouldProcessTable) + .forEach(this::startTopic); + } catch (Exception e) + { + if (running.get()) + { + LOGGER.warn("Failed to inspect Kafka topics", e); + } + } + } + + private void startTopic(String topic) + { + synchronized (sourceLifecycleLock) + { + if (!running.get() || executor.isShutdown()) + { + return; + } + KafkaRowSource source = new KafkaRowSource( + kafkaProperties, topic, tablePipelineManager); + activeSources.put(topic, source); + subscribedTopics.add(topic); + executor.submit(source); + } + } + } + + private boolean shouldProcessTable(String topic) + { + String tableName = topic.substring(topic.lastIndexOf('.') + 1); + return includeTables.length == 0 || + Arrays.stream(includeTables).anyMatch(tableName::equals); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverter.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverter.java new file mode 100644 index 0000000..44f3d86 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverter.java @@ -0,0 +1,206 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.source.kafka.serde; + +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.config.KafkaValueFormat; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; +import io.pixelsdb.pixels.sink.conversion.debezium.avro.DebeziumAvroPayloadDecoder; +import io.pixelsdb.pixels.sink.conversion.debezium.avro.DebeziumAvroRowConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.avro.DebeziumAvroTransactionConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapterRegistry; +import io.pixelsdb.pixels.sink.conversion.debezium.json.DebeziumJsonRowConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.json.DebeziumJsonTransactionConverter; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +/** + * Assembles Debezium row/tx converters for Kafka by {@code sink.kafka.value.format}. + */ +public final class KafkaRecordConverter implements AutoCloseable +{ + private static final Logger LOGGER = LoggerFactory.getLogger(KafkaRecordConverter.class); + + private enum ErrorPolicy + { + SWALLOW, + PROPAGATE + } + + @FunctionalInterface + private interface ConvertFn + { + T convert(String topic, byte[] data) throws Exception; + } + + private final ConvertFn convertFn; + private final AutoCloseable resource; + private final ErrorPolicy errorPolicy; + + private KafkaRecordConverter( + ConvertFn convertFn, AutoCloseable resource, ErrorPolicy errorPolicy) + { + this.convertFn = convertFn; + this.resource = resource; + this.errorPolicy = errorPolicy; + } + + public static KafkaRecordConverter forRow(Properties properties) + { + String format = resolveFormat(properties); + DebeziumSourceAdapter adapter = resolveAdapter(properties); + return switch (format) + { + case KafkaValueFormat.JSON -> + { + DebeziumJsonRowConverter converter = new DebeziumJsonRowConverter( + TableMetadataRegistry.Instance(), adapter); + yield new KafkaRecordConverter<>( + (topic, data) -> converter.convert(data), null, ErrorPolicy.SWALLOW); + } + case KafkaValueFormat.AVRO -> + { + DebeziumAvroPayloadDecoder decoder = new DebeziumAvroPayloadDecoder(); + decoder.configure(toConfigMap(properties), false); + DebeziumAvroRowConverter converter = new DebeziumAvroRowConverter( + TableMetadataRegistry.Instance(), adapter); + yield new KafkaRecordConverter<>( + (topic, data) -> converter.convert(decoder.decode(topic, data)), + decoder, + ErrorPolicy.SWALLOW); + } + default -> throw new IllegalArgumentException( + "Unsupported Kafka value format: " + format); + }; + } + + public static KafkaRecordConverter forTransaction( + Properties properties) + { + String format = resolveFormat(properties); + // Transaction envelopes lack source.connector; dialect must be configured up front. + DebeziumSourceAdapter adapter = requireAdapter(properties); + return switch (format) + { + case KafkaValueFormat.JSON -> + { + DebeziumJsonTransactionConverter converter = + new DebeziumJsonTransactionConverter(adapter); + yield new KafkaRecordConverter<>( + (topic, data) -> converter.convert(data), null, ErrorPolicy.PROPAGATE); + } + case KafkaValueFormat.AVRO -> + { + DebeziumAvroPayloadDecoder decoder = new DebeziumAvroPayloadDecoder(); + decoder.configure(toConfigMap(properties), false); + DebeziumAvroTransactionConverter converter = + new DebeziumAvroTransactionConverter(adapter); + yield new KafkaRecordConverter<>( + (topic, data) -> converter.convert(decoder.decode(topic, data)), + decoder, + ErrorPolicy.PROPAGATE); + } + default -> throw new IllegalArgumentException( + "Unsupported Kafka value format: " + format); + }; + } + + public T convert(String topic, byte[] data) + { + if (data == null || data.length == 0) + { + return null; + } + try + { + return convertFn.convert(topic, data); + } catch (RuntimeException e) + { + if (errorPolicy == ErrorPolicy.SWALLOW) + { + LOGGER.warn("Failed to convert Kafka record from topic {}", topic, e); + return null; + } + throw e; + } catch (Exception e) + { + if (errorPolicy == ErrorPolicy.SWALLOW) + { + LOGGER.warn("Failed to convert Kafka record from topic {}", topic, e); + return null; + } + throw new RuntimeException("Failed to convert Kafka record from " + topic, e); + } + } + + @Override + public void close() + { + if (resource != null) + { + try + { + resource.close(); + } catch (Exception e) + { + LOGGER.warn("Failed to close Kafka record converter resource", e); + } + } + } + + private static String resolveFormat(Properties properties) + { + Object configured = properties.get(PixelsSinkConstants.KAFKA_VALUE_FORMAT); + return KafkaValueFormat.resolve(configured == null ? null : configured.toString()); + } + + private static DebeziumSourceAdapter resolveAdapter(Properties properties) + { + Object dialect = properties.get(PixelsSinkConstants.SINK_DEBEZIUM_DIALECT); + if (dialect == null || dialect.toString().isBlank()) + { + return null; + } + return DebeziumSourceAdapterRegistry.resolve(dialect.toString()); + } + + private static DebeziumSourceAdapter requireAdapter(Properties properties) + { + DebeziumSourceAdapter adapter = resolveAdapter(properties); + if (adapter == null) + { + throw new IllegalStateException( + PixelsSinkConstants.SINK_DEBEZIUM_DIALECT + + " is required for Kafka transaction decoding" + + " (mysql or postgresql)"); + } + return adapter; + } + + private static Map toConfigMap(Properties properties) + { + Map configuration = new HashMap<>(); + properties.forEach((key, value) -> configuration.put(key.toString(), value)); + return configuration; + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowRecordDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowRecordDeserializer.java new file mode 100644 index 0000000..28902ba --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowRecordDeserializer.java @@ -0,0 +1,48 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.source.kafka.serde; + +import com.google.protobuf.InvalidProtocolBufferException; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.conversion.sinkproto.RowRecordConverter; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.serialization.Deserializer; + +public final class RowRecordDeserializer implements Deserializer +{ + private final RowRecordConverter converter = + new RowRecordConverter(TableMetadataRegistry.Instance()); + + @Override + public RowChangeEvent deserialize(String topic, byte[] data) + { + if (data == null || data.length == 0) + { + return null; + } + try + { + return converter.convert(SinkProto.RowRecord.parseFrom(data)); + } catch (InvalidProtocolBufferException | SinkException e) + { + throw new SerializationException( + "Failed to deserialize canonical row record from " + topic, e); + } + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataDeserializer.java new file mode 100644 index 0000000..9b4ac1b --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataDeserializer.java @@ -0,0 +1,42 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.source.kafka.serde; + +import com.google.protobuf.InvalidProtocolBufferException; +import io.pixelsdb.pixels.sink.SinkProto; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.serialization.Deserializer; + +public final class TransactionMetadataDeserializer + implements Deserializer +{ + @Override + public SinkProto.TransactionMetadata deserialize(String topic, byte[] data) + { + if (data == null || data.length == 0) + { + return null; + } + try + { + return SinkProto.TransactionMetadata.parseFrom(data); + } catch (InvalidProtocolBufferException e) + { + throw new SerializationException( + "Failed to deserialize canonical transaction metadata from " + topic, e); + } + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractMemorySinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractMemorySinkStorageSource.java deleted file mode 100644 index d3e0f7c..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractMemorySinkStorageSource.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.source.storage; - -import io.pixelsdb.pixels.common.physical.PhysicalReader; -import io.pixelsdb.pixels.common.physical.PhysicalReaderUtil; -import io.pixelsdb.pixels.common.physical.Storage; -import io.pixelsdb.pixels.core.utils.Pair; -import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; -import io.pixelsdb.pixels.sink.provider.ProtoType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.LinkedBlockingQueue; - -public abstract class AbstractMemorySinkStorageSource extends AbstractSinkStorageSource -{ - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractMemorySinkStorageSource.class); - - // All preloaded records, order preserved - // key + value buffer - private final List> preloadedRecords = new ArrayList<>(); - - @Override - public void start() - { - this.running.set(true); - this.transactionProcessorThread.start(); - this.transactionProviderThread.start(); - try - { - /* ===================================================== - * 1. Initialization phase: preload all ByteBuffers - * ===================================================== */ - for (String file : files) - { - Storage.Scheme scheme = Storage.Scheme.fromPath(file); - LOGGER.info("Preloading file {}", file); - - PhysicalReader reader = PhysicalReaderUtil.newPhysicalReader(scheme, file); - readers.add(reader); - - while (true) - { - int key; - int valueLen; - - try - { - key = reader.readInt(ByteOrder.BIG_ENDIAN); - valueLen = reader.readInt(ByteOrder.BIG_ENDIAN); - } catch (IOException eof) - { - // Reached end of file - break; - } - // Synchronous read and copy to heap buffer - ByteBuffer valueBuffer = reader.readFully(valueLen); - // Store into a single global array - ByteBuffer cleanBuffer = valueBuffer.duplicate(); - cleanBuffer.rewind(); - cleanBuffer.limit(cleanBuffer.position() + valueLen); - preloadedRecords.add(new Pair<>(key, cleanBuffer)); - } - } - - LOGGER.info("Preload finished, total records = {}", preloadedRecords.size()); - - /* ===================================================== - * Phase 2: Runtime loop - * Queue initialization, consumer startup, and feeding - * are done together in this phase - * ===================================================== */ - do - { - for (Pair record : preloadedRecords) - { - int key = record.getLeft(); - ByteBuffer src = record.getRight(); - ByteBuffer copy = ByteBuffer.allocate(src.remaining()); - copy.put(src.duplicate().rewind()); - copy.flip(); - // Lazily create queue - BlockingQueue, Integer>> queue = - queueMap.computeIfAbsent( - key, - k -> new LinkedBlockingQueue<>(PixelsSinkConstants.MAX_QUEUE_SIZE) - ); - - // Lazily start consumer thread - consumerThreads.computeIfAbsent(key, k -> - { - ProtoType protoType = getProtoType(k); - Thread t = new Thread(() -> consumeQueue(k, queue, protoType)); - t.setName("consumer-" + k); - t.start(); - return t; - }); - - ProtoType protoType = getProtoType(key); - if (protoType == ProtoType.ROW) - { - sourceRateLimiter.acquire(1); - } - - // Use completed future to keep consumer logic unchanged - CompletableFuture future = - CompletableFuture.completedFuture(copy); - - queue.put(new Pair<>(future, loopId)); - } - ++loopId; - } while (storageLoopEnabled && isRunning()); - } catch (IOException | IndexOutOfBoundsException e) - { - throw new RuntimeException(e); - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - } finally - { - clean(); - } - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractReaderSinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractReaderSinkStorageSource.java deleted file mode 100644 index 8be58ea..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractReaderSinkStorageSource.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.source.storage; - -import io.pixelsdb.pixels.common.physical.PhysicalReader; -import io.pixelsdb.pixels.common.physical.PhysicalReaderUtil; -import io.pixelsdb.pixels.common.physical.Storage; -import io.pixelsdb.pixels.core.utils.Pair; -import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; -import io.pixelsdb.pixels.sink.provider.ProtoType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.LinkedBlockingQueue; - -public abstract class AbstractReaderSinkStorageSource extends AbstractSinkStorageSource -{ - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractReaderSinkStorageSource.class); - - @Override - public void start() - { - this.running.set(true); - this.transactionProcessorThread.start(); - this.transactionProviderThread.start(); - for (String file : files) - { - Storage.Scheme scheme = Storage.Scheme.fromPath(file); - LOGGER.info("Start read from file {}", file); - PhysicalReader reader; - try - { - reader = PhysicalReaderUtil.newPhysicalReader(scheme, file); - } catch (IOException e) - { - throw new RuntimeException(e); - } - readers.add(reader); - } - do - { - for (PhysicalReader reader : readers) - { - LOGGER.info("Start Read {}", reader.getPath()); - long offset = 0; - while (true) - { - try - { - int key, valueLen; - reader.seek(offset); - try - { - key = reader.readInt(ByteOrder.BIG_ENDIAN); - valueLen = reader.readInt(ByteOrder.BIG_ENDIAN); - } catch (IOException e) - { - // EOF - break; - } - - ProtoType protoType = getProtoType(key); - offset += Integer.BYTES * 2; - CompletableFuture valueFuture = reader.readAsync(offset, valueLen) - .thenApply(this::copyToHeap) - .thenApply(buf -> buf.order(ByteOrder.BIG_ENDIAN)); - // move offset for next record - offset += valueLen; - - - // Get or create queue - BlockingQueue, Integer>> queue = - queueMap.computeIfAbsent(key, - k -> new LinkedBlockingQueue<>(PixelsSinkConstants.MAX_QUEUE_SIZE)); - - // Put future in queue - if (protoType.equals(ProtoType.ROW)) - { - sourceRateLimiter.acquire(1); - } - queue.put(new Pair<>(valueFuture, loopId)); - // Start consumer thread if not exists - consumerThreads.computeIfAbsent(key, k -> - { - Thread t = new Thread(() -> consumeQueue(k, queue, protoType)); - t.setName("consumer-" + key); - t.start(); - return t; - }); - } catch (IOException | InterruptedException e) - { - break; - } - } - } - ++loopId; - } while (storageLoopEnabled && isRunning()); - - clean(); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java index f65ae60..1e39f60 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java @@ -1,37 +1,39 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.storage; import io.pixelsdb.pixels.common.physical.PhysicalReader; import io.pixelsdb.pixels.core.utils.Pair; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.conversion.sinkproto.RowRecordConverter; +import io.pixelsdb.pixels.sink.conversion.sinkproto.TransactionMetadataConverter; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; -import io.pixelsdb.pixels.sink.processor.TransactionProcessor; +import io.pixelsdb.pixels.sink.pipeline.TablePipelineManager; +import io.pixelsdb.pixels.sink.pipeline.TransactionPipeline; import io.pixelsdb.pixels.sink.provider.ProtoType; -import io.pixelsdb.pixels.sink.provider.TableProviderAndProcessorPipelineManager; -import io.pixelsdb.pixels.sink.provider.TransactionEventStorageLoopProvider; import io.pixelsdb.pixels.sink.source.SinkSource; import io.pixelsdb.pixels.sink.util.EtcdFileRegistry; +import io.pixelsdb.pixels.sink.util.DataTransform; import io.pixelsdb.pixels.sink.util.MetricsFacade; +import io.pixelsdb.pixels.sink.util.concurrent.DecodeExecutors; +import io.pixelsdb.pixels.sink.util.concurrent.OrderedBatchDecoder; import io.pixelsdb.pixels.sink.util.rateLimiter.FlushRateLimiter; import io.pixelsdb.pixels.sink.util.rateLimiter.FlushRateLimiterFactory; import org.slf4j.Logger; @@ -39,19 +41,32 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; public abstract class AbstractSinkStorageSource implements SinkSource { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractSinkStorageSource.class); + private static final int DECODE_BATCH_SIZE = 64; + private static final long DECODE_BATCH_WAIT_MILLIS = 5; + private static final long DECODE_SHUTDOWN_TIMEOUT_SECONDS = 30; + protected static final int RECORD_HEADER_SIZE = Integer.BYTES * 2; protected final AtomicBoolean running = new AtomicBoolean(false); + private final AtomicBoolean started = new AtomicBoolean(false); + private final AtomicBoolean abortRequested = new AtomicBoolean(false); + private final CountDownLatch stopped = new CountDownLatch(1); + private volatile Thread sourceThread; protected final String topic; protected final String baseDir; @@ -62,13 +77,14 @@ public abstract class AbstractSinkStorageSource implements SinkSource protected final Map, Integer>>> queueMap = new ConcurrentHashMap<>(); protected final boolean storageLoopEnabled; protected final FlushRateLimiter sourceRateLimiter; - private final TableMetadataRegistry tableMetadataRegistry = TableMetadataRegistry.Instance(); + protected final TablePipelineManager tablePipelineManager = new TablePipelineManager(); + protected final TransactionPipeline transactionPipeline = new TransactionPipeline(); + protected final RowRecordConverter rowRecordConverter; + protected final TransactionMetadataConverter transactionMetadataConverter = + new TransactionMetadataConverter(); + protected final boolean freshnessTimestamp; private final MetricsFacade metricsFacade = MetricsFacade.getInstance(); - private final TableProviderAndProcessorPipelineManager> tablePipelineManager = new TableProviderAndProcessorPipelineManager<>(); - protected TransactionEventStorageLoopProvider> transactionEventProvider; - protected TransactionProcessor transactionProcessor; - protected Thread transactionProviderThread; - protected Thread transactionProcessorThread; + private final ExecutorService decodeExecutor; protected int loopId = 0; protected List readers = new ArrayList<>(); @@ -80,85 +96,304 @@ protected AbstractSinkStorageSource() this.etcdFileRegistry = new EtcdFileRegistry(topic, baseDir); this.files = this.etcdFileRegistry.listAllFiles(); this.storageLoopEnabled = pixelsSinkConfig.isSinkStorageLoop(); + this.rowRecordConverter = new RowRecordConverter(TableMetadataRegistry.Instance()); + this.freshnessTimestamp = pixelsSinkConfig.isSinkMonitorFreshnessTimestamp(); + this.sourceRateLimiter = FlushRateLimiterFactory.getNewInstance(); + this.decodeExecutor = DecodeExecutors.newFixedCallerRuns( + pixelsSinkConfig.getSourceDecodeThreads(), + "storage-proto-decoder"); + } + + protected void beginProcessing() + { + if (!started.compareAndSet(false, true)) + { + throw new IllegalStateException("Storage source has already been started"); + } + sourceThread = Thread.currentThread(); + running.set(true); + transactionPipeline.start(); + } - this.transactionEventProvider = new TransactionEventStorageLoopProvider<>(); - this.transactionProviderThread = new Thread(transactionEventProvider); + ProtoType getProtoType(int key) + { + return key == -1 ? ProtoType.TRANS : ProtoType.ROW; + } - this.transactionProcessor = new TransactionProcessor(transactionEventProvider); - this.transactionProcessorThread = new Thread(transactionProcessor, "debezium-processor"); - this.sourceRateLimiter = FlushRateLimiterFactory.getNewInstance(); + protected static Pair readRecord( + PhysicalReader reader, long offset, long fileLength) throws IOException + { + long remaining = fileLength - offset; + if (remaining < RECORD_HEADER_SIZE) + { + throw new IOException( + "Truncated sink proto header at offset " + offset + + " in " + reader.getPath()); + } + + int key = reader.readInt(ByteOrder.BIG_ENDIAN); + int valueLength = reader.readInt(ByteOrder.BIG_ENDIAN); + long availablePayload = remaining - RECORD_HEADER_SIZE; + if (valueLength < 0 || valueLength > availablePayload) + { + throw new IOException( + "Invalid sink proto payload length " + valueLength + + " at offset " + offset + " in " + reader.getPath()); + } + + return new Pair<>(key, reader.readFully(valueLength)); } - abstract ProtoType getProtoType(int i); + protected void submitRecord(int key, ByteBuffer valueBuffer, int recordLoopId) + throws InterruptedException + { + BlockingQueue, Integer>> queue = + queueMap.computeIfAbsent( + key, + ignored -> new LinkedBlockingQueue<>(PixelsSinkConstants.MAX_QUEUE_SIZE) + ); + + consumerThreads.computeIfAbsent(key, ignored -> + { + ProtoType protoType = getProtoType(key); + Thread thread = new Thread(() -> consumeQueue(key, queue, protoType)); + thread.setName("consumer-" + key); + thread.start(); + return thread; + }); + + if (getProtoType(key) == ProtoType.ROW) + { + sourceRateLimiter.acquire(1); + } + + queue.put(new Pair<>( + CompletableFuture.completedFuture(valueBuffer), + recordLoopId + )); + } protected void clean() { - queueMap.values().forEach(q -> + boolean interrupted = false; + try { - try + running.set(false); + if (!abortRequested.get()) { - q.put(new Pair<>(POISON_PILL, loopId)); - } catch (InterruptedException e) + for (BlockingQueue, Integer>> queue + : queueMap.values()) + { + try + { + queue.put(new Pair<>(POISON_PILL, loopId)); + } catch (InterruptedException e) + { + interrupted = true; + abortRequested.set(true); + break; + } + } + } + if (abortRequested.get()) { - Thread.currentThread().interrupt(); + consumerThreads.values().forEach(Thread::interrupt); } - }); - consumerThreads.values().forEach(t -> - { - try + for (Thread thread : consumerThreads.values()) { - t.join(); - } catch (InterruptedException e) + while (thread.isAlive()) + { + try + { + thread.join(); + } catch (InterruptedException e) + { + interrupted = true; + abortRequested.set(true); + consumerThreads.values().forEach(Thread::interrupt); + } + } + } + + if (abortRequested.get()) { - Thread.currentThread().interrupt(); + decodeExecutor.shutdownNow(); + } else + { + shutdownDecodeExecutor(); } - }); - for (PhysicalReader reader : readers) - { - try + for (PhysicalReader reader : readers) { - reader.close(); - } catch (IOException e) + try + { + reader.close(); + } catch (IOException e) + { + LOGGER.warn("Failed to close reader", e); + } + } + if (abortRequested.get()) + { + tablePipelineManager.abort(); + transactionPipeline.abort(); + } else + { + tablePipelineManager.close(); + transactionPipeline.close(); + } + } finally + { + sourceThread = null; + stopped.countDown(); + if (interrupted) { - LOGGER.warn("Failed to close reader", e); + Thread.currentThread().interrupt(); } } } - protected void handleTransactionSourceRecord(ByteBuffer record, Integer loopId) + private void shutdownDecodeExecutor() { - transactionEventProvider.putTransRawEvent(new Pair<>(record, loopId)); + DecodeExecutors.shutdownGracefully( + decodeExecutor, DECODE_SHUTDOWN_TIMEOUT_SECONDS, LOGGER); } protected void consumeQueue(int key, BlockingQueue, Integer>> queue, ProtoType protoType) { + boolean stopAfterBatch = false; try { - while (true) + while (!stopAfterBatch) { - Pair, Integer> pair = queue.take(); - CompletableFuture value = pair.getLeft(); - int loopId = pair.getRight(); - if (value == POISON_PILL) + List, Integer>> batch = + new ArrayList<>(DECODE_BATCH_SIZE); + Pair, Integer> first = queue.take(); + if (isPoisonPill(first)) { break; } - ByteBuffer valueBuffer = value.get(); - metricsFacade.recordDebeziumEvent(); - switch (protoType) + batch.add(first); + + long deadline = System.nanoTime() + + TimeUnit.MILLISECONDS.toNanos(DECODE_BATCH_WAIT_MILLIS); + while (batch.size() < DECODE_BATCH_SIZE) { - case ROW -> handleRowChangeSourceRecord(key, valueBuffer, loopId); - case TRANS -> handleTransactionSourceRecord(valueBuffer, loopId); + long remainingNanos = deadline - System.nanoTime(); + if (remainingNanos <= 0) + { + break; + } + + Pair, Integer> next = + queue.poll(remainingNanos, TimeUnit.NANOSECONDS); + if (next == null) + { + break; + } + if (isPoisonPill(next)) + { + stopAfterBatch = true; + break; + } + batch.add(next); } + + processBatch(key, batch, protoType); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); + } + } + + private boolean isPoisonPill( + Pair, Integer> record) + { + return record.getLeft() == POISON_PILL; + } + + private void processBatch( + int key, + List, Integer>> batch, + ProtoType protoType) throws InterruptedException + { + for (int i = 0; i < batch.size(); ++i) + { + metricsFacade.recordDebeziumEvent(); + } + + switch (protoType) + { + case ROW -> + { + List events = OrderedBatchDecoder.decodeInOrder( + decodeExecutor, + batch, + record -> decodeRowChangeSourceRecord(key, record)); + for (RowChangeEvent event : events) + { + if (event != null) + { + metricsFacade.recordSerdRowChange(); + tablePipelineManager.route(event); + } + } + } + case TRANS -> + { + List transactions = + OrderedBatchDecoder.decodeInOrder( + decodeExecutor, + batch, + this::decodeTransactionSourceRecord); + for (SinkProto.TransactionMetadata transaction : transactions) + { + if (transaction != null) + { + metricsFacade.recordSerdTxChange(); + transactionPipeline.publish(transaction); + } + } + } + } + } + + private RowChangeEvent decodeRowChangeSourceRecord( + int key, + Pair, Integer> record) + { + try + { + return convertRowChangeSourceRecord( + key, record.getLeft().get(), record.getRight()); + } catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + return null; } catch (ExecutionException e) { - LOGGER.error("Error in async processing", e); + LOGGER.warn("Failed to read storage row record", e.getCause()); + return null; + } + } + + private SinkProto.TransactionMetadata decodeTransactionSourceRecord( + Pair, Integer> record) + { + try + { + return transactionMetadataConverter.convert( + record.getLeft().get(), record.getRight()); + } catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + return null; + } catch (Exception e) + { + LOGGER.warn("Failed to convert storage transaction metadata", e); + return null; } } @@ -171,9 +406,30 @@ protected ByteBuffer copyToHeap(ByteBuffer directBuffer) return heapBuffer; } - protected void handleRowChangeSourceRecord(int key, ByteBuffer dataBuffer, int loopId) + protected RowChangeEvent convertRowChangeSourceRecord( + int key, ByteBuffer dataBuffer, int loopId) { - tablePipelineManager.routeRecord(key, new Pair<>(dataBuffer, loopId)); + try + { + SinkProto.RowRecord.Builder builder = + rowRecordConverter.parse(dataBuffer).toBuilder(); + if (freshnessTimestamp) + { + DataTransform.updateRecordTimestamp(builder, System.currentTimeMillis() * 1000); + } + if (builder.hasTransaction()) + { + SinkProto.TransactionInfo transaction = builder.getTransaction(); + builder.setTransaction(transaction.toBuilder() + .setId(transaction.getId() + "_" + loopId) + .build()); + } + return rowRecordConverter.convert(builder.build()); + } catch (Exception e) + { + LOGGER.warn("Failed to convert storage row record", e); + return null; + } } @Override @@ -183,11 +439,46 @@ public boolean isRunning() } @Override - public void stopProcessor() + public void close() { running.set(false); - transactionProviderThread.interrupt(); - transactionProcessorThread.interrupt(); - transactionProcessor.stopProcessor(); + if (!started.get() || Thread.currentThread() == sourceThread) + { + return; + } + + boolean interrupted = false; + while (true) + { + try + { + stopped.await(); + break; + } catch (InterruptedException e) + { + interrupted = true; + } + } + if (interrupted) + { + Thread.currentThread().interrupt(); + } + } + + @Override + public void abort() + { + abortRequested.set(true); + running.set(false); + consumerThreads.values().forEach(Thread::interrupt); + decodeExecutor.shutdownNow(); + tablePipelineManager.abort(); + transactionPipeline.abort(); + + Thread thread = sourceThread; + if (thread != null && thread != Thread.currentThread()) + { + thread.interrupt(); + } } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/FasterSinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/FasterSinkStorageSource.java deleted file mode 100644 index fc49922..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/source/storage/FasterSinkStorageSource.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.source.storage; - - -import io.pixelsdb.pixels.common.metadata.SchemaTableName; -import io.pixelsdb.pixels.sink.provider.ProtoType; -import io.pixelsdb.pixels.sink.source.SinkSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.ByteBuffer; - -/** - * @package: io.pixelsdb.pixels.sink.source - * @className: LegacySinkStorageSource - * @author: AntiO2 - * @date: 2025/10/5 11:43 - */ -public class FasterSinkStorageSource extends AbstractMemorySinkStorageSource implements SinkSource -{ - private static final Logger LOGGER = LoggerFactory.getLogger(FasterSinkStorageSource.class); - static SchemaTableName transactionSchemaTableName = new SchemaTableName("freak", "transaction"); - - public FasterSinkStorageSource() - { - super(); - } - - private static String readString(ByteBuffer buffer, int len) - { - byte[] bytes = new byte[len]; - buffer.get(bytes); - return new String(bytes); - } - - @Override - ProtoType getProtoType(int i) - { - if (i == -1) - { - return ProtoType.TRANS; - } - return ProtoType.ROW; - } - -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/LegacySinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/LegacySinkStorageSource.java deleted file mode 100644 index a8d636b..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/source/storage/LegacySinkStorageSource.java +++ /dev/null @@ -1,247 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.source.storage; - - -import io.pixelsdb.pixels.common.metadata.SchemaTableName; -import io.pixelsdb.pixels.common.physical.PhysicalReader; -import io.pixelsdb.pixels.common.physical.PhysicalReaderUtil; -import io.pixelsdb.pixels.common.physical.Storage; -import io.pixelsdb.pixels.core.utils.Pair; -import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; -import io.pixelsdb.pixels.sink.processor.TransactionProcessor; -import io.pixelsdb.pixels.sink.provider.ProtoType; -import io.pixelsdb.pixels.sink.provider.TableProviderAndProcessorPipelineManager; -import io.pixelsdb.pixels.sink.provider.TransactionEventEngineProvider; -import io.pixelsdb.pixels.sink.source.SinkSource; -import io.pixelsdb.pixels.sink.util.MetricsFacade; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.Map; -import java.util.concurrent.*; - -/** - * @package: io.pixelsdb.pixels.sink.source - * @className: LegacySinkStorageSource - * @author: AntiO2 - * @date: 2025/10/5 11:43 - */ -@Deprecated -public class LegacySinkStorageSource extends AbstractReaderSinkStorageSource implements SinkSource -{ - private static final Logger LOGGER = LoggerFactory.getLogger(LegacySinkStorageSource.class); - static SchemaTableName transactionSchemaTableName = new SchemaTableName("freak", "transaction"); - private final TransactionEventEngineProvider transactionEventProvider = TransactionEventEngineProvider.INSTANCE; - - private final TableProviderAndProcessorPipelineManager tableProvidersManagerImpl = new TableProviderAndProcessorPipelineManager<>(); - private final TransactionProcessor transactionProcessor = new TransactionProcessor(transactionEventProvider); - private final MetricsFacade metricsFacade = MetricsFacade.getInstance(); - private final Map>> queueMap = new ConcurrentHashMap<>(); - private final Map consumerThreads = new ConcurrentHashMap<>(); - private final int maxQueueCapacity = 10000; - private final TableMetadataRegistry tableMetadataRegistry = TableMetadataRegistry.Instance(); - private final CompletableFuture POISON_PILL = new CompletableFuture<>(); - - - private static String readString(ByteBuffer buffer, int len) - { - byte[] bytes = new byte[len]; - buffer.get(bytes); - return new String(bytes); - } - - @Override - ProtoType getProtoType(int i) - { - return ProtoType.fromInt(i); - } - - @Override - public void start() - { - - for (String file : files) - { - Storage.Scheme scheme = Storage.Scheme.fromPath(file); - LOGGER.info("Start read from file {}", file); - try (PhysicalReader reader = PhysicalReaderUtil.newPhysicalReader(scheme, file)) - { - long offset = 0; - BlockingQueue>> rowQueue = new LinkedBlockingQueue<>(); - BlockingQueue> transQueue = new LinkedBlockingQueue<>(); - while (true) - { - try - { - int keyLen, valueLen; - reader.seek(offset); - try - { - keyLen = reader.readInt(ByteOrder.BIG_ENDIAN); - valueLen = reader.readInt(ByteOrder.BIG_ENDIAN); - } catch (IOException e) - { - // EOF - break; - } - - ByteBuffer keyBuffer = copyToHeap(reader.readFully(keyLen)).order(ByteOrder.BIG_ENDIAN); - ProtoType protoType = getProtoType(keyBuffer.getInt()); - offset += Integer.BYTES * 2 + keyLen; - CompletableFuture valueFuture = reader.readAsync(offset, valueLen) - .thenApply(this::copyToHeap) - .thenApply(buf -> buf.order(ByteOrder.BIG_ENDIAN)); - // move offset for next record - offset += valueLen; - - // Compute queue key (for example: schemaName + tableName or protoType) - SchemaTableName queueKey = computeQueueKey(keyBuffer, protoType); - - // Get or create queue - BlockingQueue> queue = - queueMap.computeIfAbsent(queueKey, - k -> new LinkedBlockingQueue<>(maxQueueCapacity)); - - // Put future in queue - queue.put(valueFuture); - - // Start consumer thread if not exists - consumerThreads.computeIfAbsent(queueKey, k -> - { - Thread t = new Thread(() -> consumeQueue(k, queue, protoType)); - t.setName("consumer-" + queueKey); - t.start(); - return t; - }); - } catch (IOException | InterruptedException e) - { - break; - } - } - } catch (IOException e) - { - throw new RuntimeException(e); - } - - } - - // signal all queues to stop - queueMap.values().forEach(q -> - { - try - { - q.put(POISON_PILL); - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - } - }); - - // wait all consumers to finish - consumerThreads.values().forEach(t -> - { - try - { - t.join(); - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - } - }); - } - - private void consumeQueue(SchemaTableName key, BlockingQueue> queue, ProtoType protoType) - { - try - { - while (true) - { - CompletableFuture value = queue.take(); - if (value == POISON_PILL) - { - break; - } - ByteBuffer valueBuffer = value.get(); - metricsFacade.recordDebeziumEvent(); - switch (protoType) - { - case ROW -> handleRowChangeSourceRecord(0, valueBuffer, 0); - case TRANS -> handleTransactionSourceRecord(valueBuffer, 0); - } - } - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - } catch (ExecutionException e) - { - LOGGER.error("Error in async processing", e); - } - } - - private SchemaTableName computeQueueKey(ByteBuffer keyBuffer, ProtoType protoType) - { - switch (protoType) - { - case ROW -> - { - int schemaLen = keyBuffer.getInt(); - int tableLen = keyBuffer.getInt(); - String schemaName = readString(keyBuffer, schemaLen); - String tableName = readString(keyBuffer, tableLen); - return new SchemaTableName(schemaName, tableName); - } - case TRANS -> - { - return transactionSchemaTableName; - } - default -> - { - throw new IllegalArgumentException("Proto type " + protoType.toString()); - } - } - } - - private void handleRowChangeSourceRecord(SchemaTableName schemaTableName, ByteBuffer dataBuffer) - { - tableProvidersManagerImpl.routeRecord(schemaTableName, dataBuffer); - } - - private void handleRowChangeSourceRecord(ByteBuffer keyBuffer, ByteBuffer dataBuffer) - { - { - // CODE BLOCK VERSION 2 -// long tableId = keyBuffer.getLong(); -// try -// { -// schemaTableName = tableMetadataRegistry.getSchemaTableName(tableId); -// } catch (SinkException e) -// { -// throw new RuntimeException(e); -// } - } - -// tableProvidersManagerImpl.routeRecord(schemaTableName, dataBuffer); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/MemorySinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/MemorySinkStorageSource.java new file mode 100644 index 0000000..b8e1b79 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/storage/MemorySinkStorageSource.java @@ -0,0 +1,110 @@ +/* + * Copyright 2025 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.source.storage; + +import io.pixelsdb.pixels.common.physical.PhysicalReader; +import io.pixelsdb.pixels.common.physical.PhysicalReaderUtil; +import io.pixelsdb.pixels.common.physical.Storage; +import io.pixelsdb.pixels.core.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class MemorySinkStorageSource extends AbstractSinkStorageSource +{ + private static final Logger LOGGER = LoggerFactory.getLogger(MemorySinkStorageSource.class); + + // All preloaded records, order preserved + // key + value buffer + private final List> preloadedRecords = new ArrayList<>(); + + @Override + public void start() + { + beginProcessing(); + try + { + /* ===================================================== + * 1. Initialization phase: preload all ByteBuffers + * ===================================================== */ + for (String file : files) + { + if (!isRunning()) + { + break; + } + Storage.Scheme scheme = Storage.Scheme.fromPath(file); + LOGGER.info("Preloading file {}", file); + + PhysicalReader reader = PhysicalReaderUtil.newPhysicalReader(scheme, file); + readers.add(reader); + + reader.seek(0); + long offset = 0; + long fileLength = reader.getFileLength(); + while (isRunning() && offset < fileLength) + { + Pair record = readRecord(reader, offset, fileLength); + int valueLength = record.getRight().remaining(); + // Store into a single global array + ByteBuffer cleanBuffer = record.getRight().duplicate(); + cleanBuffer.rewind(); + cleanBuffer.limit(cleanBuffer.position() + valueLength); + preloadedRecords.add(new Pair<>(record.getLeft(), cleanBuffer)); + offset += RECORD_HEADER_SIZE + (long) valueLength; + } + } + + LOGGER.info("Preload finished, total records = {}", preloadedRecords.size()); + + /* ===================================================== + * Phase 2: Runtime loop + * Queue initialization, consumer startup, and feeding + * are done together in this phase + * ===================================================== */ + do + { + for (Pair record : preloadedRecords) + { + if (!isRunning()) + { + break; + } + int key = record.getLeft(); + ByteBuffer src = record.getRight(); + ByteBuffer copy = ByteBuffer.allocate(src.remaining()); + copy.put(src.duplicate().rewind()); + copy.flip(); + submitRecord(key, copy, loopId); + } + ++loopId; + } while (storageLoopEnabled && isRunning()); + } catch (IOException | IndexOutOfBoundsException e) + { + throw new RuntimeException(e); + } catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + } finally + { + clean(); + } + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/StreamingSinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/StreamingSinkStorageSource.java new file mode 100644 index 0000000..a9100c6 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/storage/StreamingSinkStorageSource.java @@ -0,0 +1,82 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.source.storage; + +import io.pixelsdb.pixels.common.physical.PhysicalReader; +import io.pixelsdb.pixels.common.physical.PhysicalReaderUtil; +import io.pixelsdb.pixels.common.physical.Storage; +import io.pixelsdb.pixels.core.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; + +public class StreamingSinkStorageSource extends AbstractSinkStorageSource +{ + private static final Logger LOGGER = LoggerFactory.getLogger(StreamingSinkStorageSource.class); + + @Override + public void start() + { + beginProcessing(); + try + { + for (String file : files) + { + if (!isRunning()) + { + break; + } + Storage.Scheme scheme = Storage.Scheme.fromPath(file); + readers.add(PhysicalReaderUtil.newPhysicalReader(scheme, file)); + } + + do + { + for (PhysicalReader reader : readers) + { + if (!isRunning()) + { + break; + } + + LOGGER.info("Start reading {}", reader.getPath()); + reader.seek(0); + long offset = 0; + long fileLength = reader.getFileLength(); + while (isRunning() && offset < fileLength) + { + Pair record = readRecord(reader, offset, fileLength); + int valueLength = record.getRight().remaining(); + submitRecord(record.getLeft(), record.getRight(), loopId); + offset += RECORD_HEADER_SIZE + (long) valueLength; + } + } + ++loopId; + } while (storageLoopEnabled && isRunning()); + } catch (IOException e) + { + throw new RuntimeException("Failed to read sink proto storage", e); + } catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + } finally + { + clean(); + } + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedMap.java b/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedMap.java index 4f649aa..7c3825c 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedMap.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedMap.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util; import java.util.Set; diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueue.java b/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueue.java new file mode 100644 index 0000000..30de2cb --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueue.java @@ -0,0 +1,119 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.util; + +import java.io.Closeable; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +/** + * A bounded blocking queue with an explicit shutdown signal. + * + *

Closing the queue finishes it gracefully. Use {@link #abort()} to + * discard pending values and wake a blocked consumer immediately.

+ */ +public final class BlockingBoundedQueue implements Closeable +{ + private static final Object POISON_PILL = new Object(); + + private final BlockingQueue queue; + private volatile boolean closed; + + public BlockingBoundedQueue(int capacity) + { + if (capacity <= 0) + { + throw new IllegalArgumentException("capacity must be positive"); + } + this.queue = new LinkedBlockingQueue<>(capacity); + } + + public void put(T value) + { + if (value == null || closed) + { + return; + } + try + { + queue.put(value); + } catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + } + } + + @SuppressWarnings("unchecked") + public T take() + { + try + { + Object value = queue.take(); + if (value == POISON_PILL) + { + return null; + } + return (T) value; + } catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + return null; + } + } + + /** + * Stops accepting new values and wakes the consumer after all queued values + * have been consumed. + * + *

The caller must stop all producers before invoking this method.

+ */ + @Override + public void close() + { + if (closed) + { + return; + } + closed = true; + boolean interrupted = false; + while (true) + { + try + { + queue.put(POISON_PILL); + break; + } catch (InterruptedException e) + { + interrupted = true; + } + } + if (interrupted) + { + Thread.currentThread().interrupt(); + } + } + + /** + * Stops accepting new values, discards pending values, and wakes a blocked + * consumer. Values already consumed are not rolled back. + */ + public void abort() + { + closed = true; + queue.clear(); + queue.offer(POISON_PILL); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/DataTransform.java b/src/main/java/io/pixelsdb/pixels/sink/util/DataTransform.java index 92b656e..c5c9536 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/DataTransform.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/DataTransform.java @@ -1,27 +1,21 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util; import com.google.protobuf.ByteString; -import io.pixelsdb.pixels.retina.RetinaProto; import io.pixelsdb.pixels.sink.SinkProto; import java.nio.ByteBuffer; @@ -36,44 +30,6 @@ private static ByteString longToByteString(long value) return ByteString.copyFrom(bytes); } - private static int byteStringToInt(ByteString bytes) - { - return ByteBuffer.wrap(bytes.toByteArray()).getInt(); - } - - @Deprecated - public static void updateTimeStamp(List updateData, long txStartTime) - { - ByteString timestampBytes = longToByteString(txStartTime); - - for (RetinaProto.TableUpdateData.Builder tableUpdateDataBuilder : updateData) - { - int insertDataCount = tableUpdateDataBuilder.getInsertDataCount(); - for (int i = 0; i < insertDataCount; i++) - { - RetinaProto.InsertData.Builder insertBuilder = tableUpdateDataBuilder.getInsertDataBuilder(i); - int colValueCount = insertBuilder.getColValuesCount(); - if (colValueCount > 0) - { - insertBuilder.setColValues(colValueCount - 1, timestampBytes); - } - } - - int updateDataCount = tableUpdateDataBuilder.getUpdateDataCount(); - for (int i = 0; i < updateDataCount; i++) - { - RetinaProto.UpdateData.Builder updateBuilder = tableUpdateDataBuilder.getUpdateDataBuilder(i); - - int colValueCount = updateBuilder.getColValuesCount(); - if (colValueCount > 0) - { - updateBuilder.setColValues(colValueCount - 1, timestampBytes); - } - } - } - } - - public static List updateRecordTimestamp(List records, long timestamp) { if (records == null || records.isEmpty()) @@ -129,22 +85,6 @@ public static void updateRecordTimestamp(SinkProto.RowRecord.Builder recordBuild } } - public static void transIdToBigint(SinkProto.RowRecord.Builder recordBuilder) - { - - if (recordBuilder.hasAfter()) - { - SinkProto.RowValue.Builder afterBuilder = recordBuilder.getAfterBuilder(); - afterBuilder.setValues(0, getTimestampColumn(byteStringToInt(afterBuilder.getValues(0).getValue()))); - } - if (recordBuilder.hasBefore()) - { - SinkProto.RowValue.Builder beforeBuilder = recordBuilder.getBeforeBuilder(); - beforeBuilder.setValues(0, getTimestampColumn(byteStringToInt(beforeBuilder.getValues(0).getValue()))); - } - } - - private static SinkProto.RowRecord updateRecordTimestamp(SinkProto.RowRecord.Builder recordBuilder, SinkProto.ColumnValue timestampColumn) { switch (recordBuilder.getOp()) diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/DateUtil.java b/src/main/java/io/pixelsdb/pixels/sink/util/DateUtil.java index 5b6735c..5aedd51 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/DateUtil.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/DateUtil.java @@ -1,35 +1,23 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util; -import io.pixelsdb.pixels.core.utils.DatetimeUtils; - import java.text.DateFormat; import java.text.SimpleDateFormat; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.Calendar; import java.util.Date; /** @@ -41,29 +29,6 @@ public class DateUtil { - public static Date fromDebeziumDate(int epochDay) - { - Calendar cal = Calendar.getInstance(); - cal.clear(); - cal.set(1970, Calendar.JANUARY, 1); // epoch 起点 - cal.add(Calendar.DAY_OF_MONTH, epochDay); // 加上天数 - return cal.getTime(); - } - - // TIMESTAMP(1), TIMESTAMP(2), TIMESTAMP(3) - public static Date fromDebeziumTimestamp(long epochTs) - { - return new Date(epochTs / 1000); - } - - public static String convertDateToDayString(Date date) - { - // "yyyy-MM-dd HH:mm:ss.SSS" - DateFormat df = new SimpleDateFormat("yyyy-MM-dd"); - String dateToString = df.format(date); - return (dateToString); - } - public static String convertDateToString(Date date) { // "yyyy-MM-dd HH:mm:ss.SSS" @@ -72,20 +37,4 @@ public static String convertDateToString(Date date) return (dateToString); } - public static String convertTimestampToString(Date date) - { - if (date == null) - { - return null; - } - SimpleDateFormat sdf = new SimpleDateFormat("HH:mm:ss.SSS"); - return sdf.format(date); - } - - public static String convertDebeziumTimestampToString(long epochTs) - { - LocalDateTime dateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(epochTs), ZoneId.systemDefault()); - DateTimeFormatter formatter = DatetimeUtils.SQL_LOCAL_DATE_TIME; - return dateTime.format(formatter); - } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistry.java b/src/main/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistry.java index a5330cf..b3ac9ac 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistry.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistry.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util; @@ -191,8 +186,4 @@ public void markFileCompleted(String fileName) LOGGER.info("Marked file [{}] as completed", fileName); } - public void cleanData() - { - etcd.deleteByPrefix(topicPrefix()); - } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/LatencySimulator.java b/src/main/java/io/pixelsdb/pixels/sink/util/LatencySimulator.java deleted file mode 100644 index ad3f86c..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/util/LatencySimulator.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.util; - -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; - -import java.util.Random; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; - -public class LatencySimulator -{ - private static final Random RANDOM = ThreadLocalRandom.current(); - private static final double longTailProb = 0.05; - private static final double longTailScale = 30; - private static final double tailVariance = 0.1; - private static final double normalVariance = 0.4; - - private static long generateLongTailDelay(long baseDelayMs) - { - if (RANDOM.nextDouble() < longTailProb) - { - double variance = 1 + (RANDOM.nextDouble() * 2 - 1) * tailVariance; - return (long) (baseDelayMs * longTailScale * variance); - } - - return (long) (baseDelayMs * (1 + (RANDOM.nextDouble() - 0.5) * normalVariance)); - } - - public static void smartDelay() - { - try - { - TimeUnit.MILLISECONDS.sleep(generateLongTailDelay(PixelsSinkConfigFactory.getInstance().getMockRpcDelay())); - } catch (InterruptedException ignored) - { - - } - } -} \ No newline at end of file diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/MetricsFacade.java b/src/main/java/io/pixelsdb/pixels/sink/util/MetricsFacade.java index 734319b..860a6db 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/MetricsFacade.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/MetricsFacade.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util; import com.google.protobuf.ByteString; diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/TableCounters.java b/src/main/java/io/pixelsdb/pixels/sink/util/TableCounters.java index b6c0850..bfd597b 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/TableCounters.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/TableCounters.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util; /** diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/DecodeExecutors.java b/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/DecodeExecutors.java new file mode 100644 index 0000000..43edf91 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/DecodeExecutors.java @@ -0,0 +1,94 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.util.concurrent; + +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; +import org.slf4j.Logger; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Shared fixed-size decode pools with caller-runs backpressure. + * Tasks submitted after shutdown are rejected. + */ +public final class DecodeExecutors +{ + private DecodeExecutors() + { + } + + public static ExecutorService newFixedCallerRuns(int threads, String threadNamePrefix) + { + if (threads <= 0) + { + throw new IllegalArgumentException("decode threads must be positive"); + } + if (threadNamePrefix == null || threadNamePrefix.isBlank()) + { + throw new IllegalArgumentException("threadNamePrefix must be non-blank"); + } + + AtomicInteger threadId = new AtomicInteger(); + return new ThreadPoolExecutor( + threads, + threads, + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(PixelsSinkConstants.MAX_QUEUE_SIZE), + runnable -> new Thread( + runnable, + threadNamePrefix + "-" + threadId.incrementAndGet()), + (task, executor) -> + { + if (executor.isShutdown()) + { + throw new RejectedExecutionException( + "Decode executor is shut down: " + threadNamePrefix); + } + task.run(); + }); + } + + public static void shutdownGracefully( + ExecutorService executor, long timeoutSeconds, Logger logger) + { + if (executor == null) + { + return; + } + executor.shutdown(); + try + { + if (!executor.awaitTermination(timeoutSeconds, TimeUnit.SECONDS)) + { + if (logger != null) + { + logger.warn("Timed out waiting for decode executor to stop"); + } + executor.shutdownNow(); + } + } catch (InterruptedException e) + { + executor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoder.java b/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoder.java new file mode 100644 index 0000000..38555c1 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoder.java @@ -0,0 +1,65 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.util.concurrent; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.function.Function; + +/** + * Decodes a batch in parallel and returns results in input order. + * Failed records become {@code null} entries. + */ +public final class OrderedBatchDecoder +{ + private static final Logger LOGGER = LoggerFactory.getLogger(OrderedBatchDecoder.class); + + private OrderedBatchDecoder() + { + } + + public static List decodeInOrder( + ExecutorService executor, + List records, + Function decoder) throws InterruptedException + { + List> futures = new ArrayList<>(records.size()); + for (T record : records) + { + futures.add(executor.submit(() -> decoder.apply(record))); + } + + List decodedRecords = new ArrayList<>(records.size()); + for (Future future : futures) + { + try + { + decodedRecords.add(future.get()); + } catch (ExecutionException e) + { + LOGGER.warn("Failed to decode record", e.getCause()); + decodedRecords.add(null); + } + } + return decodedRecords; + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoder.java b/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoder.java new file mode 100644 index 0000000..56a04d8 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoder.java @@ -0,0 +1,245 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.util.concurrent; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.function.Consumer; + +/** + * Parallel decode with per-stream publish ordering. + * Same {@code streamKey} is published FIFO; different keys may publish in parallel. + */ +public final class StreamOrderedDecoder implements AutoCloseable +{ + private static final Logger LOGGER = LoggerFactory.getLogger(StreamOrderedDecoder.class); + private static final long SHUTDOWN_TIMEOUT_SECONDS = 30; + + @FunctionalInterface + public interface ThrowingFunction + { + O apply(I input) throws Exception; + } + + public record DecodeResult(O value, Throwable failure) + { + } + + private final Object lifecycleLock = new Object(); + private final Map> streamTails = new HashMap<>(); + private final Set> activeCompletions = ConcurrentHashMap.newKeySet(); + private final ExecutorService decodeExecutor; + private final boolean ownsExecutor; + private boolean started; + private boolean accepting; + private boolean closed; + private volatile boolean aborted; + + public StreamOrderedDecoder(int decodeThreads, String threadNamePrefix) + { + this(DecodeExecutors.newFixedCallerRuns(decodeThreads, threadNamePrefix), true); + } + + private StreamOrderedDecoder(ExecutorService decodeExecutor, boolean ownsExecutor) + { + if (decodeExecutor == null) + { + throw new IllegalArgumentException("decodeExecutor must not be null"); + } + this.decodeExecutor = decodeExecutor; + this.ownsExecutor = ownsExecutor; + } + + public void start() + { + synchronized (lifecycleLock) + { + if (started) + { + return; + } + if (closed) + { + throw new IllegalStateException("StreamOrderedDecoder is closed"); + } + started = true; + accepting = true; + } + } + + public CompletableFuture submit( + Object streamKey, + I input, + ThrowingFunction decoder, + Consumer> afterDecode) + { + CompletableFuture previous; + CompletableFuture completion = new CompletableFuture<>(); + synchronized (lifecycleLock) + { + ensureAccepting(); + previous = streamTails.getOrDefault( + streamKey, CompletableFuture.completedFuture(null)); + streamTails.put(streamKey, completion); + activeCompletions.add(completion); + } + + CompletableFuture> decoded = CompletableFuture.supplyAsync( + () -> decode(input, decoder), decodeExecutor); + + previous.thenCombine(decoded, (ignored, result) -> + { + publish(result, afterDecode); + return null; + }) + .whenComplete((ignored, failure) -> + { + if (failure != null) + { + completion.completeExceptionally(failure); + } + else + { + completion.complete(null); + } + removeCompletion(streamKey, completion); + }); + return completion; + } + + private void ensureAccepting() + { + if (!started || !accepting) + { + throw new IllegalStateException( + "StreamOrderedDecoder is not accepting records"); + } + } + + private DecodeResult decode(I input, ThrowingFunction decoder) + { + try + { + return new DecodeResult<>(decoder.apply(input), null); + } catch (Exception e) + { + return new DecodeResult<>(null, e); + } + } + + private void publish(DecodeResult result, Consumer> afterDecode) + { + if (aborted) + { + throw new CancellationException("StreamOrderedDecoder aborted"); + } + afterDecode.accept(result); + } + + private void removeCompletion(Object streamKey, CompletableFuture completion) + { + activeCompletions.remove(completion); + synchronized (lifecycleLock) + { + // Keep a failed tail so later submits on the same stream fail fast + // instead of starting a fresh successful chain. + if (streamTails.get(streamKey) == completion + && !completion.isCompletedExceptionally()) + { + streamTails.remove(streamKey); + } + } + } + + @Override + public void close() + { + List> pending; + synchronized (lifecycleLock) + { + if (closed) + { + return; + } + closed = true; + accepting = false; + pending = new ArrayList<>(activeCompletions); + } + + try + { + awaitCompletions(pending); + if (ownsExecutor) + { + DecodeExecutors.shutdownGracefully( + decodeExecutor, SHUTDOWN_TIMEOUT_SECONDS, LOGGER); + } + } catch (InterruptedException e) + { + abort(); + Thread.currentThread().interrupt(); + } + } + + public void abort() + { + List> pending; + synchronized (lifecycleLock) + { + if (aborted) + { + return; + } + aborted = true; + closed = true; + accepting = false; + pending = new ArrayList<>(activeCompletions); + } + CancellationException cancellation = + new CancellationException("StreamOrderedDecoder aborted"); + pending.forEach(completion -> completion.completeExceptionally(cancellation)); + if (ownsExecutor) + { + decodeExecutor.shutdownNow(); + } + } + + private void awaitCompletions(List> pending) + throws InterruptedException + { + for (CompletableFuture completion : pending) + { + try + { + completion.get(); + } catch (ExecutionException | CancellationException e) + { + LOGGER.warn("StreamOrderedDecoder completed with an error", e); + } + } + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/FlushRateLimiter.java b/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/FlushRateLimiter.java index d62c25a..bcef26a 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/FlushRateLimiter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/FlushRateLimiter.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util.rateLimiter; public interface FlushRateLimiter diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/FlushRateLimiterFactory.java b/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/FlushRateLimiterFactory.java index 1956d31..54c6917 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/FlushRateLimiterFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/FlushRateLimiterFactory.java @@ -1,23 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util.rateLimiter; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/GuavaFlushRateLimiter.java b/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/GuavaFlushRateLimiter.java index 9c4ebd0..83a1f82 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/GuavaFlushRateLimiter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/GuavaFlushRateLimiter.java @@ -1,23 +1,18 @@ /* - * Copyright 2026 PixelsDB. + * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util.rateLimiter; import com.google.common.util.concurrent.RateLimiter; diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/NoOpFlushRateLimiter.java b/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/NoOpFlushRateLimiter.java index 05745c1..374d28a 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/NoOpFlushRateLimiter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/NoOpFlushRateLimiter.java @@ -1,23 +1,18 @@ /* - * Copyright 2026 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util.rateLimiter; public class NoOpFlushRateLimiter implements FlushRateLimiter diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/SemaphoreFlushRateLimiter.java b/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/SemaphoreFlushRateLimiter.java index 52c2c5a..1a37653 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/SemaphoreFlushRateLimiter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/SemaphoreFlushRateLimiter.java @@ -1,23 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util.rateLimiter; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/AbstractBucketedWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/AbstractBucketedWriter.java index 52bd4b3..3723fe4 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/AbstractBucketedWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/AbstractBucketedWriter.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/NoneWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/NoneWriter.java index f40bae0..aa9d5ec 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/NoneWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/NoneWriter.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkMode.java b/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkMode.java index c5c6e15..5f53498 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkMode.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkMode.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkWriter.java index 8e2744a..11cb3f5 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkWriter.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkWriterFactory.java b/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkWriterFactory.java index 5697ecb..cc901d2 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkWriterFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkWriterFactory.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/csv/CsvWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/csv/CsvWriter.java index 540d0f7..0165a52 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/csv/CsvWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/csv/CsvWriter.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.csv; import com.google.common.util.concurrent.ThreadFactoryBuilder; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/flink/FlinkPollingWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/flink/FlinkPollingWriter.java index 33881bc..72c5985 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/flink/FlinkPollingWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/flink/FlinkPollingWriter.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.flink; import io.pixelsdb.pixels.common.metadata.SchemaTableName; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/flink/PixelsPollingServiceImpl.java b/src/main/java/io/pixelsdb/pixels/sink/writer/flink/PixelsPollingServiceImpl.java index 9c33848..a6db288 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/flink/PixelsPollingServiceImpl.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/flink/PixelsPollingServiceImpl.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.flink; import io.grpc.stub.StreamObserver; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/flink/PollingRpcServer.java b/src/main/java/io/pixelsdb/pixels/sink/writer/flink/PollingRpcServer.java index 333f21b..5233667 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/flink/PollingRpcServer.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/flink/PollingRpcServer.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.flink; import io.grpc.Server; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/proto/ProtoWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/proto/ProtoWriter.java index 23f5dd0..f5273f2 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/proto/ProtoWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/proto/ProtoWriter.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.proto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/proto/RotatingWriterManager.java b/src/main/java/io/pixelsdb/pixels/sink/writer/proto/RotatingWriterManager.java index 5d9ff13..b45d305 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/proto/RotatingWriterManager.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/proto/RotatingWriterManager.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.proto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/InFlightControlManager.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/InFlightControlManager.java index 7298506..9d0fb9b 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/InFlightControlManager.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/InFlightControlManager.java @@ -1,23 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaBucketDispatcher.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaBucketDispatcher.java index 9dbb492..4055fce 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaBucketDispatcher.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaBucketDispatcher.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.sink.event.RowChangeEvent; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaServiceProxy.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaServiceProxy.java index 7ded27f..6d33416 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaServiceProxy.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaServiceProxy.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.common.exception.RetinaException; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriter.java index eaf4f3b..8e20f91 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriter.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.sink.SinkProto; @@ -90,7 +85,6 @@ public boolean writeRow(RowChangeEvent event) } metricsFacade.recordRowChange(event.getTable(), event.getOp()); - event.startLatencyTimer(); if (event.getTransaction() == null || event.getTransaction().getId().isEmpty()) { handleNonTxEvent(event); diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/SinkContext.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/SinkContext.java index dbea99f..d66b7ac 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/SinkContext.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/SinkContext.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.common.transaction.TransContext; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/SinkContextManager.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/SinkContextManager.java index e74829e..01ba91b 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/SinkContextManager.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/SinkContextManager.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.common.transaction.TransContext; @@ -42,7 +37,6 @@ public class SinkContextManager { private static final Logger LOGGER = LoggerFactory.getLogger(SinkContextManager.class); - private static final Logger BUCKET_TRACE_LOGGER = LoggerFactory.getLogger("bucket_trace"); private static volatile SinkContextManager instance; private final BlockingBoundedMap activeTxContexts; // private final ConcurrentMap activeTxContexts = new ConcurrentHashMap<>(10000); diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableCrossTxWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableCrossTxWriter.java index c73864b..cd29f0f 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableCrossTxWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableCrossTxWriter.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; @@ -106,29 +101,39 @@ public void flush(List batch) // flushRateLimiter.acquire(batch.size()); long txStartTime = System.currentTimeMillis(); -// if(freshnessLevel.equals("embed")) -// { -// long freshness_ts = txStartTime * 1000; -// FreshnessClient.getInstance().addMonitoredTable(tableName); -// DataTransform.updateTimeStamp(tableUpdateDataBuilderList, freshness_ts); -// } - List tableUpdateData = new ArrayList<>(tableUpdateDataBuilderList.size()); for (RetinaProto.TableUpdateData.Builder tableUpdateDataItem : tableUpdateDataBuilderList) { tableUpdateData.add(tableUpdateDataItem.build()); } + int rowCount = batch.size(); inFlightControlManager.acquire(1); + LOGGER.debug("Sending {} rows of table {} to retina, txIds={}", rowCount, tableName, txIds); CompletableFuture updateRecordResponseCompletableFuture = delegate.writeBatchAsync(batch.get(0).getSchemaName(), tableUpdateData); + if (updateRecordResponseCompletableFuture == null) + { + inFlightControlManager.release(1); + LOGGER.error("Failed to submit {} rows of table {} to retina, txIds={}", rowCount, tableName, txIds); + failCtxs(txIds); + return; + } - updateRecordResponseCompletableFuture.thenAccept( - resp -> + updateRecordResponseCompletableFuture.whenComplete( + (resp, err) -> { inFlightControlManager.release(1); - if (resp.getHeader().getErrorCode() != 0) + if (err != null) + { + LOGGER.error("Retina write failed for {} rows of table {}, txIds={}", + rowCount, tableName, txIds, err); + failCtxs(txIds); + } else if (resp.getHeader().getErrorCode() != 0) { + LOGGER.error("Retina rejected {} rows of table {}, txIds={}, errorCode={}, errorMsg={}", + rowCount, tableName, txIds, resp.getHeader().getErrorCode(), + resp.getHeader().getErrorMsg()); failCtxs(txIds); } else { @@ -138,6 +143,7 @@ public void flush(List batch) metricsFacade.recordFreshness(txEndTime - txStartTime); } updateCtxCounters(txIds, fullTableName, tableUpdateCount); + LOGGER.debug("Retina acked {} rows of table {}, txIds={}", rowCount, tableName, txIds); } } ); diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableSingleRecordWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableSingleRecordWriter.java index 9f02226..737f4c8 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableSingleRecordWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableSingleRecordWriter.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.common.transaction.TransContext; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableSingleTxWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableSingleTxWriter.java index 6ccea6e..1ff9846 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableSingleTxWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableSingleTxWriter.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.retina.RetinaProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableWriter.java index 3472121..9508a30 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableWriter.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableWriterProxy.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableWriterProxy.java index 071011c..bd04800 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableWriterProxy.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableWriterProxy.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.common.node.BucketCache; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TransactionMode.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TransactionMode.java index 1ff0347..1d7de8d 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TransactionMode.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TransactionMode.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; public enum TransactionMode diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TransactionProxy.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TransactionProxy.java index 8128a7e..5c8713d 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TransactionProxy.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TransactionProxy.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.common.exception.TransException; @@ -125,12 +120,6 @@ private void requestTransactions() } } - @Deprecated - public TransContext getNewTransContext() - { - return getNewTransContext("None"); - } - public TransContext getNewTransContext(String txId) { beginCount.incrementAndGet(); diff --git a/src/main/resources/META-INF/services/io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter b/src/main/resources/META-INF/services/io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter new file mode 100644 index 0000000..aa5df8c --- /dev/null +++ b/src/main/resources/META-INF/services/io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter @@ -0,0 +1,2 @@ +io.pixelsdb.pixels.sink.conversion.debezium.dialect.MySqlSourceAdapter +io.pixelsdb.pixels.sink.conversion.debezium.dialect.PostgresSourceAdapter diff --git a/src/main/resources/log4j2.properties b/src/main/resources/log4j2.properties index e1fe75b..77fbf7b 100644 --- a/src/main/resources/log4j2.properties +++ b/src/main/resources/log4j2.properties @@ -1,41 +1,49 @@ -status=info +status=warn name=pixels-sink filter.threshold.type=ThresholdFilter -filter.threshold.level=info +# allow DEBUG for selected loggers (e.g. DebeziumConsumer receive path) +filter.threshold.level=debug appender.console.type=Console appender.console.name=STDOUT appender.console.layout.type=PatternLayout appender.console.layout.pattern=%-d{yyyy-MM-dd HH:mm:ss,SSS} [%c]-[%p] %m%n -appender.rolling.type=File +appender.rolling.type=RollingFile appender.rolling.name=log appender.rolling.append=true appender.rolling.fileName=${env:PIXELS_HOME}/logs/pixels-sink.log +appender.rolling.filePattern=${env:PIXELS_HOME}/logs/pixels-sink-%d{yyyy-MM-dd}-%i.log.gz appender.rolling.layout.type=PatternLayout appender.rolling.layout.pattern=%-d{yyyy-MM-dd HH:mm:ss,SSS} [%c]-[%p] %m%n +appender.rolling.policies.type=Policies +appender.rolling.policies.time.type=TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval=1 +appender.rolling.policies.time.modulate=true +appender.rolling.policies.size.type=SizeBasedTriggeringPolicy +appender.rolling.policies.size.size=100MB +appender.rolling.strategy.type=DefaultRolloverStrategy +appender.rolling.strategy.max=10 rootLogger.level=info rootLogger.appenderRef.stdout.ref=STDOUT rootLogger.appenderRef.log.ref=log -logger.transaction.name=io.pixelsdb.pixels.sink.sink.retina.RetinaWriter -logger.transaction.level=info -logger.transaction.appenderRef.log.ref=log -logger.transaction.appenderRef.stdout.ref=STDOUT -logger.transaction.additivity=false +# CDC receive: topic/offset/tx/gtid/binlog pos +logger.debeziumConsumer.name=io.pixelsdb.pixels.sink.source.engine.PixelsDebeziumConsumer +logger.debeziumConsumer.level=debug +logger.debeziumConsumer.additivity=false +logger.debeziumConsumer.appenderRef.stdout.ref=STDOUT +logger.debeziumConsumer.appenderRef.log.ref=log +# Retina batch flush: send / ack (DEBUG), table/txIds/row count +logger.tableCrossFlush.name=io.pixelsdb.pixels.sink.writer.retina.TableCrossTxWriter +logger.tableCrossFlush.level=debug +logger.tableCrossFlush.additivity=false +logger.tableCrossFlush.appenderRef.stdout.ref=STDOUT +logger.tableCrossFlush.appenderRef.log.ref=log logger.grpc.name=io.grpc.netty.shaded.io.grpc.netty.NettyClientHandler -logger.grpc.level=info +logger.grpc.level=warn logger.grpc.additivity=false logger.grpc.appenderRef.log.ref=log logger.grpc.appenderRef.stdout.ref=STDOUT -log4j2.logger.io.grpc.netty.shaded.io.grpc.netty.NettyClientHandler=OFF logger.netty-shaded.name=io.grpc.netty.shaded.io.netty -logger.netty-shaded.level=info +logger.netty-shaded.level=warn logger.netty-shaded.additivity=false -appender.bucket_trace.type=File -appender.bucket_trace.name=BUCKET_TRACE -appender.bucket_trace.append=true -appender.bucket_trace.fileName=${env:PIXELS_HOME}/logs/bucket_trace.log -appender.bucket_trace.layout.type=PatternLayout -appender.bucket_trace.layout.pattern=%m%n -logger.bucket_trace.name=bucket_trace -logger.bucket_trace.level=warn -logger.bucket_trace.additivity=false -logger.bucket_trace.appenderRef.bucket_trace.ref=BUCKET_TRACE +logger.netty-shaded.appenderRef.log.ref=log +logger.netty-shaded.appenderRef.stdout.ref=STDOUT diff --git a/src/main/resources/logging.properties b/src/main/resources/logging.properties deleted file mode 100644 index 2c1fd24..0000000 --- a/src/main/resources/logging.properties +++ /dev/null @@ -1,3 +0,0 @@ -.level=INFO -io.grpc.level=INFO -io.grpc.netty.level=INFO diff --git a/src/main/resources/pixels-sink.aws.properties b/src/main/resources/pixels-sink.aws.properties index 5cabc00..badfca5 100644 --- a/src/main/resources/pixels-sink.aws.properties +++ b/src/main/resources/pixels-sink.aws.properties @@ -1,5 +1,8 @@ # engine | kafka | storage sink.datasource=storage +sink.storage.mode=stream +# CDC dialect for envelope normalization: mysql | postgresql +sink.debezium.dialect=postgresql # Sink Config: retina | csv | proto | none sink.mode=none # Kafka Config @@ -7,8 +10,7 @@ bootstrap.servers=realtime-kafka-2:29092 group.id=3078 auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -#value.deserializer=io.pixelsdb.pixels.writer.deserializer.RowChangeEventAvroDeserializer -value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventJsonDeserializer +sink.kafka.value.format=json # Topic & Database Config topic.prefix=postgresql.oltp_server consumer.capture_database=pixels_bench_sf1x @@ -38,8 +40,6 @@ sink.proto.maxRecords=1000000 sink.registry.url=http://localhost:8080/apis/registry/v2 # Transaction Config transaction.topic.suffix=transaction -#transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.deserializer.TransactionAvroMessageDeserializer -transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.TransactionJsonMessageDeserializer sink.trans.batch.size=200 ## Batch or trans or record sink.trans.mode=batch @@ -50,7 +50,6 @@ sink.monitor.report.interval=5000 sink.monitor.report.file=/home/ubuntu/pixels-sink/tmp/readonly3.csv # Interact with other rpc sink.rpc.enable=true -sink.rpc.mock.delay=20 # debezium debezium.name=testEngine debezium.connector.class=io.debezium.connector.postgresql.PostgresConnector diff --git a/src/main/resources/pixels-sink.local.properties b/src/main/resources/pixels-sink.local.properties index 360cfb1..51fff2d 100644 --- a/src/main/resources/pixels-sink.local.properties +++ b/src/main/resources/pixels-sink.local.properties @@ -1,5 +1,8 @@ # engine | kafka | storage sink.datasource=storage +sink.storage.mode=stream +# CDC dialect for envelope normalization: mysql | postgresql +sink.debezium.dialect=postgresql # Sink Config: retina | csv | proto | none sink.mode=proto # Kafka Config @@ -7,8 +10,7 @@ bootstrap.servers=localhost:29092 group.id=3107 auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -#value.deserializer=io.pixelsdb.pixels.sink.deserializer.RowChangeEventAvroDeserializer -value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventJsonDeserializer +sink.kafka.value.format=json # Topic & Database Config topic.prefix=postgresql.oltp_server consumer.capture_database=pixels_bench_sf1x @@ -39,8 +41,6 @@ sink.proto.maxRecords=1000000 sink.registry.url=http://localhost:8080/apis/registry/v2 # Transaction Config transaction.topic.suffix=transaction -#transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.deserializer.TransactionAvroMessageDeserializer -transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.TransactionJsonMessageDeserializer sink.trans.batch.size=100 ## Batch or trans or record sink.trans.mode=batch @@ -49,7 +49,6 @@ sink.monitor.enable=true sink.monitor.port=9464 # Interact with other rpc sink.rpc.enable=true -sink.rpc.mock.delay=20 # debezium debezium.name=testEngine debezium.connector.class=io.debezium.connector.postgresql.PostgresConnector diff --git a/src/main/resources/pixels-sink.properties b/src/main/resources/pixels-sink.properties index a903fea..71cf117 100644 --- a/src/main/resources/pixels-sink.properties +++ b/src/main/resources/pixels-sink.properties @@ -1,5 +1,6 @@ # Source Config sink.datasource=kafka +sink.storage.mode=stream sink.datasource.rate.limit=-1 # Kafka Config @@ -7,7 +8,7 @@ bootstrap.servers=pixels_kafka:9092 group.id=docker-pixels-table auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventJsonDeserializer +sink.kafka.value.format=json # Topic & Database Config topic.prefix=oltp_server @@ -40,7 +41,6 @@ sink.registry.url=http://apicurio:8080/apis/registry/v2 # Transaction Config transaction.topic.suffix=transaction transaction.topic.group_id=transaction_consumer -transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.TransactionJsonMessageDeserializer sink.trans.batch.size=100 sink.trans.mode=batch transaction.timeout=300 @@ -51,4 +51,3 @@ sink.monitor.port=9464 # Interact with other rpc sink.rpc.enable=false -sink.rpc.mock.delay=100 diff --git a/src/main/resources/pixels-sink.properties.template b/src/main/resources/pixels-sink.properties.template index c7b4a2d..4231326 100644 --- a/src/main/resources/pixels-sink.properties.template +++ b/src/main/resources/pixels-sink.properties.template @@ -3,7 +3,7 @@ bootstrap.servers=localhost:9092 group.id= auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -value.deserializer=org.apache.kafka.common.serialization.StringDeserializer +sink.kafka.value.format=json # Topic & Database Config topic.prefix= @@ -11,4 +11,4 @@ consumer.capture_database= consumer.include_tables= # Sink Config -output.directory= \ No newline at end of file +output.directory= diff --git a/src/test/java/io/pixelsdb/pixels/sink/DebeziumEngineTest.java b/src/test/java/io/pixelsdb/pixels/sink/DebeziumEngineTest.java index 58659ff..32ae53e 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/DebeziumEngineTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/DebeziumEngineTest.java @@ -12,10 +12,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ - - package io.pixelsdb.pixels.sink; @@ -24,9 +21,13 @@ import io.debezium.engine.RecordChangeEvent; import io.debezium.engine.format.ChangeEventFormat; import org.apache.kafka.connect.source.SourceRecord; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import java.util.List; +import java.nio.file.Path; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -37,10 +38,12 @@ * @author: AntiO2 * @date: 2025/9/25 12:16 */ -public class DebeziumEngineTest +@Tag("integration") +class DebeziumEngineTest { @Test - public void testPostgresCDC() + @Disabled("Manual PostgreSQL integration test; requires an external database") + void testPostgresCDC(@TempDir Path tempDir) throws Exception { final Properties props = new Properties(); @@ -49,17 +52,24 @@ public void testPostgresCDC() props.setProperty("provide.transaction.metadata", "true"); props.setProperty("offset.storage", "org.apache.kafka.connect.storage.FileOffsetBackingStore"); - props.setProperty("offset.storage.file.filename", "/tmp/offsets.dat"); + props.setProperty("offset.storage.file.filename", + tempDir.resolve("offsets.dat").toString()); props.setProperty("offset.flush.interval.ms", "60000"); props.setProperty("schema.history.internal", "io.debezium.storage.file.history.FileSchemaHistory"); - props.setProperty("schema.history.internal.file.filename", "/tmp/schemahistory.dat"); - - props.setProperty("database.hostname", "localhost"); - props.setProperty("database.port", "5432"); - props.setProperty("database.user", "pixels"); - props.setProperty("database.password", "pixels_realtime_crud"); - props.setProperty("database.dbname", "pixels_bench_sf1x"); + props.setProperty("schema.history.internal.file.filename", + tempDir.resolve("schemahistory.dat").toString()); + + props.setProperty("database.hostname", + requiredProperty("pixels.sink.debezium.database.hostname")); + props.setProperty("database.port", + requiredProperty("pixels.sink.debezium.database.port")); + props.setProperty("database.user", + requiredProperty("pixels.sink.debezium.database.user")); + props.setProperty("database.password", + requiredProperty("pixels.sink.debezium.database.password")); + props.setProperty("database.dbname", + requiredProperty("pixels.sink.debezium.database.name")); props.setProperty("plugin.name", "pgoutput"); props.setProperty("database.server.id", "1"); props.setProperty("schema.include.list", "public"); @@ -81,11 +91,24 @@ public void testPostgresCDC() ExecutorService executor = Executors.newSingleThreadExecutor(); executor.execute(engine); - - while (true) + try { + Thread.sleep(Long.getLong("pixels.sink.debezium.run.millis", 1000L)); + } finally + { + engine.close(); + executor.shutdownNow(); + } + } + private static String requiredProperty(String key) + { + String value = System.getProperty(key); + if (value == null || value.isBlank()) + { + throw new IllegalStateException("Set -D" + key + " to run this integration test"); } + return value; } class MyChangeConsumer implements DebeziumEngine.ChangeConsumer> diff --git a/src/test/java/io/pixelsdb/pixels/sink/TestConfig.java b/src/test/java/io/pixelsdb/pixels/sink/TestConfig.java new file mode 100644 index 0000000..ef1be78 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/TestConfig.java @@ -0,0 +1,63 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink; + +import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import org.junit.jupiter.api.Assumptions; + +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Objects; + +public final class TestConfig +{ + public static final String INTEGRATION_CONFIG_PROPERTY = + "pixels.sink.integration.config"; + + private TestConfig() + { + } + + public static void initializeUnitConfig() throws Exception + { + PixelsSinkConfigFactory.reset(); + PixelsSinkConfigFactory.initialize(resourcePath("pixels-sink-test.properties")); + } + + public static void initializeIntegrationConfig() throws Exception + { + String configPath = System.getProperty(INTEGRATION_CONFIG_PROPERTY); + Assumptions.assumeTrue(configPath != null && !configPath.isBlank(), + "Set -D" + INTEGRATION_CONFIG_PROPERTY + " to run integration tests"); + PixelsSinkConfigFactory.reset(); + PixelsSinkConfigFactory.initialize(configPath); + } + + public static String resourcePath(String resourceName) + { + try + { + Path path = Paths.get(Objects.requireNonNull( + TestConfig.class.getClassLoader().getResource(resourceName), + "Missing test resource: " + resourceName).toURI()); + return path.toString(); + } catch (URISyntaxException e) + { + throw new IllegalStateException("Invalid test resource: " + resourceName, e); + } + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/config/EngineValueFormatTest.java b/src/test/java/io/pixelsdb/pixels/sink/config/EngineValueFormatTest.java new file mode 100644 index 0000000..1b08a53 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/config/EngineValueFormatTest.java @@ -0,0 +1,42 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.config; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class EngineValueFormatTest +{ + @Test + void shouldDefaultToConnect() + { + assertEquals("connect", EngineValueFormat.resolve(null)); + assertEquals("connect", EngineValueFormat.resolve("")); + assertEquals("connect", EngineValueFormat.resolve("CONNECT")); + } + + @Test + void shouldRejectNonConnectOnResolve() + { + IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, + () -> EngineValueFormat.resolve("json")); + assertTrue(error.getMessage().contains("not implemented")); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/config/KafkaValueFormatTest.java b/src/test/java/io/pixelsdb/pixels/sink/config/KafkaValueFormatTest.java new file mode 100644 index 0000000..4c7294b --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/config/KafkaValueFormatTest.java @@ -0,0 +1,46 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.config; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class KafkaValueFormatTest +{ + @Test + void shouldDefaultToJson() + { + assertEquals("json", KafkaValueFormat.resolve(null)); + assertEquals("json", KafkaValueFormat.resolve("")); + assertEquals("json", KafkaValueFormat.resolve(" ")); + } + + @Test + void shouldNormalizeConfiguredFormat() + { + assertEquals("json", KafkaValueFormat.resolve("JSON")); + assertEquals("avro", KafkaValueFormat.resolve("Avro")); + } + + @Test + void shouldRejectUnknownFormat() + { + assertThrows(IllegalArgumentException.class, + () -> KafkaValueFormat.resolve("protobuf")); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/config/LegacyKafkaValueFormatMigrationTest.java b/src/test/java/io/pixelsdb/pixels/sink/config/LegacyKafkaValueFormatMigrationTest.java new file mode 100644 index 0000000..152c806 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/config/LegacyKafkaValueFormatMigrationTest.java @@ -0,0 +1,94 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.config; + +import org.junit.jupiter.api.Test; + +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class LegacyKafkaValueFormatMigrationTest +{ + @Test + void shouldDefaultToJsonWhenNoFormatOrLegacyKeys() + { + assertEquals("json", PixelsSinkConfig.resolveKafkaValueFormat(new Properties())); + } + + @Test + void shouldInferJsonFromLegacyDeserializerClass() + { + Properties props = new Properties(); + props.setProperty( + "value.deserializer", + "io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer"); + props.setProperty( + "transaction.topic.value.deserializer", + "io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer"); + assertEquals("json", PixelsSinkConfig.resolveKafkaValueFormat(props)); + } + + @Test + void shouldInferAvroFromLegacyDeserializerClass() + { + Properties props = new Properties(); + props.setProperty( + "value.deserializer", + "io.pixelsdb.pixels.sink.source.kafka.serde.RowChangeEventAvroDeserializer"); + assertEquals("avro", PixelsSinkConfig.resolveKafkaValueFormat(props)); + } + + @Test + void shouldPreferExplicitFormatAndIgnoreLegacyKeys() + { + Properties props = new Properties(); + props.setProperty("sink.kafka.value.format", "avro"); + props.setProperty( + "value.deserializer", + "io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer"); + assertEquals("avro", PixelsSinkConfig.resolveKafkaValueFormat(props)); + } + + @Test + void shouldFailFastOnConflictingLegacyFormats() + { + Properties props = new Properties(); + props.setProperty( + "value.deserializer", + "io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer"); + props.setProperty( + "transaction.topic.value.deserializer", + "io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataAvroDeserializer"); + IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, + () -> PixelsSinkConfig.resolveKafkaValueFormat(props)); + assertTrue(error.getMessage().contains("Conflicting")); + } + + @Test + void shouldFailFastOnUnrecognizedLegacyClass() + { + Properties props = new Properties(); + props.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, + () -> PixelsSinkConfig.resolveKafkaValueFormat(props)); + assertTrue(error.getMessage().contains("Unable to migrate")); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/consumer/AvroConsumerTest.java b/src/test/java/io/pixelsdb/pixels/sink/consumer/AvroConsumerTest.java deleted file mode 100644 index 3ee867d..0000000 --- a/src/test/java/io/pixelsdb/pixels/sink/consumer/AvroConsumerTest.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.pixelsdb.pixels.sink.consumer; - -import io.apicurio.registry.rest.client.RegistryClient; -import io.apicurio.registry.rest.client.RegistryClientFactory; -import io.apicurio.registry.serde.SerdeConfig; -import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventAvroDeserializer; -import io.pixelsdb.pixels.sink.exception.SinkException; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.time.Duration; -import java.util.Collections; -import java.util.Properties; - -public class AvroConsumerTest -{ - - private static final String TOPIC = "oltp_server.pixels_realtime_crud.customer"; - private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v2"; - private static final String BOOTSTRAP_SERVERS = "localhost:29092"; - private static final String GROUP_ID = "avro-consumer-test-group-1"; - - private static KafkaConsumer getRowChangeEventAvroKafkaConsumer() - { - Properties props = new Properties(); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); - props.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID); - props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); - props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, RowChangeEventAvroDeserializer.class.getName()); - props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - props.put(SerdeConfig.REGISTRY_URL, REGISTRY_URL); - props.put(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true"); - props.put(SerdeConfig.CHECK_PERIOD_MS, "30000"); - - KafkaConsumer consumer = new KafkaConsumer<>(props); - return consumer; - } - - private static void processRecord(RowChangeEvent event) - { -// RetinaProto.RowValue.Builder builder = RetinaProto.RowValue.newBuilder(); -// for (SinkProto.ColumnValue value : event.getRowRecord().getAfter().getValuesList()) { -// builder.addValues(value.getValue()); -// } -// builder.build(); - } - - private static KafkaConsumer getStringGenericRecordKafkaConsumer() - { - Properties props = new Properties(); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); - props.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID); - props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); - props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, AvroKafkaDeserializer.class.getName()); - props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - props.put(SerdeConfig.REGISTRY_URL, REGISTRY_URL); - props.put(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true"); - props.put(SerdeConfig.CHECK_PERIOD_MS, "30000"); - - KafkaConsumer consumer = new KafkaConsumer<>(props); - return consumer; - } - - private static RowChangeEvent convertToRowChangeEvent(GenericRecord record, Schema schema) throws SinkException - { - return new RowChangeEvent(SinkProto.RowRecord.newBuilder().build(), null); - } - - private static void processRecord(ConsumerRecord record, RegistryClient registryClient) - { - try - { - GenericRecord avroRecord = record.value(); - Schema schema = avroRecord.getSchema(); - - String schemaId = getSchemaIdFromRegistry(registryClient, schema); - System.out.println("Schema ID: " + schemaId); - - RowChangeEvent event = convertToRowChangeEvent(avroRecord, schema); - - System.out.println("Successfully processed message:"); - System.out.println("Topic: " + record.topic()); - System.out.println("Partition: " + record.partition()); - System.out.println("Offset: " + record.offset()); - System.out.println("Event: " + event); - - } catch (Exception e) - { - System.err.println("Error processing message: " + e.getMessage()); - e.printStackTrace(); - } - } - - private static String getSchemaIdFromRegistry(RegistryClient client, Schema schema) - { - String schemaContent = schema.toString(); - try - { - return ""; - } catch (Exception e) - { - throw new RuntimeException("Schema not found in registry: " + schema.getFullName(), e); - } - } - - @Test - public void avroConsumerTest() - { - KafkaConsumer consumer = getStringGenericRecordKafkaConsumer(); - consumer.subscribe(Collections.singletonList(TOPIC)); - - RegistryClient registryClient = RegistryClientFactory.create(REGISTRY_URL); - - try - { - while (true) - { - ConsumerRecords records = consumer.poll(Duration.ofMillis(100)); - for (ConsumerRecord record : records) - { - processRecord(record, registryClient); - } - } - } finally - { - consumer.close(); - } - } - - @Test - public void sinkConsumerTest() throws IOException - { - PixelsSinkConfigFactory.initialize("/home/anti/work/pixels-sink/src/main/resources/pixels-sink.local.properties"); - KafkaConsumer consumer = getRowChangeEventAvroKafkaConsumer(); - consumer.subscribe(Collections.singletonList(TOPIC)); - - try - { - while (true) - { - ConsumerRecords records = consumer.poll(Duration.ofMillis(100)); - for (ConsumerRecord record : records) - { - processRecord(record.value()); - } - } - } finally - { - consumer.close(); - } - } -} \ No newline at end of file diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectConverterTest.java new file mode 100644 index 0000000..8be88ba --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectConverterTest.java @@ -0,0 +1,206 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.connect; + +import io.pixelsdb.pixels.common.metadata.SchemaTableName; +import io.pixelsdb.pixels.common.metadata.domain.Column; +import io.pixelsdb.pixels.common.metadata.domain.Table; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.TestConfig; +import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.MySqlSourceAdapter; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.source.SourceRecord; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class DebeziumConnectConverterTest +{ + private static final SchemaTableName TABLE = + new SchemaTableName("cdc_verify", "binlog_test"); + private static final Schema ROW_SCHEMA = SchemaBuilder.struct().optional() + .field("id", Schema.INT64_SCHEMA) + .field("name", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + private static final Schema SOURCE_SCHEMA = SchemaBuilder.struct() + .field("connector", Schema.STRING_SCHEMA) + .field("db", Schema.STRING_SCHEMA) + .field("table", Schema.STRING_SCHEMA) + .build(); + private static final Schema TRANSACTION_INFO_SCHEMA = SchemaBuilder.struct().optional() + .field("id", Schema.STRING_SCHEMA) + .field("total_order", Schema.INT64_SCHEMA) + .field("data_collection_order", Schema.INT64_SCHEMA) + .build(); + + private static TableMetadata previousMetadata; + private static boolean hadMetadata; + + private final DebeziumConnectRowConverter rowConverter = + new DebeziumConnectRowConverter( + TableMetadataRegistry.Instance(), MySqlSourceAdapter.INSTANCE); + private final DebeziumConnectTransactionConverter transactionConverter = + new DebeziumConnectTransactionConverter(MySqlSourceAdapter.INSTANCE); + + @BeforeAll + static void setUpConfig() throws Exception + { + TestConfig.initializeUnitConfig(); + Map registry = metadataRegistry(); + hadMetadata = registry.containsKey(TABLE); + previousMetadata = registry.get(TABLE); + registry.put(TABLE, tableMetadata()); + } + + @AfterAll + static void resetConfig() throws ReflectiveOperationException + { + Map registry = metadataRegistry(); + if (hadMetadata) + { + registry.put(TABLE, previousMetadata); + } + else + { + registry.remove(TABLE); + } + PixelsSinkConfigFactory.reset(); + } + + @Test + void shouldConvertInsertRow() throws Exception + { + RowChangeEvent event = rowConverter.convert(insertRecord()); + + assertTrue(event.isInsert()); + assertEquals("binlog_test", event.getTable()); + assertTrue(event.hasAfterData()); + assertEquals("gtid:1", event.getTransaction().getId()); + } + + @Test + void shouldConvertTransactionBoundary() + { + Schema collectionSchema = SchemaBuilder.struct() + .field("data_collection", Schema.STRING_SCHEMA) + .field("event_count", Schema.INT64_SCHEMA) + .build(); + Schema schema = SchemaBuilder.struct() + .field("status", Schema.STRING_SCHEMA) + .field("id", Schema.STRING_SCHEMA) + .field("event_count", Schema.OPTIONAL_INT64_SCHEMA) + .field("data_collections", SchemaBuilder.array(collectionSchema).optional().build()) + .field("ts_ms", Schema.INT64_SCHEMA) + .build(); + Struct collection = new Struct(collectionSchema) + .put("data_collection", "cdc_verify.binlog_test") + .put("event_count", 1L); + Struct value = new Struct(schema) + .put("status", "END") + .put("id", "gtid:9") + .put("event_count", 1L) + .put("data_collections", List.of(collection)) + .put("ts_ms", 1750000000000L); + + SinkProto.TransactionMetadata transaction = transactionConverter.convert( + new SourceRecord( + Map.of("server", "mysql-cdc"), + Map.of("file", "binlog.000004", "pos", 152L), + "mysql-cdc.transaction", + null, + null, + schema, + value)); + + assertEquals(SinkProto.TransactionStatus.END, transaction.getStatus()); + assertEquals("gtid:9", transaction.getId()); + assertEquals("cdc_verify.binlog_test", + transaction.getDataCollections(0).getDataCollection()); + } + + private SourceRecord insertRecord() + { + Struct source = new Struct(SOURCE_SCHEMA) + .put("connector", "mysql") + .put("db", "cdc_verify") + .put("table", "binlog_test"); + Struct transaction = new Struct(TRANSACTION_INFO_SCHEMA) + .put("id", "gtid:1") + .put("total_order", 1L) + .put("data_collection_order", 1L); + Schema envelopeSchema = SchemaBuilder.struct() + .field("before", ROW_SCHEMA) + .field("after", ROW_SCHEMA) + .field("source", SOURCE_SCHEMA) + .field("transaction", TRANSACTION_INFO_SCHEMA) + .field("op", Schema.STRING_SCHEMA) + .build(); + Struct envelope = new Struct(envelopeSchema) + .put("before", null) + .put("after", new Struct(ROW_SCHEMA).put("id", 1L).put("name", "row-1")) + .put("source", source) + .put("transaction", transaction) + .put("op", "c"); + return new SourceRecord( + Map.of("server", "mysql-cdc"), + Map.of("file", "binlog.000004", "pos", 152L), + "mysql-cdc.cdc_verify.binlog_test", + null, + null, + envelopeSchema, + envelope); + } + + private static TableMetadata tableMetadata() throws Exception + { + Table table = new Table(); + table.setId(1); + table.setName("binlog_test"); + return new TableMetadata(table, null, List.of( + column("id", "bigint"), + column("name", "varchar(64)"))); + } + + private static Column column(String name, String type) + { + Column column = new Column(); + column.setName(name); + column.setType(type); + return column; + } + + @SuppressWarnings("unchecked") + private static Map metadataRegistry() + throws ReflectiveOperationException + { + Field registry = TableMetadataRegistry.class.getDeclaredField("registry"); + registry.setAccessible(true); + return (Map) registry.get(TableMetadataRegistry.Instance()); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonConverterTest.java new file mode 100644 index 0000000..e1c50d5 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonConverterTest.java @@ -0,0 +1,223 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.json; + +import io.pixelsdb.pixels.common.metadata.SchemaTableName; +import io.pixelsdb.pixels.common.metadata.domain.Column; +import io.pixelsdb.pixels.common.metadata.domain.Table; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.TestConfig; +import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.MySqlSourceAdapter; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class DebeziumJsonConverterTest +{ + private static final SchemaTableName REGION_TABLE = + new SchemaTableName("pixels_realtime_crud", "region"); + private static final SchemaTableName NATION_TABLE = + new SchemaTableName("pixels_realtime_crud", "nation"); + + private static TableMetadata previousRegionMetadata; + private static TableMetadata previousNationMetadata; + private static boolean hadRegionMetadata; + private static boolean hadNationMetadata; + + private final DebeziumJsonRowConverter rowConverter = + new DebeziumJsonRowConverter(TableMetadataRegistry.Instance(), null); + private final DebeziumJsonTransactionConverter transactionConverter = + new DebeziumJsonTransactionConverter(MySqlSourceAdapter.INSTANCE); + + @BeforeAll + static void setUpConfig() throws Exception + { + TestConfig.initializeUnitConfig(); + Map registry = metadataRegistry(); + hadRegionMetadata = registry.containsKey(REGION_TABLE); + previousRegionMetadata = registry.get(REGION_TABLE); + hadNationMetadata = registry.containsKey(NATION_TABLE); + previousNationMetadata = registry.get(NATION_TABLE); + // MySQL delete/update fixtures use uppercase column names. + registry.put(REGION_TABLE, tableMetadata(2, "region", + List.of("R_REGIONKEY", "R_NAME", "R_COMMENT"), + List.of("int", "string", "string"))); + registry.put(NATION_TABLE, tableMetadata(3, "nation", + List.of("n_nationkey", "n_name", "n_regionkey", "n_comment"), + List.of("int", "string", "int", "string"))); + } + + @AfterAll + static void resetConfig() throws ReflectiveOperationException + { + Map registry = metadataRegistry(); + restore(registry, REGION_TABLE, hadRegionMetadata, previousRegionMetadata); + restore(registry, NATION_TABLE, hadNationMetadata, previousNationMetadata); + PixelsSinkConfigFactory.reset(); + } + + @Test + void shouldHandleDeleteOperation() throws Exception + { + RowChangeEvent event = convertFixture("records/mysql-region-delete.json"); + assertTrue(event.isDelete()); + } + + @Test + void shouldHandleUpdateOperation() throws Exception + { + RowChangeEvent event = convertFixture("records/mysql-region-update.json"); + assertTrue(event.isUpdate()); + assertEquals("region", event.getTable()); + assertTrue(event.hasBeforeData()); + assertTrue(event.hasAfterData()); + } + + @Test + void shouldHandlePostgresInsert() throws Exception + { + Map registry = metadataRegistry(); + TableMetadata mysqlRegion = registry.get(REGION_TABLE); + // Postgres insert fixture uses lowercase column names. + registry.put(REGION_TABLE, tableMetadata(2, "region", + List.of("r_regionkey", "r_name", "r_comment"), + List.of("int", "string", "string"))); + try + { + RowChangeEvent event = convertFixture("records/postgresql-region-insert.json"); + assertTrue(event.isInsert()); + assertEquals("public.region", event.getFullTableName()); + assertEquals("779", event.getTransaction().getId()); + assertTrue(event.hasAfterData()); + } finally + { + registry.put(REGION_TABLE, mysqlRegion); + } + } + + @Test + void shouldHandlePostgresSnapshotNation() throws Exception + { + RowChangeEvent event = convertFixture("records/postgresql-nation-snapshot.json"); + assertTrue(event.isSnapshot()); + assertEquals("public.nation", event.getFullTableName()); + assertTrue(event.hasAfterData()); + } + + @Test + void shouldConvertTransactionMetadata() throws Exception + { + String json = """ + { + "payload": { + "status": "END", + "id": "mysql-tx-1", + "event_count": 1, + "ts_ms": 1750000000000, + "data_collections": [ + {"data_collection": "pixels_realtime_crud.region", "event_count": 1} + ] + } + } + """; + + SinkProto.TransactionMetadata transaction = transactionConverter.convert( + json.getBytes(StandardCharsets.UTF_8)); + + assertEquals(SinkProto.TransactionStatus.END, transaction.getStatus()); + assertEquals("mysql-tx-1", transaction.getId()); + assertEquals("pixels_realtime_crud.region", + transaction.getDataCollections(0).getDataCollection()); + } + + private RowChangeEvent convertFixture(String filename) throws Exception + { + return rowConverter.convert(loadFixture(filename).getBytes(StandardCharsets.UTF_8)); + } + + private String loadFixture(String filename) throws IOException, URISyntaxException + { + ClassLoader classLoader = getClass().getClassLoader(); + return Files.readString(Paths.get( + Objects.requireNonNull(classLoader.getResource(filename)).toURI()), + StandardCharsets.UTF_8); + } + + private static TableMetadata tableMetadata( + long id, String name, List columnNames, List columnTypes) + throws Exception + { + Table table = new Table(); + table.setId(id); + table.setName(name); + List columns = new java.util.ArrayList<>(columnNames.size()); + for (int i = 0; i < columnNames.size(); i++) + { + columns.add(column(columnNames.get(i), columnTypes.get(i))); + } + return new TableMetadata(table, null, columns); + } + + private static Column column(String name, String type) + { + Column column = new Column(); + column.setName(name); + column.setType(type); + return column; + } + + private static void restore( + Map registry, + SchemaTableName key, + boolean hadPrevious, + TableMetadata previous) + { + if (hadPrevious) + { + registry.put(key, previous); + } + else + { + registry.remove(key); + } + } + + @SuppressWarnings("unchecked") + private static Map metadataRegistry() + throws ReflectiveOperationException + { + Field registry = TableMetadataRegistry.class.getDeclaredField("registry"); + registry.setAccessible(true); + return (Map) registry.get(TableMetadataRegistry.Instance()); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumEnvelopeNormalizerTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumEnvelopeNormalizerTest.java new file mode 100644 index 0000000..9c6844b --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumEnvelopeNormalizerTest.java @@ -0,0 +1,226 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.support; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.pixelsdb.pixels.common.metadata.SchemaTableName; +import io.pixelsdb.pixels.common.metadata.domain.Table; +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.TestConfig; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapterRegistry; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.MySqlSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.PostgresSourceAdapter; +import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.apache.kafka.connect.data.Struct; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class DebeziumEnvelopeNormalizerTest +{ + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final SchemaTableName MYSQL_TABLE = + new SchemaTableName("cdc_verify", "binlog_test"); + private static TableMetadata previousMySqlMetadata; + private static boolean hadMySqlMetadata; + + @BeforeAll + static void setUpConfig() throws Exception + { + TestConfig.initializeUnitConfig(); + Map registry = metadataRegistry(); + hadMySqlMetadata = registry.containsKey(MYSQL_TABLE); + previousMySqlMetadata = registry.get(MYSQL_TABLE); + registry.put(MYSQL_TABLE, tableMetadata(1, "binlog_test")); + } + + @AfterAll + static void resetConfig() throws ReflectiveOperationException + { + Map registry = metadataRegistry(); + if (hadMySqlMetadata) + { + registry.put(MYSQL_TABLE, previousMySqlMetadata); + } + else + { + registry.remove(MYSQL_TABLE); + } + PixelsSinkConfigFactory.reset(); + } + + @Test + void shouldNormalizeMySqlSourceWithoutSchema() throws SinkException + { + Struct source = mysqlSource(); + + SinkProto.SourceInfo normalized = DebeziumEnvelopeNormalizer.normalizeSource( + source, MySqlSourceAdapter.INSTANCE); + + assertEquals("cdc_verify", normalized.getDb()); + assertEquals("", normalized.getSchema()); + assertEquals("binlog_test", normalized.getTable()); + assertEquals("cdc_verify.binlog_test", rowChangeEventFor(normalized).getFullTableName()); + } + + @Test + void shouldNormalizePostgresSourceStruct() + { + Schema sourceSchema = SchemaBuilder.struct() + .field("connector", Schema.STRING_SCHEMA) + .field("db", Schema.STRING_SCHEMA) + .field("schema", Schema.STRING_SCHEMA) + .field("table", Schema.STRING_SCHEMA) + .build(); + Struct source = new Struct(sourceSchema) + .put("connector", "postgresql") + .put("db", "pixels_realtime_crud") + .put("schema", "public") + .put("table", "region"); + + SinkProto.SourceInfo normalized = DebeziumEnvelopeNormalizer.normalizeSource( + source, PostgresSourceAdapter.INSTANCE); + + assertEquals("pixels_realtime_crud", normalized.getDb()); + assertEquals("public", normalized.getSchema()); + assertEquals("region", normalized.getTable()); + assertEquals("779", + PostgresSourceAdapter.INSTANCE.normalizeTransactionId("779")); + } + + @Test + void shouldRejectConfiguredConnectorMismatch() + { + IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, + () -> DebeziumEnvelopeNormalizer.normalizeSource( + mysqlSource(), PostgresSourceAdapter.INSTANCE)); + + assertTrue(error.getMessage().contains("source.connector mysql")); + } + + @Test + void shouldResolveConnectorAdaptersThroughSpi() + { + assertEquals("mysql", + DebeziumSourceAdapterRegistry.resolve( + "io.debezium.connector.mysql.MySqlConnector").connector()); + assertEquals("postgresql", + DebeziumSourceAdapterRegistry.resolve("postgresql").connector()); + } + + @Test + void shouldRejectBlankConnector() + { + assertThrows(IllegalArgumentException.class, + () -> DebeziumSourceAdapterRegistry.resolve(" ")); + } + + @Test + void shouldRejectUnsupportedConnector() + { + IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, + () -> DebeziumSourceAdapterRegistry.resolve("oracle")); + + assertTrue(error.getMessage().contains("Unsupported Debezium connector")); + } + + @Test + void shouldPreserveMySqlGtidAsOpaqueTransactionId() throws Exception + { + String gtid = "8d02ae5e-aeff-ca52-daa1-ab14474cee00:3"; + JsonNode end = OBJECT_MAPPER.readTree(""" + { + "status": "END", + "id": "%s", + "event_count": 4, + "ts_ms": 1750000000000, + "data_collections": [ + {"data_collection": "cdc_verify.binlog_test", "event_count": 4} + ] + } + """.formatted(gtid)); + + SinkProto.TransactionMetadata transaction = + DebeziumEnvelopeNormalizer.normalizeTransactionMetadata( + end, MySqlSourceAdapter.INSTANCE); + + assertEquals(SinkProto.TransactionStatus.END, transaction.getStatus()); + assertEquals(gtid, transaction.getId()); + assertEquals(4, transaction.getEventCount()); + assertEquals("cdc_verify.binlog_test", + transaction.getDataCollections(0).getDataCollection()); + } + + private Struct mysqlSource() + { + Schema sourceSchema = SchemaBuilder.struct() + .field("connector", Schema.STRING_SCHEMA) + .field("db", Schema.STRING_SCHEMA) + .field("table", Schema.STRING_SCHEMA) + .build(); + return new Struct(sourceSchema) + .put("connector", "mysql") + .put("db", "cdc_verify") + .put("table", "binlog_test"); + } + + private static RowChangeEvent rowChangeEventFor(SinkProto.SourceInfo source) throws SinkException + { + return new RowChangeEvent( + SinkProto.RowRecord.newBuilder() + .setSource(source) + .setOp(SinkProto.OperationType.SNAPSHOT) + .setAfter(SinkProto.RowValue.newBuilder().build()) + .build(), + TypeDescription.createSchemaFromStrings(new ArrayList<>(), new ArrayList<>())); + } + + private static TableMetadata tableMetadata(long id, String name) throws Exception + { + Table table = new Table(); + table.setId(id); + table.setName(name); + return new TableMetadata(table, null, List.of()); + } + + @SuppressWarnings("unchecked") + private static Map metadataRegistry() + throws ReflectiveOperationException + { + Field registry = TableMetadataRegistry.class.getDeclaredField("registry"); + registry.setAccessible(true); + return (Map) registry.get(TableMetadataRegistry.Instance()); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowValueConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowValueConverterTest.java new file mode 100644 index 0000000..53b6e79 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowValueConverterTest.java @@ -0,0 +1,129 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.support; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.sink.SinkProto; +import org.apache.avro.generic.GenericData; +import org.apache.kafka.connect.data.Decimal; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.apache.kafka.connect.data.Struct; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.Base64; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class DebeziumRowValueConverterTest +{ + @Test + void shouldEncodeCanonicalStructJsonAndAvroValues() throws Exception + { + TypeDescription typeDescription = TypeDescription.createSchemaFromStrings( + List.of("id", "name", "amount", "note", "empty_value"), + List.of("bigint", "varchar(64)", "decimal(10,2)", "varchar(64)", "varchar(64)")); + Schema rowSchema = SchemaBuilder.struct() + .field("id", Schema.INT64_SCHEMA) + .field("name", Schema.STRING_SCHEMA) + .field("amount", Decimal.builder(2) + .parameter("connect.decimal.precision", "10").build()) + .field("note", Schema.OPTIONAL_STRING_SCHEMA) + .field("empty_value", Schema.STRING_SCHEMA) + .build(); + Struct row = new Struct(rowSchema) + .put("id", 9223372036854775806L) + .put("name", "TDSQL value ") + .put("amount", new BigDecimal("24710.35")) + .put("note", null) + .put("empty_value", ""); + + SinkProto.RowValue.Builder builder = SinkProto.RowValue.newBuilder(); + new DebeziumRowValueConverter(typeDescription).parse(row, builder); + SinkProto.RowValue value = builder.build(); + + assertEquals(9223372036854775806L, + ByteBuffer.wrap(value.getValues(0).getValue().toByteArray()).getLong()); + assertEquals("TDSQL value ", value.getValues(1).getValue().toStringUtf8()); + assertEquals("24710.35", value.getValues(2).getValue().toStringUtf8()); + assertEquals(0, value.getValues(3).getValue().size()); + assertEquals("", value.getValues(4).getValue().toStringUtf8()); + + ObjectNode jsonRow = new ObjectMapper().createObjectNode(); + jsonRow.put("id", 9223372036854775806L); + jsonRow.put("name", "TDSQL value "); + jsonRow.put("amount", Base64.getEncoder().encodeToString( + new BigDecimal("24710.35").unscaledValue().toByteArray())); + jsonRow.putNull("note"); + jsonRow.put("empty_value", ""); + SinkProto.RowValue.Builder jsonBuilder = SinkProto.RowValue.newBuilder(); + new DebeziumRowValueConverter(typeDescription).parse(jsonRow, jsonBuilder); + + org.apache.avro.Schema avroSchema = new org.apache.avro.Schema.Parser().parse(""" + { + "type": "record", + "name": "CanonicalRow", + "fields": [ + {"name": "id", "type": "long"}, + {"name": "name", "type": "string"}, + {"name": "amount", "type": "bytes"}, + {"name": "note", "type": ["null", "string"], "default": null}, + {"name": "empty_value", "type": "string"} + ] + } + """); + GenericData.Record avroRow = new GenericData.Record(avroSchema); + avroRow.put("id", 9223372036854775806L); + avroRow.put("name", "TDSQL value "); + avroRow.put("amount", new BigDecimal("24710.35")); + avroRow.put("note", null); + avroRow.put("empty_value", ""); + SinkProto.RowValue.Builder avroBuilder = SinkProto.RowValue.newBuilder(); + new DebeziumRowValueConverter(typeDescription).parse(avroRow, avroBuilder); + + assertEquals(value, jsonBuilder.build()); + assertEquals(value, avroBuilder.build()); + } + + @Test + void shouldPreserveExistingIntegerWidths() throws Exception + { + TypeDescription typeDescription = TypeDescription.createSchemaFromStrings( + List.of("tiny", "small", "integer"), + List.of("tinyint", "smallint", "integer")); + Schema rowSchema = SchemaBuilder.struct() + .field("tiny", Schema.INT8_SCHEMA) + .field("small", Schema.INT16_SCHEMA) + .field("integer", Schema.INT32_SCHEMA) + .build(); + Struct row = new Struct(rowSchema) + .put("tiny", (byte) 7) + .put("small", (short) 1024) + .put("integer", 65536); + + SinkProto.RowValue.Builder builder = SinkProto.RowValue.newBuilder(); + new DebeziumRowValueConverter(typeDescription).parse(row, builder); + + assertEquals(1, builder.getValues(0).getValue().size()); + assertEquals(2, builder.getValues(1).getValue().size()); + assertEquals(4, builder.getValues(2).getValue().size()); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverterTest.java new file mode 100644 index 0000000..60cade7 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverterTest.java @@ -0,0 +1,98 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.conversion.sinkproto; + +import io.pixelsdb.pixels.common.metadata.SchemaTableName; +import io.pixelsdb.pixels.common.metadata.domain.Column; +import io.pixelsdb.pixels.common.metadata.domain.Table; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class RowRecordConverterTest +{ + private static final SchemaTableName TABLE = + new SchemaTableName("storage_test", "records"); + private Map registry; + private TableMetadata previousMetadata; + + @BeforeEach + @SuppressWarnings("unchecked") + void setUp() throws Exception + { + Field field = TableMetadataRegistry.class.getDeclaredField("registry"); + field.setAccessible(true); + registry = (Map) + field.get(TableMetadataRegistry.Instance()); + previousMetadata = registry.put(TABLE, metadata()); + } + + @AfterEach + void tearDown() + { + if (previousMetadata == null) + { + registry.remove(TABLE); + } else + { + registry.put(TABLE, previousMetadata); + } + } + + @Test + void shouldConvertCanonicalRowRecordIntoRuntimeEvent() + throws Exception + { + SinkProto.RowRecord record = SinkProto.RowRecord.newBuilder() + .setOp(SinkProto.OperationType.INSERT) + .setSource(SinkProto.SourceInfo.newBuilder() + .setDb(TABLE.getSchemaName()) + .setTable(TABLE.getTableName())) + .setAfter(SinkProto.RowValue.newBuilder() + .addValues(SinkProto.ColumnValue.newBuilder() + .setValue(com.google.protobuf.ByteString.EMPTY)) + .build()) + .build(); + + RowChangeEvent event = + new RowRecordConverter(TableMetadataRegistry.Instance()).convert(record); + + assertEquals(record, event.getRowRecord()); + assertEquals(TABLE.getSchemaName(), event.getDb()); + assertEquals(TABLE.getTableName(), event.getTable()); + } + + private TableMetadata metadata() throws Exception + { + Table table = new Table(); + table.setId(1); + table.setName(TABLE.getTableName()); + Column column = new Column(); + column.setName("id"); + column.setType("int"); + return new TableMetadata(table, null, List.of(column)); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/event/RowBatchTest.java b/src/test/java/io/pixelsdb/pixels/sink/event/RowBatchTest.java new file mode 100644 index 0000000..d86cc55 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/event/RowBatchTest.java @@ -0,0 +1,70 @@ +/* + * Copyright 2025 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.event; + +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.core.vector.BinaryColumnVector; +import io.pixelsdb.pixels.core.vector.IntColumnVector; +import io.pixelsdb.pixels.core.vector.VectorizedRowBatch; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class RowBatchTest +{ + @Test + void integerRowBatchTest() + { + TypeDescription schema = TypeDescription.createSchemaFromStrings( + java.util.List.of("id"), java.util.List.of("int")); + VectorizedRowBatch rowBatch = schema.createRowBatch( + 3, TypeDescription.Mode.CREATE_INT_VECTOR_FOR_INT); + IntColumnVector vector = (IntColumnVector) rowBatch.cols[0]; + vector.add(10); + vector.add(20); + vector.add(30); + rowBatch.size = 3; + + VectorizedRowBatch newRowBatch = VectorizedRowBatch.deserialize(rowBatch.serialize()); + + assertEquals(3, newRowBatch.size); + IntColumnVector newVector = (IntColumnVector) newRowBatch.cols[0]; + assertEquals(10, newVector.vector[0]); + assertEquals(20, newVector.vector[1]); + assertEquals(30, newVector.vector[2]); + } + + @Test + void varcharRowBatchTest() + { + TypeDescription schema = TypeDescription.createSchemaFromStrings( + java.util.List.of("name"), java.util.List.of("varchar(100)")); + VectorizedRowBatch rowBatch = schema.createRowBatch(3, TypeDescription.Mode.CREATE_INT_VECTOR_FOR_INT); + BinaryColumnVector v = (BinaryColumnVector) rowBatch.cols[0]; + v.add("rr"); + v.add("zz"); + v.add("rr"); + rowBatch.size = 3; + + VectorizedRowBatch newRowBatch = VectorizedRowBatch.deserialize(rowBatch.serialize()); + + assertEquals(3, newRowBatch.size); + BinaryColumnVector newVector = (BinaryColumnVector) newRowBatch.cols[0]; + assertEquals("rr", new String(newVector.vector[0], newVector.start[0], newVector.lens[0])); + assertEquals("zz", new String(newVector.vector[1], newVector.start[1], newVector.lens[1])); + assertEquals("rr", new String(newVector.vector[2], newVector.start[2], newVector.lens[2])); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactoryTest.java b/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactoryTest.java new file mode 100644 index 0000000..bcc9d04 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactoryTest.java @@ -0,0 +1,111 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.event; + +import com.google.protobuf.ByteString; +import io.pixelsdb.pixels.common.metadata.SchemaTableName; +import io.pixelsdb.pixels.common.metadata.domain.Column; +import io.pixelsdb.pixels.common.metadata.domain.Table; +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class RowChangeEventFactoryTest +{ + private static final SchemaTableName TABLE = + new SchemaTableName("factory_test", "records"); + private Map registry; + private TableMetadata previousMetadata; + + @BeforeEach + @SuppressWarnings("unchecked") + void setUp() throws Exception + { + Field field = TableMetadataRegistry.class.getDeclaredField("registry"); + field.setAccessible(true); + registry = (Map) + field.get(TableMetadataRegistry.Instance()); + previousMetadata = registry.put(TABLE, metadata()); + } + + @AfterEach + void tearDown() + { + if (previousMetadata == null) + { + registry.remove(TABLE); + } else + { + registry.put(TABLE, previousMetadata); + } + } + + @Test + void shouldBuildCanonicalRowChangeEvent() + throws Exception + { + TypeDescription schema = TypeDescription.createSchemaFromStrings( + List.of("id"), List.of("int")); + SinkProto.SourceInfo source = SinkProto.SourceInfo.newBuilder() + .setDb(TABLE.getSchemaName()) + .setTable(TABLE.getTableName()) + .build(); + SinkProto.RowValue after = SinkProto.RowValue.newBuilder() + .addValues(SinkProto.ColumnValue.newBuilder() + .setValue(ByteString.copyFromUtf8("1"))) + .build(); + + RowChangeEvent event = RowChangeEventFactory.create( + SinkProto.OperationType.INSERT, + source, + SinkProto.TransactionInfo.newBuilder().setId("tx-1").build(), + null, + after, + schema); + + assertEquals(SinkProto.OperationType.INSERT, event.getOp()); + assertEquals(TABLE.getSchemaName(), event.getDb()); + assertEquals(TABLE.getTableName(), event.getTable()); + assertEquals("tx-1", event.getTransaction().getId()); + assertEquals(after, event.getAfter()); + } + + private TableMetadata metadata() throws Exception + { + Table table = new Table(); + table.setId(1); + table.setName(TABLE.getTableName()); + return new TableMetadata(table, null, List.of(column("id", "int"))); + } + + private Column column(String name, String type) + { + Column column = new Column(); + column.setName(name); + column.setType(type); + return column; + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventTest.java b/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventTest.java index 139be39..d26cba2 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventTest.java @@ -1,58 +1,45 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.event; import com.google.protobuf.ByteString; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.IOException; import java.nio.ByteBuffer; -public class RowChangeEventTest -{ - private static Logger LOGGER = LoggerFactory.getLogger(RowChangeEventTest.class); - - @BeforeAll - public static void init() throws IOException - { - PixelsSinkConfigFactory.initialize("/home/ubuntu/pixels-sink/conf/pixels-sink.aws.properties"); - } - +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +class RowChangeEventTest +{ @Test - public void testSameHash() + void shouldReturnStableNonNegativeBucket() { + ByteString indexKey = getIndexKey(0); + int expectedBucket = RowChangeEvent.getBucketIdFromByteBuffer(indexKey); + for (int i = 0; i < 10; ++i) { - ByteString indexKey = getIndexKey(0); int bucket = RowChangeEvent.getBucketIdFromByteBuffer(indexKey); - LOGGER.info("Bucket: {}", bucket); + assertEquals(expectedBucket, bucket); } + assertTrue(expectedBucket >= 0); } - private ByteString getIndexKey(int key) + private static ByteString getIndexKey(int key) { int keySize = Integer.BYTES; ByteBuffer byteBuffer = ByteBuffer.allocate(keySize); diff --git a/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/RowBatchTest.java b/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/RowBatchTest.java deleted file mode 100644 index 0c36b72..0000000 --- a/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/RowBatchTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - -import io.pixelsdb.pixels.core.TypeDescription; -import io.pixelsdb.pixels.core.vector.BinaryColumnVector; -import io.pixelsdb.pixels.core.vector.VectorizedRowBatch; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.List; - -public class RowBatchTest -{ - private final List columnNames = new ArrayList<>(); - private final List columnTypes = new ArrayList<>(); - - @Test - public void integerRowBatchTest() - { - columnNames.add("id"); - columnTypes.add("int"); - - TypeDescription schema = TypeDescription.createSchemaFromStrings(columnNames, columnTypes); - VectorizedRowBatch rowBatch = schema.createRowBatch(3, TypeDescription.Mode.CREATE_INT_VECTOR_FOR_INT); - VectorizedRowBatch newRowBatch = VectorizedRowBatch.deserialize(rowBatch.serialize()); - - } - - @Test - public void varcharRowBatchTest() - { - columnNames.add("name"); - columnTypes.add("varchar(100)"); - - TypeDescription schema = TypeDescription.createSchemaFromStrings(columnNames, columnTypes); - VectorizedRowBatch rowBatch = schema.createRowBatch(3, TypeDescription.Mode.CREATE_INT_VECTOR_FOR_INT); - BinaryColumnVector v = (BinaryColumnVector) rowBatch.cols[0]; - v.add("rr"); - v.add("zz"); - v.add("rr"); - VectorizedRowBatch newRowBatch = VectorizedRowBatch.deserialize(rowBatch.serialize()); - - } -} diff --git a/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventDeserializerTest.java b/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventDeserializerTest.java deleted file mode 100644 index 783bbcb..0000000 --- a/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventDeserializerTest.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import org.apache.kafka.common.serialization.Deserializer; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Objects; - -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class RowChangeEventDeserializerTest -{ - - private final Deserializer deserializer = new RowChangeEventJsonDeserializer(); - - private String loadSchemaFromFile(String filename) throws IOException, URISyntaxException - { - ClassLoader classLoader = getClass().getClassLoader(); - return new String(Files.readAllBytes(Paths.get( - Objects.requireNonNull(classLoader.getResource(filename)).toURI() - ))); - } - - // @ParameterizedTest -// @EnumSource(value = OperationType.class, names = {"INSERT", "UPDATE"}) -// //, , "SNAPSHOT" -// void shouldParseValidOperations(OperationType opType) throws Exception { -// String jsonData = loadSchemaFromFile("records/" + opType.name().toLowerCase() + ".json"); -// RowChangeEvent event = deserializer.deserialize("test_topic", jsonData.getBytes()); -// -// assertNotNull(event); -// assertEquals(opType, event.getOp()); -// assertEquals("region", event.getTable()); -// -// Map data = opType == OperationType.DELETE ? -// event.getBeforeData() : event.getAfterData(); -// assertNotNull(data); -// } - - @Test - void shouldHandleDeleteOperation() throws Exception - { - String jsonData = loadSchemaFromFile("records/delete.json"); - RowChangeEvent event = deserializer.deserialize("test_topic", jsonData.getBytes()); - - assertTrue(event.isDelete()); -// assertNotNull(event.getBeforeData()); -// assertNull(event.getAfterData()); - } - - - @Test - void shouldHandleEmptyData() - { - RowChangeEvent event = deserializer.deserialize("empty_topic", new byte[0]); - assertNull(event); - } -} - diff --git a/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/RowDataParserTest.java b/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/RowDataParserTest.java deleted file mode 100644 index 0eee54b..0000000 --- a/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/RowDataParserTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - -import io.pixelsdb.pixels.sink.util.DateUtil; -import org.junit.jupiter.api.Test; - -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.util.Date; - -public class RowDataParserTest -{ - // BqQ= 17 - // JbR7 24710.35 -// @ParameterizedTest -// @CsvSource({ -// // encodedValue, expectedValue, precision, scale -// "BqQ=, 17.00, 15, 2", -// "JbR7, 24710.35, 15, 2", -// }) -// void testParseDecimalValid(String encodedValue, String expectedValue, int precision, int scale) { -// JsonNode node = new TextNode(encodedValue); -// TypeDescription type = TypeDescription.createDecimal(precision, scale); -// RowDataParser rowDataParser = new RowDataParser(type); -// BigDecimal result = rowDataParser.parseDecimal(node, type); -// assertEquals(new BigDecimal(expectedValue), result); -// } - - @Test - void testParseDate() - { - int day = 17059; - Date debeziumDate = DateUtil.fromDebeziumDate(day); - String dayString = DateUtil.convertDateToDayString(debeziumDate); - long ts = 1473927308302000L; - LocalDateTime localDateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts / 1000), ZoneOffset.UTC); - ZonedDateTime zonedDateTime = Instant.ofEpochMilli(ts).atZone(ZoneOffset.UTC); - boolean pause = true; - } -} diff --git a/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/SchemaDeserializerTest.java b/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/SchemaDeserializerTest.java deleted file mode 100644 index bdd14e4..0000000 --- a/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/SchemaDeserializerTest.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.pixelsdb.pixels.core.TypeDescription; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Objects; - -public class SchemaDeserializerTest -{ - private static final ObjectMapper objectMapper = new ObjectMapper(); - - @BeforeEach - public void setUp() throws IOException, URISyntaxException - { - // String schemaStr = schemaNode.toString(); - - } - - private String loadSchemaFromFile(String filename) throws IOException, URISyntaxException - { - ClassLoader classLoader = getClass().getClassLoader(); - return new String(Files.readAllBytes(Paths.get(Objects.requireNonNull(classLoader.getResource(filename)).toURI()))); - } - - @Test - public void testParseNationStruct() throws IOException, URISyntaxException - { - String jsonSchema = loadSchemaFromFile("records/nation.json"); - JsonNode rootNode = objectMapper.readTree(jsonSchema); - JsonNode schemaNode = rootNode.get("schema"); - - TypeDescription typeDesc = SchemaDeserializer.parseFromBeforeOrAfter(schemaNode, "after"); - - Assertions.assertEquals(TypeDescription.Category.STRUCT, typeDesc.getCategory()); - Assertions.assertEquals(4, typeDesc.getChildren().size()); - Assertions.assertEquals("n_nationkey", typeDesc.getFieldNames().get(0)); - Assertions.assertEquals(TypeDescription.Category.INT, typeDesc.getChildren().get(0).getCategory()); - Assertions.assertEquals("n_name", typeDesc.getFieldNames().get(1)); - Assertions.assertEquals(TypeDescription.Category.STRING, typeDesc.getChildren().get(1).getCategory()); - Assertions.assertEquals("n_regionkey", typeDesc.getFieldNames().get(2)); - Assertions.assertEquals(TypeDescription.Category.INT, typeDesc.getChildren().get(2).getCategory()); - Assertions.assertEquals("n_comment", typeDesc.getFieldNames().get(3)); - Assertions.assertEquals(TypeDescription.Category.STRING, typeDesc.getChildren().get(3).getCategory()); - } - - @Test - public void testParseDecimalType() throws IOException - { - String jsonSchema = "[{" - + "\"type\": \"bytes\"," - + "\"name\": \"org.apache.kafka.connect.data.Decimal\"," - + "\"parameters\": {" - + " \"scale\": \"2\"," - + " \"connect.decimal.precision\": \"15\"" - + "}," - + "\"field\": \"price\"" - + "}]"; - - JsonNode rootNode = objectMapper.readTree(jsonSchema); - - TypeDescription typeDesc = SchemaDeserializer.parseStruct(rootNode); - TypeDescription filedType = typeDesc.getChildren().get(0); - Assertions.assertEquals(TypeDescription.Category.DECIMAL, filedType.getCategory()); - Assertions.assertEquals(15, filedType.getPrecision()); - Assertions.assertEquals(2, filedType.getScale()); - } - - @Test - public void testParseDateType() throws IOException - { - - String jsonSchema = "[{" - + "\"type\": \"int32\"," - + "\"name\": \"io.debezium.time.Date\"," - + "\"field\": \"created_at\"" - + "}]"; - JsonNode rootNode = objectMapper.readTree(jsonSchema); - TypeDescription typeDesc = SchemaDeserializer.parseStruct(rootNode); - TypeDescription filedType = typeDesc.getChildren().get(0); - Assertions.assertEquals(TypeDescription.Category.DATE, filedType.getCategory()); - } - - // 测试未知类型异常 - @Test - public void testParseInvalidType() throws IOException - { - String jsonSchema = "[{" - + "\"type\": \"unknown_type\"," - + "\"field\": \"invalid_field\"" - + "}]"; - JsonNode rootNode = objectMapper.readTree(jsonSchema); - Assertions.assertThrows( - IllegalArgumentException.class, () -> - { - SchemaDeserializer.parseStruct(rootNode); - } - ); - } - - @Test - public void testMissingRequiredField() throws IOException - { - String jsonSchema = "{" - + "\"type\": \"struct\"," - + "\"fields\": [" - + " {\"field\": \"id\"}" // lack type - + "]" - + "}"; - JsonNode rootNode = objectMapper.readTree(jsonSchema); - Assertions.assertThrows( - IllegalArgumentException.class, () -> - { - SchemaDeserializer.parseFieldType(rootNode); - } - ); - } -} \ No newline at end of file diff --git a/src/test/java/io/pixelsdb/pixels/sink/freshness/TestFreshnessClient.java b/src/test/java/io/pixelsdb/pixels/sink/freshness/FreshnessClientTest.java similarity index 57% rename from src/test/java/io/pixelsdb/pixels/sink/freshness/TestFreshnessClient.java rename to src/test/java/io/pixelsdb/pixels/sink/freshness/FreshnessClientTest.java index a524b0a..3d35472 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/freshness/TestFreshnessClient.java +++ b/src/test/java/io/pixelsdb/pixels/sink/freshness/FreshnessClientTest.java @@ -1,41 +1,42 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.freshness; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.TestConfig; import io.pixelsdb.pixels.sink.util.MetricsFacade; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import static org.junit.jupiter.api.Assertions.assertTrue; + // We extend FreshnessClient to access the protected queryAndCalculateFreshness method -public class TestFreshnessClient +@Tag("integration") +class FreshnessClientTest { // Mocks for JDBC dependencies @@ -49,44 +50,53 @@ public class TestFreshnessClient private FreshnessClient client; // The instance of the client to test @BeforeAll - public static void setUp() throws IOException + static void setUp() throws Exception { - // Initialization as per the user's template - PixelsSinkConfigFactory.initialize("/home/ubuntu/disk1/opt/pixels-sink/conf/pixels-sink.hudi.properties"); + TestConfig.initializeIntegrationConfig(); } @Test - public void testFreshnessCalculationSuccess() throws Exception + void testFreshnessCalculationSuccess() throws Exception { FreshnessClient freshnessClient = FreshnessClient.getInstance(); freshnessClient.addMonitoredTable("customer"); - freshnessClient.start(); - while (true) + try + { + freshnessClient.start(); + Thread.sleep(1000); + } finally { + freshnessClient.stop(); } } @Test - public void testSnapshotTs() throws SQLException + void testSnapshotTs() throws SQLException { FreshnessClient freshnessClient = FreshnessClient.getInstance(); - Connection connection = freshnessClient.createNewConnection(123456L); - String query = String.format("SELECT max(freshness_ts) FROM company"); - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(query); - resultSet.next(); + try (Connection connection = freshnessClient.createNewConnection(123456L); + Statement statement = connection.createStatement()) + { + String query = "SELECT max(freshness_ts) FROM company"; + try (ResultSet resultSet = statement.executeQuery(query)) + { + assertTrue(resultSet.next()); + } + } } @Test - public void testLoanTransQueryPerformance() throws SQLException + void testLoanTransQueryPerformance(@org.junit.jupiter.api.io.TempDir Path tempDir) + throws SQLException { FreshnessClient freshnessClient = FreshnessClient.getInstance(); Connection connection = freshnessClient.createNewConnection(12345689100L); String query = "SELECT max(freshness_ts) FROM nation"; - String csvFileName = "loantrans_query_results.csv"; - int iterations = 1000; - try (PrintWriter writer = new PrintWriter(new FileWriter(csvFileName))) + Path csvFile = tempDir.resolve("loantrans_query_results.csv"); + int iterations = Integer.getInteger("pixels.sink.test.iterations", 10); + try (PrintWriter writer = new PrintWriter( + Files.newBufferedWriter(csvFile, StandardCharsets.UTF_8))) { for (int i = 0; i < iterations; i++) @@ -113,7 +123,7 @@ public void testLoanTransQueryPerformance() throws SQLException writer.printf("%d,%d,%d%n", startTime, maxFreshnessTs, durationMs); writer.flush(); } - System.out.println("Test completed. Results saved to: " + csvFileName); + System.out.println("Test completed. Results saved to: " + csvFile); } catch (IOException e) { e.printStackTrace(); diff --git a/src/test/java/io/pixelsdb/pixels/sink/metadata/TestIndexService.java b/src/test/java/io/pixelsdb/pixels/sink/metadata/IndexServiceTest.java similarity index 93% rename from src/test/java/io/pixelsdb/pixels/sink/metadata/TestIndexService.java rename to src/test/java/io/pixelsdb/pixels/sink/metadata/IndexServiceTest.java index 0804b7f..70a9067 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/metadata/TestIndexService.java +++ b/src/test/java/io/pixelsdb/pixels/sink/metadata/IndexServiceTest.java @@ -12,10 +12,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ - - package io.pixelsdb.pixels.sink.metadata; import com.google.protobuf.ByteString; @@ -23,6 +20,7 @@ import io.pixelsdb.pixels.common.exception.MetadataException; import io.pixelsdb.pixels.common.index.IndexOption; import io.pixelsdb.pixels.common.index.service.IndexService; +import io.pixelsdb.pixels.common.index.service.IndexServiceProvider; import io.pixelsdb.pixels.common.metadata.MetadataService; import io.pixelsdb.pixels.common.metadata.domain.Layout; import io.pixelsdb.pixels.common.metadata.domain.SinglePointIndex; @@ -30,21 +28,18 @@ import io.pixelsdb.pixels.daemon.MetadataProto; import io.pixelsdb.pixels.index.IndexProto; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.nio.ByteBuffer; -/** - * @package: io.pixelsdb.pixels.sink.metadata - * @className: TestIndexService - * @author: AntiO2 - * @date: 2025/8/5 04:34 - */ -public class TestIndexService +@Tag("integration") +class IndexServiceTest { private final MetadataService metadataService = MetadataService.Instance(); - private final IndexService indexService = null; // TODO + private final IndexService indexService = + IndexServiceProvider.getService(IndexServiceProvider.ServiceMode.rpc); @Test public void testCreateFreshnessIndex() throws MetadataException @@ -67,7 +62,6 @@ public void testCreateFreshnessIndex() throws MetadataException SinglePointIndex index = new SinglePointIndex(singlePointIndexbuilder.build()); boolean result = metadataService.createSinglePointIndex(index); Assertions.assertTrue(result); - boolean pause = true; } @Test @@ -93,7 +87,6 @@ public void testCreateIndex() throws MetadataException SinglePointIndex index = new SinglePointIndex(singlePointIndexbuilder.build()); boolean result = metadataService.createSinglePointIndex(index); Assertions.assertTrue(result); - boolean pause = true; } @Test @@ -106,7 +99,6 @@ public void testGetIndex() throws MetadataException SinglePointIndex index = metadataService.getPrimaryIndex(id); Assertions.assertNotNull(index); - boolean pause = true; } @Test @@ -115,7 +107,6 @@ public void testGetRowID() throws MetadataException, IndexException int numRowIds = 10000; IndexProto.RowIdBatch rowIdBatch = indexService.allocateRowIdBatch(4, numRowIds); Assertions.assertEquals(rowIdBatch.getLength(), numRowIds); - boolean pause = true; } @Test @@ -164,6 +155,5 @@ public void testPutAndDelete() throws MetadataException, IndexException IndexProto.RowLocation rowLocation = indexService.deletePrimaryIndexEntry(builder.getIndexKey(), indexOption); - boolean pause = false; } } diff --git a/src/test/java/io/pixelsdb/pixels/sink/pipeline/PipelineTest.java b/src/test/java/io/pixelsdb/pixels/sink/pipeline/PipelineTest.java new file mode 100644 index 0000000..50ee917 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/pipeline/PipelineTest.java @@ -0,0 +1,74 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.pipeline; + +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.TestConfig; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + +class PipelineTest +{ + @Test + void shouldStartAndCloseTablePipeline() throws Exception + { + TestConfig.initializeUnitConfig(); + + assertDoesNotThrow(() -> { + try (TablePipeline pipeline = new TablePipeline()) + { + pipeline.start(); + } + }); + } + + @Test + void shouldStartTransactionPipelineOnlyOnce() throws Exception + { + TestConfig.initializeUnitConfig(); + + assertDoesNotThrow(() -> { + try (TransactionPipeline pipeline = new TransactionPipeline()) + { + pipeline.start(); + pipeline.start(); + pipeline.publish(SinkProto.TransactionMetadata.newBuilder() + .setId("transaction-1") + .setStatus(SinkProto.TransactionStatus.END) + .build()); + } + }); + } + + @Test + void shouldAbortPipelines() throws Exception + { + TestConfig.initializeUnitConfig(); + + assertDoesNotThrow(() -> { + try (TablePipeline tablePipeline = new TablePipeline(); + TransactionPipeline transactionPipeline = new TransactionPipeline()) + { + tablePipeline.start(); + transactionPipeline.start(); + tablePipeline.abort(); + transactionPipeline.abort(); + } + }); + } + +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/processor/TableProcessorTest.java b/src/test/java/io/pixelsdb/pixels/sink/processor/TableProcessorTest.java new file mode 100644 index 0000000..9bcba2c --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/processor/TableProcessorTest.java @@ -0,0 +1,110 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.processor; + +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.TestConfig; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.util.BlockingBoundedQueue; +import io.pixelsdb.pixels.sink.writer.PixelsSinkWriter; +import io.pixelsdb.pixels.core.TypeDescription; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class TableProcessorTest +{ + @Test + void shouldForwardRowsToWriter() throws Exception + { + TestConfig.initializeUnitConfig(); + BlockingBoundedQueue queue = new BlockingBoundedQueue<>(2); + RecordingWriter writer = new RecordingWriter(); + TableProcessor processor = new TableProcessor(queue, writer); + RowChangeEvent first = rowEvent(); + RowChangeEvent second = rowEvent(); + + try + { + processor.run(); + queue.put(first); + queue.put(second); + queue.close(); + processor.awaitTermination(); + + assertTrue(writer.rowsWritten.await(1, TimeUnit.SECONDS)); + assertEquals(List.of(first, second), writer.rows); + } finally + { + processor.abort(); + queue.abort(); + processor.awaitTermination(); + } + } + + private static RowChangeEvent rowEvent() throws Exception + { + SinkProto.RowRecord record = SinkProto.RowRecord.newBuilder() + .setOp(SinkProto.OperationType.INSERT) + .setSource(SinkProto.SourceInfo.newBuilder() + .setDb("test_db") + .setTable("test_table")) + .setAfter(SinkProto.RowValue.newBuilder() + .addValues(SinkProto.ColumnValue.newBuilder() + .setValue(com.google.protobuf.ByteString.EMPTY))) + .build(); + TypeDescription schema = TypeDescription.createSchemaFromStrings( + List.of("id"), List.of("int")); + return new RowChangeEvent(record, schema, null); + } + + private static final class RecordingWriter implements PixelsSinkWriter + { + private final CountDownLatch rowsWritten = new CountDownLatch(2); + private final List rows = new CopyOnWriteArrayList<>(); + + @Override + public void flush() + { + } + + @Override + public boolean writeRow(RowChangeEvent rowChangeEvent) + { + rows.add(rowChangeEvent); + rowsWritten.countDown(); + return true; + } + + @Override + public boolean writeTrans(SinkProto.TransactionMetadata transactionMetadata) + { + return false; + } + + @Override + public void close() throws IOException + { + } + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/processor/TransactionProcessorTest.java b/src/test/java/io/pixelsdb/pixels/sink/processor/TransactionProcessorTest.java new file mode 100644 index 0000000..15c8767 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/processor/TransactionProcessorTest.java @@ -0,0 +1,100 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.processor; + +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.util.BlockingBoundedQueue; +import io.pixelsdb.pixels.sink.writer.PixelsSinkWriter; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class TransactionProcessorTest +{ + @Test + void shouldForwardTransactionsAndStop() throws Exception + { + BlockingBoundedQueue queue = + new BlockingBoundedQueue<>(2); + RecordingWriter writer = new RecordingWriter(); + TransactionProcessor processor = new TransactionProcessor(queue, writer); + Thread processorThread = new Thread(processor, "transaction-processor-test"); + SinkProto.TransactionMetadata first = SinkProto.TransactionMetadata.newBuilder() + .setId("transaction-1") + .build(); + SinkProto.TransactionMetadata second = SinkProto.TransactionMetadata.newBuilder() + .setId("transaction-2") + .build(); + + try + { + processorThread.start(); + queue.put(first); + queue.put(second); + queue.close(); + processorThread.join(1000); + + assertTrue(writer.transactionsWritten.await(1, TimeUnit.SECONDS)); + assertEquals(List.of(first, second), writer.transactions); + assertFalse(processorThread.isAlive()); + } finally + { + processor.abort(); + queue.abort(); + processorThread.interrupt(); + processorThread.join(1000); + } + } + + private static final class RecordingWriter implements PixelsSinkWriter + { + private final CountDownLatch transactionsWritten = new CountDownLatch(2); + private final List transactions = + new CopyOnWriteArrayList<>(); + + @Override + public void flush() + { + } + + @Override + public boolean writeRow(io.pixelsdb.pixels.sink.event.RowChangeEvent rowChangeEvent) + { + return false; + } + + @Override + public boolean writeTrans(SinkProto.TransactionMetadata transactionMetadata) + { + transactions.add(transactionMetadata); + transactionsWritten.countDown(); + return true; + } + + @Override + public void close() throws IOException + { + } + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifierTest.java b/src/test/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifierTest.java new file mode 100644 index 0000000..ac63342 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifierTest.java @@ -0,0 +1,179 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.source.engine; + +import io.pixelsdb.pixels.common.metadata.SchemaTableName; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.source.SourceRecord; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static io.pixelsdb.pixels.sink.source.engine.DebeziumRecordType.ROW; +import static io.pixelsdb.pixels.sink.source.engine.DebeziumRecordType.TOMBSTONE; +import static io.pixelsdb.pixels.sink.source.engine.DebeziumRecordType.TRANSACTION; +import static io.pixelsdb.pixels.sink.source.engine.DebeziumRecordType.UNKNOWN_CONTROL; +import static org.junit.jupiter.api.Assertions.assertEquals; + +class ConnectEventClassifierTest +{ + private static final String PREFIX = "mysql-cdc"; + private static final String ROW_TOPIC = PREFIX + ".cdc_verify.binlog_test"; + private static final String TRANSACTION_TOPIC = PREFIX + ".transaction"; + private static final Schema ROW_SCHEMA = SchemaBuilder.struct().optional() + .field("id", Schema.INT64_SCHEMA) + .field("name", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + private static final Schema SOURCE_SCHEMA = SchemaBuilder.struct() + .field("connector", Schema.STRING_SCHEMA) + .field("db", Schema.STRING_SCHEMA) + .field("table", Schema.OPTIONAL_STRING_SCHEMA) + .field("gtid", Schema.OPTIONAL_STRING_SCHEMA) + .field("file", Schema.STRING_SCHEMA) + .field("pos", Schema.INT64_SCHEMA) + .field("row", Schema.INT32_SCHEMA) + .build(); + private static final Schema TRANSACTION_INFO_SCHEMA = SchemaBuilder.struct().optional() + .field("id", Schema.STRING_SCHEMA) + .field("total_order", Schema.INT64_SCHEMA) + .field("data_collection_order", Schema.INT64_SCHEMA) + .build(); + + private final ConnectEventClassifier classifier = new ConnectEventClassifier(); + + @Test + void shouldClassifyMySqlRowOperations() + { + assertEquals(ROW, classify(rowRecord("c", null, row(1), "gtid:3", 1))); + assertEquals(ROW, classify(rowRecord("u", row(1), row(1), "gtid:3", 2))); + assertEquals(ROW, classify(rowRecord("d", row(1), null, "gtid:3", 3))); + assertEquals(ROW, classify(rowRecord("r", null, row(1), "", 0))); + } + + @Test + void shouldClassifyTransactionBoundaries() + { + assertEquals(TRANSACTION, classify(transactionRecord("BEGIN", "gtid:5"))); + assertEquals(TRANSACTION, classify(transactionRecord("END", "gtid:5"))); + assertEquals(UNKNOWN_CONTROL, classify(transactionRecord("COMMIT", "gtid:5"))); + } + + @Test + void shouldSkipControlEvents() + { + assertEquals(TOMBSTONE, classify(sourceRecord(ROW_TOPIC, null, null))); + assertEquals(UNKNOWN_CONTROL, classify(sourceRecord( + PREFIX + ".signal", SchemaBuilder.struct() + .field("type", Schema.STRING_SCHEMA).build(), + new Struct(SchemaBuilder.struct() + .field("type", Schema.STRING_SCHEMA).build()).put("type", "x")))); + } + + @Test + void shouldRejectMalformedRows() + { + assertEquals(UNKNOWN_CONTROL, + classify(rowRecord("u", null, row(1), "gtid:6", 1))); + assertEquals(UNKNOWN_CONTROL, + classify(rowRecord("d", null, null, "gtid:6", 1))); + } + + @Test + void shouldExtractTableStreamKey() + { + SchemaTableName table = classifier.tableOf(rowRecord("c", null, row(1), "gtid:1", 1)); + assertEquals("cdc_verify", table.getSchemaName()); + assertEquals("binlog_test", table.getTableName()); + } + + private DebeziumRecordType classify(SourceRecord record) + { + return classifier.classify(record, TRANSACTION_TOPIC); + } + + private SourceRecord rowRecord( + String op, Struct before, Struct after, String transactionId, long order) + { + Struct transaction = transactionId.isEmpty() ? null : new Struct(TRANSACTION_INFO_SCHEMA) + .put("id", transactionId) + .put("total_order", order) + .put("data_collection_order", order); + Struct source = new Struct(SOURCE_SCHEMA) + .put("connector", "mysql") + .put("db", "cdc_verify") + .put("table", "binlog_test") + .put("gtid", transactionId.isEmpty() ? null : transactionId) + .put("file", "binlog.000004") + .put("pos", 152L) + .put("row", (int) order); + Schema envelopeSchema = SchemaBuilder.struct() + .name("mysql-cdc.cdc_verify.binlog_test.Envelope") + .field("before", ROW_SCHEMA) + .field("after", ROW_SCHEMA) + .field("source", SOURCE_SCHEMA) + .field("transaction", TRANSACTION_INFO_SCHEMA) + .field("op", Schema.STRING_SCHEMA) + .build(); + Struct envelope = new Struct(envelopeSchema) + .put("before", before) + .put("after", after) + .put("source", source) + .put("transaction", transaction) + .put("op", op); + return sourceRecord(ROW_TOPIC, envelopeSchema, envelope); + } + + private SourceRecord transactionRecord(String status, String id) + { + Schema collectionSchema = SchemaBuilder.struct() + .field("data_collection", Schema.STRING_SCHEMA) + .field("event_count", Schema.INT64_SCHEMA) + .build(); + Schema schema = SchemaBuilder.struct() + .field("status", Schema.STRING_SCHEMA) + .field("id", Schema.STRING_SCHEMA) + .field("event_count", Schema.OPTIONAL_INT64_SCHEMA) + .field("data_collections", SchemaBuilder.array(collectionSchema).optional().build()) + .field("ts_ms", Schema.INT64_SCHEMA) + .build(); + Struct value = new Struct(schema) + .put("status", status) + .put("id", id) + .put("event_count", status.equals("END") ? 4L : null) + .put("data_collections", null) + .put("ts_ms", 1750000000000L); + return sourceRecord(TRANSACTION_TOPIC, schema, value); + } + + private SourceRecord sourceRecord(String topic, Schema schema, Object value) + { + return new SourceRecord( + Map.of("server", PREFIX), + Map.of("file", "binlog.000004", "pos", 152L), + topic, + null, + null, + schema, + value); + } + + private Struct row(long id) + { + return new Struct(ROW_SCHEMA).put("id", id).put("name", "row-" + id); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/source/kafka/AvroConsumerTest.java b/src/test/java/io/pixelsdb/pixels/sink/source/kafka/AvroConsumerTest.java new file mode 100644 index 0000000..fe5b7cf --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/source/kafka/AvroConsumerTest.java @@ -0,0 +1,128 @@ +/* + * Copyright 2025 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.source.kafka; + +import io.apicurio.registry.serde.SerdeConfig; +import io.pixelsdb.pixels.sink.TestConfig; +import io.pixelsdb.pixels.sink.config.KafkaValueFormat; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.source.kafka.serde.KafkaRecordConverter; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.Collections; +import java.util.Properties; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Tag("integration") +class AvroConsumerTest +{ + private static final int MAX_POLL_CYCLES = 100; + + @Test + void shouldConsumeRowChangeEvents() throws Exception + { + TestConfig.initializeIntegrationConfig(); + KafkaConsumer consumer = rowConsumer(); + consumer.subscribe(Collections.singletonList(topic())); + + try (KafkaRecordConverter converter = + KafkaRecordConverter.forRow(avroConverterProperties())) + { + int recordCount = 0; + for (int i = 0; i < MAX_POLL_CYCLES; ++i) + { + ConsumerRecords records = consumer.poll(Duration.ofMillis(100)); + for (ConsumerRecord record : records) + { + RowChangeEvent event = converter.convert(record.topic(), record.value()); + if (event != null) + { + recordCount++; + } + } + } + assertTrue(recordCount > 0, "No row-change records were consumed"); + } finally + { + consumer.close(); + } + } + + private static KafkaConsumer rowConsumer() + { + Properties props = new Properties(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers()); + props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId()); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + props.put(SerdeConfig.REGISTRY_URL, registryUrl()); + props.put(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true"); + props.put(SerdeConfig.CHECK_PERIOD_MS, "30000"); + return new KafkaConsumer<>(props); + } + + private static Properties avroConverterProperties() + { + Properties properties = new Properties(); + properties.put(PixelsSinkConstants.KAFKA_VALUE_FORMAT, KafkaValueFormat.AVRO); + properties.put(SerdeConfig.REGISTRY_URL, registryUrl()); + properties.put(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true"); + properties.put(SerdeConfig.CHECK_PERIOD_MS, "30000"); + return properties; + } + + private static String topic() + { + return requiredProperty("pixels.sink.test.topic"); + } + + private static String registryUrl() + { + return requiredProperty("pixels.sink.test.registry.url"); + } + + private static String bootstrapServers() + { + return requiredProperty("pixels.sink.test.bootstrap.servers"); + } + + private static String groupId() + { + return System.getProperty( + "pixels.sink.test.group.id", "pixels-sink-test-" + UUID.randomUUID()); + } + + private static String requiredProperty(String key) + { + String value = System.getProperty(key); + Assumptions.assumeTrue(value != null && !value.isBlank(), + "Set -D" + key + " to run this integration test"); + return value; + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverterTest.java new file mode 100644 index 0000000..8793f3b --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverterTest.java @@ -0,0 +1,97 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.source.kafka.serde; + +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class KafkaRecordConverterTest +{ + @Test + void shouldCreateJsonRowAndTransactionConverters() + { + Properties properties = jsonProperties(); + try (KafkaRecordConverter row = KafkaRecordConverter.forRow(properties); + KafkaRecordConverter tx = + KafkaRecordConverter.forTransaction(properties)) + { + assertNotNull(row); + assertNotNull(tx); + } + } + + @Test + void shouldReturnNullForEmptyOrNullRowPayload() + { + try (KafkaRecordConverter converter = + KafkaRecordConverter.forRow(jsonProperties())) + { + assertNull(converter.convert("topic", null)); + assertNull(converter.convert("topic", new byte[0])); + } + } + + @Test + void shouldSwallowInvalidRowPayload() + { + try (KafkaRecordConverter converter = + KafkaRecordConverter.forRow(jsonProperties())) + { + assertNull(converter.convert("topic", "not-json".getBytes(StandardCharsets.UTF_8))); + } + } + + @Test + void shouldPropagateInvalidTransactionPayload() + { + try (KafkaRecordConverter converter = + KafkaRecordConverter.forTransaction(jsonProperties())) + { + assertThrows(RuntimeException.class, + () -> converter.convert("topic", "not-json".getBytes(StandardCharsets.UTF_8))); + } + } + + @Test + void shouldRejectTransactionConverterWithoutDialect() + { + Properties properties = new Properties(); + properties.put(PixelsSinkConstants.KAFKA_VALUE_FORMAT, "json"); + + IllegalStateException error = assertThrows( + IllegalStateException.class, + () -> KafkaRecordConverter.forTransaction(properties)); + assertTrue(error.getMessage().contains(PixelsSinkConstants.SINK_DEBEZIUM_DIALECT)); + } + + private static Properties jsonProperties() + { + Properties properties = new Properties(); + properties.put(PixelsSinkConstants.KAFKA_VALUE_FORMAT, "json"); + properties.put(PixelsSinkConstants.SINK_DEBEZIUM_DIALECT, "postgresql"); + return properties; + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueueTest.java b/src/test/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueueTest.java new file mode 100644 index 0000000..d8cb81e --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueueTest.java @@ -0,0 +1,92 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.util; + +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class BlockingBoundedQueueTest +{ + @Test + void shouldRejectNonPositiveCapacity() + { + assertThrows(IllegalArgumentException.class, () -> new BlockingBoundedQueue<>(0)); + assertThrows(IllegalArgumentException.class, () -> new BlockingBoundedQueue<>(-1)); + } + + @Test + void shouldPutAndTakeValue() + { + BlockingBoundedQueue queue = new BlockingBoundedQueue<>(1); + queue.put(7); + + assertEquals(7, queue.take()); + queue.close(); + } + + @Test + void shouldWakeConsumerWhenClosed() + throws Exception + { + BlockingBoundedQueue queue = new BlockingBoundedQueue<>(1); + CountDownLatch finished = new CountDownLatch(1); + Thread consumer = new Thread(() -> + { + assertNull(queue.take()); + finished.countDown(); + }); + consumer.start(); + + queue.close(); + + assertTrue(finished.await(1, TimeUnit.SECONDS)); + consumer.join(); + } + + @Test + void shouldDiscardPendingValuesWhenAborted() + { + BlockingBoundedQueue queue = new BlockingBoundedQueue<>(1); + queue.put(7); + queue.put(null); + + queue.abort(); + + assertNull(queue.take()); + queue.close(); + } + + @Test + void shouldDrainPendingValuesWhenClosed() + { + try (BlockingBoundedQueue queue = new BlockingBoundedQueue<>(2)) + { + queue.put(7); + + queue.close(); + + assertEquals(7, queue.take()); + assertNull(queue.take()); + } + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistryTest.java b/src/test/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistryTest.java index aee4a0e..e6eb2a1 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistryTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistryTest.java @@ -12,18 +12,21 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ - - package io.pixelsdb.pixels.sink.util; +import io.pixelsdb.pixels.common.utils.EtcdUtil; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.nio.file.Path; import java.util.List; +import java.util.UUID; + +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; /** * @package: io.pixelsdb.pixels.sink.util @@ -31,24 +34,28 @@ * @author: AntiO2 * @date: 2025/10/5 08:54 */ -public class EtcdFileRegistryTest +@Tag("integration") +class EtcdFileRegistryTest { - private static final Logger LOGGER = LoggerFactory.getLogger(EtcdFileRegistryTest.class); - @Test - public void testCreateFile() + void shouldCreateAndListCompletedFiles(@TempDir Path tempDir) { - EtcdFileRegistry etcdFileRegistry = new EtcdFileRegistry("test", "file:///tmp/test/ray"); - for (int i = 0; i < 10; i++) + String topic = "test-" + UUID.randomUUID(); + try { - String newFile = etcdFileRegistry.createNewFile(); - etcdFileRegistry.markFileCompleted(newFile); - } - List files = etcdFileRegistry.listAllFiles(); - for (String file : files) + EtcdFileRegistry etcdFileRegistry = new EtcdFileRegistry( + topic, tempDir.resolve("ray").toUri().toString()); + for (int i = 0; i < 10; i++) + { + String newFile = etcdFileRegistry.createNewFile(); + etcdFileRegistry.markFileCompleted(newFile); + } + List files = etcdFileRegistry.listAllFiles(); + + assertEquals(10, files.size()); + } finally { - LOGGER.info(file); + EtcdUtil.Instance().deleteByPrefix("/sink/proto/registry/" + topic); } - etcdFileRegistry.cleanData(); } } diff --git a/src/test/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoderTest.java b/src/test/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoderTest.java new file mode 100644 index 0000000..724b2af --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoderTest.java @@ -0,0 +1,84 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.util.concurrent; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +class OrderedBatchDecoderTest +{ + private ExecutorService executor; + + @AfterEach + void tearDown() + { + if (executor != null) + { + DecodeExecutors.shutdownGracefully(executor, 5, null); + } + } + + @Test + void shouldPreserveInputOrder() throws Exception + { + executor = DecodeExecutors.newFixedCallerRuns(4, "ordered-batch-test"); + List input = List.of(3, 1, 2); + List decoded = OrderedBatchDecoder.decodeInOrder( + executor, input, value -> + { + try + { + Thread.sleep(5L * value); + } catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + return value * 10; + }); + assertEquals(List.of(30, 10, 20), decoded); + } + + @Test + void shouldReplaceFailedRecordsWithNull() throws Exception + { + executor = DecodeExecutors.newFixedCallerRuns(2, "ordered-batch-fail"); + AtomicInteger calls = new AtomicInteger(); + List decoded = OrderedBatchDecoder.decodeInOrder( + executor, + List.of(1, 2, 3), + value -> + { + calls.incrementAndGet(); + if (value == 2) + { + throw new IllegalStateException("boom"); + } + return value; + }); + assertEquals(3, calls.get()); + assertEquals(1, decoded.get(0)); + assertNull(decoded.get(1)); + assertEquals(3, decoded.get(2)); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoderTest.java b/src/test/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoderTest.java new file mode 100644 index 0000000..d53e763 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoderTest.java @@ -0,0 +1,141 @@ +/* + * Copyright 2026 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.util.concurrent; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class StreamOrderedDecoderTest +{ + @Test + void shouldPublishSameStreamInSubmitOrder() throws Exception + { + try (StreamOrderedDecoder decoder = + new StreamOrderedDecoder(4, "stream-order-test")) + { + decoder.start(); + List published = Collections.synchronizedList(new ArrayList<>()); + CountDownLatch slowStarted = new CountDownLatch(1); + CountDownLatch releaseSlow = new CountDownLatch(1); + + CompletableFuture first = decoder.submit( + "s1", + 1, + value -> + { + slowStarted.countDown(); + assertTrue(releaseSlow.await(5, TimeUnit.SECONDS)); + return value; + }, + result -> published.add(result.value())); + assertTrue(slowStarted.await(5, TimeUnit.SECONDS)); + + CompletableFuture second = decoder.submit( + "s1", + 2, + value -> value, + result -> published.add(result.value())); + + releaseSlow.countDown(); + first.get(5, TimeUnit.SECONDS); + second.get(5, TimeUnit.SECONDS); + assertEquals(List.of(1, 2), published); + } + } + + @Test + void shouldAllowDifferentStreamsToPublishIndependently() throws Exception + { + try (StreamOrderedDecoder decoder = + new StreamOrderedDecoder(4, "stream-parallel-test")) + { + decoder.start(); + List published = Collections.synchronizedList(new ArrayList<>()); + CountDownLatch aStarted = new CountDownLatch(1); + CountDownLatch releaseA = new CountDownLatch(1); + + CompletableFuture a = decoder.submit( + "a", + "a1", + value -> + { + aStarted.countDown(); + assertTrue(releaseA.await(5, TimeUnit.SECONDS)); + return value; + }, + result -> published.add(result.value())); + assertTrue(aStarted.await(5, TimeUnit.SECONDS)); + + CompletableFuture b = decoder.submit( + "b", + "b1", + value -> value, + result -> published.add(result.value())); + b.get(5, TimeUnit.SECONDS); + assertEquals(List.of("b1"), published); + + releaseA.countDown(); + a.get(5, TimeUnit.SECONDS); + assertEquals(List.of("b1", "a1"), published); + } + } + + @Test + void shouldFailFastOnSameStreamWhenPreviousPublishFails() throws Exception + { + try (StreamOrderedDecoder decoder = + new StreamOrderedDecoder(4, "stream-fail-fast-test")) + { + decoder.start(); + List published = Collections.synchronizedList(new ArrayList<>()); + + CompletableFuture first = decoder.submit( + "s1", + 1, + value -> value, + result -> + { + throw new IllegalStateException("publish failed"); + }); + ExecutionException firstError = assertThrows( + ExecutionException.class, () -> first.get(5, TimeUnit.SECONDS)); + assertInstanceOf(IllegalStateException.class, firstError.getCause()); + + // Submit after the stream has already failed; the failed tail must remain. + CompletableFuture second = decoder.submit( + "s1", + 2, + value -> value, + result -> published.add(result.value())); + ExecutionException secondError = assertThrows( + ExecutionException.class, () -> second.get(5, TimeUnit.SECONDS)); + assertInstanceOf(IllegalStateException.class, secondError.getCause()); + assertTrue(published.isEmpty()); + } + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/RpcEndToEndTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/RpcEndToEndTest.java deleted file mode 100644 index 5fd1764..0000000 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/RpcEndToEndTest.java +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package io.pixelsdb.pixels.sink.writer; - -import com.google.protobuf.ByteString; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.pixelsdb.pixels.sink.PixelsPollingServiceGrpc; -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.exception.SinkException; -import io.pixelsdb.pixels.sink.source.SinkSource; -import io.pixelsdb.pixels.sink.source.SinkSourceFactory; -import io.pixelsdb.pixels.sink.util.MetricsFacade; -import io.pixelsdb.pixels.sink.writer.retina.SinkContextManager; -import io.prometheus.client.exporter.HTTPServer; -import io.prometheus.client.hotspot.DefaultExports; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -public class RpcEndToEndTest -{ - private static final int TEST_PORT = 9091; - private static final int RECORD_COUNT = 5; - private static final String TEST_SCHEMA = "pixels_bench_sf10x"; - private static final String TEST_TABLE = "savingaccount"; - private static final String FULL_TABLE_NAME = TEST_SCHEMA + "." + TEST_TABLE; - private static final String CONFIG_FILE_PATH = "conf/pixels-sink.aws.properties"; - private static SinkSource sinkSource; // Keep a reference to stop it later - private static HTTPServer prometheusHttpServer; - - // This method contains the setup logic from PixelsSinkApp - private static void startServer() - { - System.out.println("[SETUP] Starting server with full initialization sequence..."); - try - { - // === 1. Mimic init() method from PixelsSinkApp === - System.out.println("[SETUP] Initializing configuration from " + CONFIG_FILE_PATH + "..."); - PixelsSinkConfigFactory.initialize(CONFIG_FILE_PATH); - - System.out.println("[SETUP] Initializing MetricsFacade..."); - MetricsFacade.getInstance().setSinkContextManager(SinkContextManager.getInstance()); - - // For determinism in testing, override the port from the config file - System.setProperty("sink.flink.server.port", String.valueOf(TEST_PORT)); - // === 2. Mimic main() method from PixelsSinkApp === - PixelsSinkConfig config = PixelsSinkConfigFactory.getInstance(); - System.out.println("[SETUP] Creating SinkSource application engine..."); - sinkSource = SinkSourceFactory.createSinkSource(); - System.out.println("[SETUP] Setting up Prometheus monitoring server..."); - if (config.isMonitorEnabled()) - { - DefaultExports.initialize(); - // To avoid port conflicts during tests, you could use port 0 for a random port - // For this example, we'll assume the config port is fine. - prometheusHttpServer = new HTTPServer(config.getMonitorPort()); - System.out.println("[SETUP] Prometheus server started on port: " + config.getMonitorPort()); - } - // === 3. THE MOST CRITICAL STEP: Start the processor === - System.out.println("[SETUP] Starting SinkSource processor threads..."); - sinkSource.start(); - System.out.println("[SETUP] Server is fully initialized and running."); - } catch (IOException e) - { - // If setup fails, we cannot run the test. - throw new RuntimeException("Failed to start the server for the test", e); - } - } - - public static void main(String[] args) throws InterruptedException, IOException - { - // ========== 1. SETUP PHASE ========== - // Start the server components within this same process. - startServer(); - - // Give the server a moment to fully start up and bind to the port. - Thread.sleep(2000); // 2 seconds, adjust if needed - // [REFACTORED] To ensure the FlinkPollingWriter uses our test port, - // we set it as a system property before the writer is created. - // The PixelsSinkConfigFactory should be configured to read this property. - System.setProperty("sink.flink.server.port", String.valueOf(TEST_PORT)); - - // [REFACTORED] The setup is now dramatically simpler. - // Instantiating FlinkPollingWriter is the ONLY step needed. - // Its constructor automatically creates the service and starts the gRPC server. - System.out.println("[SETUP] Initializing FlinkPollingWriter, which will start the gRPC server..."); - PixelsSinkWriter writer = PixelsSinkWriterFactory.getWriter(); - - // [REFACTORED] The following lines are no longer needed and have been removed: - // PixelsPollingServiceImpl serviceImpl = new PixelsPollingServiceImpl(writer); - // PollingRpcServer server = new PollingRpcServer(serviceImpl, TEST_PORT); - // server.start(); - - ExecutorService executor = Executors.newFixedThreadPool(2); - - try - { - // 3. Start the mock data producer (in a separate thread) - executor.submit(() -> - { - try - { - System.out.println("[PRODUCER] Starting mock data producer..."); - // Simulate producing 5 INSERT records - for (int i = 1; i <= 5; i++) - { - Thread.sleep(1000); // Simulate data production interval - - SinkProto.SourceInfo sourceInfo = SinkProto.SourceInfo.newBuilder() - .setDb(TEST_SCHEMA) - .setTable(TEST_TABLE) - .build(); - - byte[] idBytes = String.valueOf(i).getBytes(StandardCharsets.UTF_8); - SinkProto.ColumnValue idColumnValue = SinkProto.ColumnValue.newBuilder() - .setValue(ByteString.copyFrom(idBytes)) - .build(); - - SinkProto.RowValue afterImage = SinkProto.RowValue.newBuilder() - .addValues(idColumnValue) - .build(); - - SinkProto.RowRecord record = SinkProto.RowRecord.newBuilder() - .setSource(sourceInfo) - .setOp(SinkProto.OperationType.INSERT) - .setAfter(afterImage) - .build(); - - RowChangeEvent event = new RowChangeEvent(record); - System.out.printf("[PRODUCER] Writing INSERT record %d for table '%s.%s'%n", i, TEST_SCHEMA, TEST_TABLE); - writer.writeRow(event); - } - System.out.println("[PRODUCER] Finished writing data."); - } catch (InterruptedException e) - { - System.err.println("[PRODUCER] Producer thread was interrupted."); - Thread.currentThread().interrupt(); - } catch (SinkException e) - { - System.err.println("[PRODUCER] SinkException occurred: %s" + e.getMessage()); - } - }); - - // 4. Start the gRPC client (in another thread) - executor.submit(() -> - { - System.out.println("[CLIENT] Starting gRPC client..."); - ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", TEST_PORT) - .usePlaintext() - .build(); - - try - { - PixelsPollingServiceGrpc.PixelsPollingServiceBlockingStub client = PixelsPollingServiceGrpc.newBlockingStub(channel); - int recordsPolled = 0; - int maxRetries = 10; - int retryCount = 0; - while (recordsPolled < 5 && retryCount < maxRetries) - { - System.out.println("[CLIENT] Sending poll request for table '" + FULL_TABLE_NAME + "'..."); - SinkProto.PollRequest request = SinkProto.PollRequest.newBuilder() - .setSchemaName(TEST_SCHEMA) - .setTableName(TEST_TABLE) - .build(); - SinkProto.PollResponse response = client.pollEvents(request); - - if (response.getRecordsCount() > 0) - { - System.out.printf("[CLIENT] SUCCESS: Polled %d record(s).%n", response.getRecordsCount()); - response.getRecordsList().forEach(record -> System.out.println(" -> " + record.toString().trim())); - recordsPolled += response.getRecordsCount(); - } else - { - System.out.println("[CLIENT] Polled 0 records, will retry..."); - retryCount++; - } - } - if (recordsPolled == 5) - { - System.out.println("[CLIENT] Test finished successfully. Polled all 5 records."); - } else - { - System.err.println("[CLIENT] Test FAILED. Did not poll all records in time."); - } - } finally - { - channel.shutdownNow(); - } - }); - } finally - { - // 5. Wait for the test to complete and clean up resources - executor.shutdown(); - if (!executor.awaitTermination(1, TimeUnit.MINUTES)) - { - System.err.println("[CLEANUP] Test timed out."); - executor.shutdownNow(); - } else - { - System.out.println("[CLEANUP] Test completed."); - } - - // [REFACTORED] The correct way to stop the server is now by closing the writer. - writer.close(); - System.out.println("[CLEANUP] FlinkPollingWriter closed and server stopped."); - } - // ========== 3. CLEANUP PHASE ========== - System.out.println("[CLEANUP] Tearing down server components..."); - if (sinkSource != null) - { - sinkSource.stopProcessor(); - System.out.println("[CLEANUP] SinkSource processor stopped."); - // Note: The writer is part of sinkSource, and its resources - // should be cleaned up by stopProcessor or an equivalent close method. - } - if (prometheusHttpServer != null) - { - prometheusHttpServer.close(); - System.out.println("[CLEANUP] Prometheus server stopped."); - } - System.out.println("[CLEANUP] Test finished."); - } -} diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/TestProtoWriter.java b/src/test/java/io/pixelsdb/pixels/sink/writer/TestProtoWriter.java deleted file mode 100644 index 0c932a3..0000000 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/TestProtoWriter.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - - -package io.pixelsdb.pixels.sink.writer; - - -import com.google.protobuf.ByteString; -import io.pixelsdb.pixels.common.physical.*; -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import io.pixelsdb.pixels.sink.writer.proto.ProtoWriter; -import io.pixelsdb.pixels.storage.localfs.PhysicalLocalReader; -import lombok.SneakyThrows; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; - -/** - * @package: io.pixelsdb.pixels.sink.writer - * @className: TestProtoWriter - * @author: AntiO2 - * @date: 2025/10/5 09:24 - */ -public class TestProtoWriter -{ - public static String schemaName = "test"; - public static String tableName = "ray"; - - @BeforeAll - public static void setUp() throws IOException - { - PixelsSinkConfigFactory.initialize("/home/pixels/projects/pixels-writer/src/main/resources/pixels-writer.local.properties"); -// PixelsSinkConfigFactory.initialize("/home/ubuntu/pixels-writer/src/main/resources/pixels-writer.aws.properties"); - } - - private static SinkProto.RowRecord getRowRecord(int i) - { - byte[][] cols = new byte[3][]; - - cols[0] = Integer.toString(i).getBytes(StandardCharsets.UTF_8); - cols[1] = Long.toString(i * 1000L).getBytes(StandardCharsets.UTF_8); - cols[2] = ("row_" + i).getBytes(StandardCharsets.UTF_8); - SinkProto.RowValue.Builder afterValueBuilder = SinkProto.RowValue.newBuilder(); - afterValueBuilder - .addValues( - SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFrom((cols[0]))).build()) - .addValues( - SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFrom((cols[1]))).build()) - .addValues( - SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFrom((cols[2]))).build()); - - - SinkProto.RowRecord.Builder builder = SinkProto.RowRecord.newBuilder(); - builder.setOp(SinkProto.OperationType.INSERT) - .setAfter(afterValueBuilder) - .setSource( - SinkProto.SourceInfo.newBuilder() - .setDb(schemaName) - .setTable(tableName) - .build() - ); - return builder.build(); - } - - private static SinkProto.TransactionMetadata getTrans(int i, SinkProto.TransactionStatus status) - { - SinkProto.TransactionMetadata.Builder builder = SinkProto.TransactionMetadata.newBuilder(); - builder.setId(Integer.toString(i)); - builder.setStatus(status); - builder.setTimestamp(System.currentTimeMillis()); - return builder.build(); - } - - @SneakyThrows - @Test - public void testWriteTransInfo() - { - ProtoWriter transWriter = new ProtoWriter(); - int maxTx = 1000; - - for (int i = 0; i < maxTx; i++) - { - transWriter.writeTrans(getTrans(i, SinkProto.TransactionStatus.BEGIN)); - transWriter.writeTrans(getTrans(i, SinkProto.TransactionStatus.END)); - } - transWriter.close(); - } - - @Test - public void testWriteFile() throws IOException - { - String path = "/home/pixels/projects/pixels-writer/tmp/write.dat"; - PhysicalWriter writer = PhysicalWriterUtil.newPhysicalWriter(Storage.Scheme.file, path); - - int writeNum = 3; - - ByteBuffer buf = ByteBuffer.allocate(writeNum * Integer.BYTES); - for (int i = 0; i < 3; i++) - { - buf.putInt(i); - } - writer.append(buf); - writer.close(); - } - - @Test - public void testReadFile() throws IOException - { - String path = "/home/pixels/projects/pixels-writer/tmp/write.dat"; - PhysicalLocalReader reader = (PhysicalLocalReader) PhysicalReaderUtil.newPhysicalReader(Storage.Scheme.file, path); - - int writeNum = 12; - for (int i = 0; i < writeNum; i++) - { - reader.readLong(ByteOrder.BIG_ENDIAN); - } - } - - @Test - public void testReadEmptyFile() throws IOException - { - String path = "/home/pixels/projects/pixels-writer/tmp/empty.dat"; - PhysicalReader reader = PhysicalReaderUtil.newPhysicalReader(Storage.Scheme.file, path); - - int v = reader.readInt(ByteOrder.BIG_ENDIAN); - - return; - } - - @Test - public void testWriteRowInfo() throws IOException - { - ProtoWriter transWriter = new ProtoWriter(); - int maxTx = 10000000; - int rowCnt = 0; - for (int i = 0; i < maxTx; i++) - { - transWriter.writeTrans(getTrans(i, SinkProto.TransactionStatus.BEGIN)); - for (int j = i; j < 3; j++) - { - transWriter.write(getRowRecord(rowCnt++)); - } - transWriter.writeTrans(getTrans(i, SinkProto.TransactionStatus.END)); - } - transWriter.close(); - } -} diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/TpcHTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/TpcHTest.java index a3edbca..23dab0b 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/TpcHTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/TpcHTest.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer; import io.pixelsdb.pixels.common.exception.RetinaException; @@ -29,15 +24,18 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.nio.charset.StandardCharsets; -import java.util.logging.Logger; -public class TpcHTest +@Tag("integration") +class TpcHTest { - static Logger logger = Logger.getLogger(TpcHTest.class.getName()); + static Logger logger = LoggerFactory.getLogger(TpcHTest.class); static RetinaService retinaService; static MetadataService metadataService; static TransService transService; diff --git a/src/test/java/io/pixelsdb/pixels/sink/concurrent/TransactionServiceTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/TransServiceTest.java similarity index 74% rename from src/test/java/io/pixelsdb/pixels/sink/concurrent/TransactionServiceTest.java rename to src/test/java/io/pixelsdb/pixels/sink/writer/TransServiceTest.java index 0f590f1..9fc30ac 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/concurrent/TransactionServiceTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/TransServiceTest.java @@ -1,30 +1,26 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - -package io.pixelsdb.pixels.sink.concurrent; +package io.pixelsdb.pixels.sink.writer; import io.pixelsdb.pixels.common.exception.TransException; import io.pixelsdb.pixels.common.transaction.TransContext; import io.pixelsdb.pixels.common.transaction.TransService; import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,16 +31,17 @@ import static org.junit.jupiter.api.Assertions.assertTrue; @Slf4j -public class TransactionServiceTest +@Tag("integration") +class TransServiceTest { - private static final Logger logger = LoggerFactory.getLogger(TransactionServiceTest.class); + private static final Logger logger = LoggerFactory.getLogger(TransServiceTest.class); @Test public void testTransactionService() { - int numTransactions = 10; + int numTransactions = Integer.getInteger("pixels.sink.test.transactions", 10); - TransService transService = TransService.CreateInstance("localhost", 18889); + TransService transService = TransService.Instance(); try { List transContexts = transService.beginTransBatch(numTransactions, false); @@ -66,9 +63,9 @@ public void testTransactionService() @Test public void testBatchRequest() { - int numTransactions = 1000; + int numTransactions = Integer.getInteger("pixels.sink.test.batch.transactions", 100); - TransService transService = TransService.CreateInstance("localhost", 18889); + TransService transService = TransService.Instance(); try { List transContexts = transService.beginTransBatch(numTransactions, false); diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/proto/ProtoWriterTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/proto/ProtoWriterTest.java new file mode 100644 index 0000000..9609268 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/proto/ProtoWriterTest.java @@ -0,0 +1,173 @@ +/* + * Copyright 2025 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.pixelsdb.pixels.sink.writer.proto; + + +import com.google.protobuf.ByteString; +import io.pixelsdb.pixels.common.physical.*; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.TestConfig; +import io.pixelsdb.pixels.storage.localfs.PhysicalLocalReader; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class ProtoWriterTest +{ + private static final String SCHEMA_NAME = "test"; + private static final String TABLE_NAME = "ray"; + + private static SinkProto.RowRecord getRowRecord(int i) + { + byte[][] cols = new byte[3][]; + + cols[0] = Integer.toString(i).getBytes(StandardCharsets.UTF_8); + cols[1] = Long.toString(i * 1000L).getBytes(StandardCharsets.UTF_8); + cols[2] = ("row_" + i).getBytes(StandardCharsets.UTF_8); + SinkProto.RowValue.Builder afterValueBuilder = SinkProto.RowValue.newBuilder(); + afterValueBuilder + .addValues( + SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFrom((cols[0]))).build()) + .addValues( + SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFrom((cols[1]))).build()) + .addValues( + SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFrom((cols[2]))).build()); + + + SinkProto.RowRecord.Builder builder = SinkProto.RowRecord.newBuilder(); + builder.setOp(SinkProto.OperationType.INSERT) + .setAfter(afterValueBuilder) + .setSource( + SinkProto.SourceInfo.newBuilder() + .setDb(SCHEMA_NAME) + .setTable(TABLE_NAME) + .build() + ); + return builder.build(); + } + + private static SinkProto.TransactionMetadata getTrans(int i, SinkProto.TransactionStatus status) + { + SinkProto.TransactionMetadata.Builder builder = SinkProto.TransactionMetadata.newBuilder(); + builder.setId(Integer.toString(i)); + builder.setStatus(status); + builder.setTimestamp(System.currentTimeMillis()); + return builder.build(); + } + + @Tag("integration") + @Test + void testWriteTransInfo() throws Exception + { + TestConfig.initializeIntegrationConfig(); + int maxTx = Integer.getInteger("pixels.sink.test.transactions", 10); + try (ProtoWriter transWriter = new ProtoWriter()) + { + for (int i = 0; i < maxTx; i++) + { + transWriter.writeTrans(getTrans(i, SinkProto.TransactionStatus.BEGIN)); + transWriter.writeTrans(getTrans(i, SinkProto.TransactionStatus.END)); + } + } + } + + @Test + void testWriteFile(@TempDir Path tempDir) throws IOException + { + Path path = tempDir.resolve("write.dat"); + try (PhysicalWriter writer = PhysicalWriterUtil.newPhysicalWriter( + Storage.Scheme.file, path.toString())) + { + int writeNum = 3; + ByteBuffer buf = ByteBuffer.allocate(writeNum * Integer.BYTES); + for (int i = 0; i < writeNum; i++) + { + buf.putInt(i); + } + assertEquals(0, writer.append(buf)); + } + + assertEquals(3L * Integer.BYTES, Files.size(path)); + } + + @Test + void testReadFile(@TempDir Path tempDir) throws IOException + { + Path path = tempDir.resolve("write.dat"); + try (PhysicalWriter writer = PhysicalWriterUtil.newPhysicalWriter( + Storage.Scheme.file, path.toString())) + { + ByteBuffer buffer = ByteBuffer.allocate(3 * Long.BYTES); + buffer.putLong(11L).putLong(22L).putLong(33L); + writer.append(buffer); + } + + try (PhysicalLocalReader reader = (PhysicalLocalReader) PhysicalReaderUtil + .newPhysicalReader(Storage.Scheme.file, path.toString())) + { + assertEquals(3L * Long.BYTES, reader.getFileLength()); + assertEquals(11L, reader.readLong(ByteOrder.BIG_ENDIAN)); + assertEquals(22L, reader.readLong(ByteOrder.BIG_ENDIAN)); + assertEquals(33L, reader.readLong(ByteOrder.BIG_ENDIAN)); + } + } + + @Test + void testReadEmptyFile(@TempDir Path tempDir) throws IOException + { + Path path = tempDir.resolve("empty.dat"); + Files.createFile(path); + + try (PhysicalReader reader = PhysicalReaderUtil.newPhysicalReader( + Storage.Scheme.file, path.toString())) + { + assertEquals(0, reader.getFileLength()); + assertThrows(IOException.class, () -> reader.readInt(ByteOrder.BIG_ENDIAN)); + } + } + + @Tag("integration") + @Test + void testWriteRowInfo() throws Exception + { + TestConfig.initializeIntegrationConfig(); + int maxTx = Integer.getInteger("pixels.sink.test.transactions", 10); + int rowCnt = 0; + try (ProtoWriter transWriter = new ProtoWriter()) + { + for (int i = 0; i < maxTx; i++) + { + transWriter.writeTrans(getTrans(i, SinkProto.TransactionStatus.BEGIN)); + for (int j = 0; j < 3; j++) + { + transWriter.write(getRowRecord(rowCnt++)); + } + transWriter.writeTrans(getTrans(i, SinkProto.TransactionStatus.END)); + } + } + assertEquals(maxTx * 3, rowCnt); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/TestRetinaWriter.java b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterIntegrationTest.java similarity index 92% rename from src/test/java/io/pixelsdb/pixels/sink/writer/TestRetinaWriter.java rename to src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterIntegrationTest.java index 15e43ea..32e4bb9 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/TestRetinaWriter.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterIntegrationTest.java @@ -1,24 +1,19 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - -package io.pixelsdb.pixels.sink.writer; +package io.pixelsdb.pixels.sink.writer.retina; import com.google.protobuf.ByteString; import io.pixelsdb.pixels.common.exception.RetinaException; @@ -29,15 +24,16 @@ import io.pixelsdb.pixels.index.IndexProto; import io.pixelsdb.pixels.retina.RetinaProto; import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.TestConfig; import io.pixelsdb.pixels.sink.event.RowChangeEvent; import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.core.utils.DatetimeUtils; import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; -import io.pixelsdb.pixels.sink.util.DateUtil; -import io.pixelsdb.pixels.sink.writer.retina.RetinaServiceProxy; -import io.pixelsdb.pixels.sink.writer.retina.TransactionProxy; +import io.pixelsdb.pixels.sink.writer.PixelsSinkWriter; +import io.pixelsdb.pixels.sink.writer.PixelsSinkWriterFactory; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,6 +41,9 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; import java.util.ArrayList; import java.util.List; import java.util.Random; @@ -53,10 +52,11 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -public class TestRetinaWriter +@Tag("integration") +class RetinaWriterIntegrationTest { - static Logger logger = LoggerFactory.getLogger(TestRetinaWriter.class.getName()); + static Logger logger = LoggerFactory.getLogger(RetinaWriterIntegrationTest.class.getName()); static RetinaService retinaService; static TableMetadataRegistry metadataRegistry; static TransService transService; @@ -65,15 +65,16 @@ public class TestRetinaWriter private final ExecutorService executor = Executors.newFixedThreadPool(16); @BeforeAll - public static void setUp() throws IOException + static void setUp() throws Exception { - PixelsSinkConfigFactory.initialize("/home/pixels/projects/pixels-sink/src/main/resources/pixels-sink.local.properties"); -// PixelsSinkConfigFactory.initialize("/home/ubuntu/pixels-sink/src/main/resources/pixels-sink.aws.properties"); + TestConfig.initializeIntegrationConfig(); retinaService = RetinaService.Instance(); metadataRegistry = TableMetadataRegistry.Instance(); transService = TransService.Instance(); - retinaPerformanceTestRowCount = 5_000_000; - retinaPerformanceTestMaxId = 2_000_000; + retinaPerformanceTestRowCount = Integer.getInteger( + "pixels.sink.test.retina.row.count", 1_000); + retinaPerformanceTestMaxId = Integer.getInteger( + "pixels.sink.test.retina.max.id", 2_000); } @Test @@ -228,7 +229,7 @@ public void testCheckingAccountInsertPerformance() throws for (int sb = 0; sb < samllBatchCount; sb++) { ++b; - TransContext ctx = manager.getNewTransContext(); + TransContext ctx = manager.getNewTransContext("batch-" + b); long timeStamp = ctx.getTimestamp(); RetinaProto.TableUpdateData.Builder tableUpdateDataBuilder = @@ -305,8 +306,7 @@ public void testCheckingAccountInsertPerformance() throws Assertions.assertNotNull(writer); if (!writer.writeTrans(schemaName, tableUpdateData)) { - logger.error("Error Write Trans"); - System.exit(-1); + throw new AssertionError("Failed to write transaction"); } } @@ -361,7 +361,7 @@ public void testCheckingAccountUpdatePerformance() throws // 轮询选择客户端 RetinaServiceProxy writer = writers.get(batchIndex % clientCount); - TransContext ctx = manager.getNewTransContext(); + TransContext ctx = manager.getNewTransContext("batch-" + batchIndex); long timeStamp = ctx.getTimestamp(); List tableUpdateData = new ArrayList<>(); @@ -386,14 +386,14 @@ public void testCheckingAccountUpdatePerformance() throws .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Integer.toString(userID))).build()) .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Float.toString(oldBalance))).build()) .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Integer.toString(isBlocked))).build()) - .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(DateUtil.convertDebeziumTimestampToString(oldTs))).build()); + .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(formatTimestamp(oldTs))).build()); SinkProto.RowValue.Builder afterValueBuilder = SinkProto.RowValue.newBuilder() .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Integer.toString(accountID))).build()) .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Integer.toString(userID))).build()) .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Float.toString(newBalance))).build()) .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Integer.toString(isBlocked))).build()) - .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(DateUtil.convertDebeziumTimestampToString(newTs))).build()); + .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(formatTimestamp(newTs))).build()); SinkProto.RowRecord.Builder rowBuilder = SinkProto.RowRecord.newBuilder() .setOp(SinkProto.OperationType.UPDATE) @@ -415,7 +415,7 @@ public void testCheckingAccountUpdatePerformance() throws .addColValues(ByteString.copyFromUtf8(Integer.toString(userID))) .addColValues(ByteString.copyFromUtf8(Float.toString(newBalance))) .addColValues(ByteString.copyFromUtf8(Integer.toString(isBlocked))) - .addColValues(ByteString.copyFromUtf8(DateUtil.convertDebeziumTimestampToString(newTs))) + .addColValues(ByteString.copyFromUtf8(formatTimestamp(newTs))) .addIndexKeys(rowChangeEvent.getAfterKey()); tableUpdateDataBuilder.addInsertData(insertDataBuilder.build()); } @@ -425,8 +425,7 @@ public void testCheckingAccountUpdatePerformance() throws long startTime = System.currentTimeMillis(); if (!writer.writeTrans(schemaName, tableUpdateData)) { - logger.error("Error Write Trans"); - System.exit(-1); + throw new AssertionError("Failed to write transaction"); } long endTime = System.currentTimeMillis(); logger.debug("writeTrans batch " + batchIndex + " took " + (endTime - startTime) + " ms"); @@ -481,7 +480,7 @@ public void testInsertTwoTablePerformance() throws for (int b = 0; b < batchCount; b++) { - TransContext ctx = manager.getNewTransContext(); + TransContext ctx = manager.getNewTransContext("batch-" + b); long timeStamp = ctx.getTimestamp(); List tableUpdateData = new ArrayList<>(); @@ -506,7 +505,7 @@ public void testInsertTwoTablePerformance() throws cols[1] = Integer.toString(userID).getBytes(StandardCharsets.UTF_8); cols[2] = Float.toString(balance).getBytes(StandardCharsets.UTF_8); cols[3] = Integer.toString(isBlocked).getBytes(StandardCharsets.UTF_8); - cols[4] = DateUtil.convertDebeziumTimestampToString(ts).getBytes(StandardCharsets.UTF_8); + cols[4] = formatTimestamp(ts).getBytes(StandardCharsets.UTF_8); // cols[4] = Long.toString(ts).getBytes(StandardCharsets.UTF_8); // after row SinkProto.RowValue.Builder afterValueBuilder = SinkProto.RowValue.newBuilder() @@ -599,4 +598,10 @@ public void testInsertTwoTablePerformance() throws double transPerSec = batchCount * 2 / seconds; logger.info("Inserted " + totalInserts + " rows in " + seconds + "s, rate=" + insertsPerSec + " inserts/s," + transPerSec + "trans/s"); } + + private static String formatTimestamp(long epochTs) + { + return LocalDateTime.ofInstant(Instant.ofEpochMilli(epochTs), ZoneId.systemDefault()) + .format(DatetimeUtils.SQL_LOCAL_DATE_TIME); + } } diff --git a/src/test/java/io/pixelsdb/pixels/sink/concurrent/RetinaWriterTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterTest.java similarity index 92% rename from src/test/java/io/pixelsdb/pixels/sink/concurrent/RetinaWriterTest.java rename to src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterTest.java index 74b9e4f..2bfd9f5 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/concurrent/RetinaWriterTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterTest.java @@ -12,18 +12,15 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ - -package io.pixelsdb.pixels.sink.concurrent; +package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.TestUtils; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.TestConfig; import io.pixelsdb.pixels.sink.event.RowChangeEvent; import io.pixelsdb.pixels.sink.exception.SinkException; -import io.pixelsdb.pixels.sink.writer.retina.RetinaWriter; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; @@ -38,11 +35,13 @@ import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +@Tag("integration") class RetinaWriterTest { private static final Logger LOGGER = LoggerFactory.getLogger(RetinaWriterTest.class); @@ -52,11 +51,16 @@ class RetinaWriterTest private CountDownLatch latch; @BeforeEach - void setUp() throws IOException + void setUp() throws Exception { - PixelsSinkConfigFactory.initialize(""); + TestConfig.initializeIntegrationConfig(); - testExecutor = TestUtils.synchronousExecutor(); + testExecutor = Executors.newSingleThreadExecutor(runnable -> + { + Thread thread = new Thread(runnable); + thread.setDaemon(true); + return thread; + }); dispatchedEvents = Collections.synchronizedList(new ArrayList<>()); coordinator = new RetinaWriter(); @@ -155,16 +159,12 @@ void shouldProcessNonTransactionalEvents(SinkProto.OperationType opType) throws coordinator.writeRow(event); TimeUnit.MILLISECONDS.sleep(10); assertEquals(1, dispatchedEvents.size()); - PixelsSinkConfigFactory.reset(); } @ParameterizedTest @ValueSource(ints = {1, 3, 9, 16}) void shouldHandleConcurrentEvents(int threadCount) throws SinkException, IOException, InterruptedException { - PixelsSinkConfigFactory.reset(); - PixelsSinkConfigFactory.initialize(""); - latch = new CountDownLatch(threadCount); coordinator.writeTrans(buildBeginTx("tx5")); // concurrently send event @@ -192,6 +192,5 @@ void shouldHandleConcurrentEvents(int threadCount) throws SinkException, IOExcep LOGGER.debug("Thread Count: {} DispatchedEvents size: {}", threadCount, dispatchedEvents.size()); LOGGER.debug("Thread Count: {} DispatchedEvents size: {}", threadCount, dispatchedEvents.size()); assertEquals(threadCount, dispatchedEvents.size()); - PixelsSinkConfigFactory.reset(); } } \ No newline at end of file diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/retina/TableWriterProxyTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/TableWriterProxyTest.java index fa4e996..53f0d40 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/retina/TableWriterProxyTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/TableWriterProxyTest.java @@ -1,54 +1,52 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.TestConfig; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; -public class TableWriterProxyTest -{ - private static final Logger LOGGER = LoggerFactory.getLogger(TableWriterProxyTest.class); +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; - - String tableName = "test"; +@Tag("integration") +class TableWriterProxyTest +{ + private static final String TABLE_NAME = "test"; @BeforeAll - public static void init() throws IOException + static void init() throws Exception { - PixelsSinkConfigFactory.initialize("/home/ubuntu/pixels-sink/conf/pixels-sink.aws.properties"); + TestConfig.initializeIntegrationConfig(); } @Test - public void testGetSameTableWriter() throws IOException + void shouldReuseTableWriter() throws IOException { TableWriterProxy tableWriterProxy = TableWriterProxy.getInstance(); + TableWriter first = tableWriterProxy.getTableWriter(TABLE_NAME, 0, 0); for (int i = 0; i < 10; i++) { - TableWriter tableWriter = tableWriterProxy.getTableWriter(tableName, 0, 0); + TableWriter tableWriter = tableWriterProxy.getTableWriter(TABLE_NAME, 0, 0); + assertNotNull(tableWriter); + assertSame(first, tableWriter); } } } diff --git a/src/test/resources/log4j2.properties b/src/test/resources/log4j2.properties deleted file mode 100644 index 09ab7d9..0000000 --- a/src/test/resources/log4j2.properties +++ /dev/null @@ -1,27 +0,0 @@ -status=info -name=pixels-sink -filter.threshold.type=ThresholdFilter -filter.threshold.level=info -appender.console.type=Console -appender.console.name=STDOUT -appender.console.layout.type=PatternLayout -appender.console.layout.pattern=%-d{yyyy-MM-dd HH:mm:ss,SSS} [%c]-[%p] %m%n -appender.rolling.type=File -appender.rolling.name=log -appender.rolling.append=true -appender.rolling.fileName=${env:PIXELS_HOME}/logs/pixels-sink.log -appender.rolling.layout.type=PatternLayout -appender.rolling.layout.pattern=%-d{yyyy-MM-dd HH:mm:ss,SSS} [%c]-[%p] %m%n -rootLogger.level=info -rootLogger.appenderRef.stdout.ref=STDOUT -rootLogger.appenderRef.log.ref=log -logger.transaction.name=io.pixelsdb.pixels.sink.sink.retina.RetinaWriter -logger.transaction.level=info -logger.transaction.appenderRef.log.ref=log -logger.transaction.appenderRef.stdout.ref=STDOUT -logger.transaction.additivity=false -logger.grpc.name=io.grpc.netty.shaded.io.grpc.netty.NettyClientHandler -logger.grpc.level=info -logger.grpc.additivity=false -logger.grpc.appenderRef.log.ref=log -logger.grpc.appenderRef.stdout.ref=STDOUT diff --git a/src/test/resources/pixels-sink-test.properties b/src/test/resources/pixels-sink-test.properties new file mode 100644 index 0000000..c7e33b3 --- /dev/null +++ b/src/test/resources/pixels-sink-test.properties @@ -0,0 +1,5 @@ +sink.monitor.enable=false +sink.monitor.report.enable=false +sink.mode=none +sink.debezium.dialect=mysql +debezium.connector.class=io.debezium.connector.mysql.MySqlConnector diff --git a/src/test/resources/records/delete.json b/src/test/resources/records/mysql-region-delete.json similarity index 100% rename from src/test/resources/records/delete.json rename to src/test/resources/records/mysql-region-delete.json diff --git a/src/test/resources/records/update.json b/src/test/resources/records/mysql-region-update.json similarity index 100% rename from src/test/resources/records/update.json rename to src/test/resources/records/mysql-region-update.json diff --git a/src/test/resources/records/nation.json b/src/test/resources/records/postgresql-nation-snapshot.json similarity index 100% rename from src/test/resources/records/nation.json rename to src/test/resources/records/postgresql-nation-snapshot.json diff --git a/src/test/resources/records/insert.json b/src/test/resources/records/postgresql-region-insert.json similarity index 100% rename from src/test/resources/records/insert.json rename to src/test/resources/records/postgresql-region-insert.json