From 7c4d5258860f7e7ec12aa4aa86062f6d228e09af Mon Sep 17 00:00:00 2001 From: dannygeng Date: Mon, 20 Jul 2026 17:39:12 +0800 Subject: [PATCH 01/13] feat: Refactor sink source conversion and event pipelines --- conf/pixels-sink.aws.properties | 9 +- conf/pixels-sink.ch.properties | 9 +- conf/pixels-sink.mysql.properties | 99 ++++++ conf/pixels-sink.pg.properties | 9 +- docs/configuration.md | 13 +- docs/overview.md | 74 ++--- pom.xml | 11 + .../pixels/sink/config/PixelsSinkConfig.java | 11 +- .../sink/config/PixelsSinkConstants.java | 5 +- .../sink/config/PixelsSinkDefaultConfig.java | 1 - .../pixels/sink/config/TransactionConfig.java | 4 +- .../factory/RowRecordKafkaPropFactory.java | 17 +- .../factory/TransactionKafkaPropFactory.java | 18 +- .../debezium/DebeziumAvroRowConverter.java | 106 +++++++ .../DebeziumAvroTransactionConverter.java | 48 +++ .../debezium/DebeziumEnvelopeNormalizer.java | 113 +++++++ .../debezium/DebeziumJsonRowConverter.java | 128 ++++++++ .../debezium/DebeziumRecordUtil.java} | 57 ++-- .../debezium/DebeziumRowRecordAssembler.java | 71 +++++ .../debezium/DebeziumRowValueConverter.java} | 268 +++++++--------- .../RowChangeEventAvroDeserializer.java | 31 ++ .../RowChangeEventJsonDeserializer.java | 31 ++ .../RowChangeEventStructConverter.java | 157 ++++++++++ .../TransactionMetadataAvroDeserializer.java | 31 ++ .../TransactionMetadataJsonDeserializer.java | 31 ++ .../source/DebeziumSourceAdapter.java | 39 +++ .../source/DebeziumSourceAdapterRegistry.java | 57 ++++ .../source/DebeziumSourceMetadata.java | 19 ++ .../debezium/source/MySqlSourceAdapter.java | 43 +++ .../source/PostgresSourceAdapter.java | 49 +++ .../kafka/KafkaRecordConverter.java | 58 ++++ .../sinkproto/RowRecordConverter.java | 56 ++++ .../TransactionMetadataConverter.java | 39 +++ .../pixels/sink/event/RowChangeEvent.java | 55 ++-- .../sink/event/RowChangeEventFactory.java | 106 +++++++ .../RowChangeEventAvroDeserializer.java | 169 ----------- .../RowChangeEventJsonDeserializer.java | 173 ----------- .../RowChangeEventStructDeserializer.java | 145 --------- .../deserializer/SchemaDeserializer.java | 231 -------------- .../TransactionAvroMessageDeserializer.java | 77 ----- .../TransactionJsonMessageDeserializer.java | 70 ----- .../TransactionStructMessageDeserializer.java | 78 ----- .../pixels/sink/pipeline/TablePipeline.java | 58 ++++ .../sink/pipeline/TablePipelineManager.java | 54 ++++ .../sink/pipeline/TransactionPipeline.java | 64 ++++ .../pixels/sink/processor/TableProcessor.java | 22 +- .../pixels/sink/processor/TopicProcessor.java | 285 ------------------ .../sink/processor/TransactionProcessor.java | 11 +- .../sink/provider/BlockingEventProvider.java | 88 ++++++ .../pixels/sink/provider/EventProvider.java | 185 ------------ .../sink/provider/RowEventProvider.java | 36 +++ .../provider/TableEventEngineProvider.java | 55 ---- .../provider/TableEventKafkaProvider.java | 114 ------- .../sink/provider/TableEventProvider.java | 54 ---- .../TableEventStorageLoopProvider.java | 90 ------ .../provider/TableEventStorageProvider.java | 62 ---- ...leProviderAndProcessorPipelineManager.java | 96 ------ .../TransactionEventEngineProvider.java | 52 ---- .../TransactionEventKafkaProvider.java | 98 ------ .../provider/TransactionEventProvider.java | 16 +- .../TransactionEventStorageLoopProvider.java | 47 --- .../TransactionEventStorageProvider.java | 43 --- .../source/engine/PixelsDebeziumConsumer.java | 178 +++++++++-- .../sink/source/engine/SinkEngineSource.java | 1 + .../DebeziumSourceAdapterSelector.java | 47 +++ .../engine/adapter/DebeziumStructAdapter.java | 58 ++++ .../sink/source/kafka/KafkaRowSource.java | 144 +++++++++ .../source/kafka/KafkaTransactionSource.java | 121 ++++++++ .../sink/source/kafka/SinkKafkaSource.java | 36 ++- .../sink/source/kafka/TopicProcessor.java | 184 +++++++++++ .../kafka/serde/KafkaRecordConverter.java | 86 ++++++ .../serde/RowChangeEventAvroDeserializer.java | 79 +++++ .../serde/RowChangeEventJsonDeserializer.java | 78 +++++ .../kafka/serde/RowRecordDeserializer.java | 53 ++++ .../TransactionMetadataAvroDeserializer.java | 78 +++++ .../TransactionMetadataDeserializer.java | 47 +++ .../TransactionMetadataJsonDeserializer.java | 78 +++++ .../AbstractMemorySinkStorageSource.java | 3 +- .../AbstractReaderSinkStorageSource.java | 124 -------- .../storage/AbstractSinkStorageSource.java | 71 +++-- .../storage/FasterSinkStorageSource.java | 2 +- .../storage/LegacySinkStorageSource.java | 247 --------------- .../pixels/sink/util/DataTransform.java | 55 ---- .../pixelsdb/pixels/sink/util/DateUtil.java | 46 --- .../pixels/sink/util/EtcdFileRegistry.java | 4 - .../pixels/sink/util/LatencySimulator.java | 58 ---- .../sink/writer/retina/RetinaWriter.java | 1 - .../writer/retina/TableCrossTxWriter.java | 7 - .../sink/writer/retina/TransactionProxy.java | 6 - ...sion.debezium.source.DebeziumSourceAdapter | 2 + src/main/resources/pixels-sink.aws.properties | 9 +- .../resources/pixels-sink.local.properties | 9 +- src/main/resources/pixels-sink.properties | 5 +- .../pixels/sink/DebeziumEngineTest.java | 2 + .../sink/consumer/AvroConsumerTest.java | 2 +- .../DebeziumEnvelopeNormalizerTest.java | 251 +++++++++++++++ .../DebeziumRowValueConverterTest.java | 169 +++++++++++ .../RowChangeEventJsonDeserializerTest.java | 182 +++++++++++ .../kafka/KafkaRecordConverterTest.java | 56 ++++ .../sinkproto/RowRecordConverterTest.java | 102 +++++++ .../{deserializer => }/RowBatchTest.java | 2 +- .../sink/event/RowChangeEventFactoryTest.java | 116 +++++++ .../RowChangeEventDeserializerTest.java | 81 ----- .../event/deserializer/RowDataParserTest.java | 58 ---- .../deserializer/SchemaDeserializerTest.java | 142 --------- .../provider/BlockingEventProviderTest.java | 61 ++++ .../engine/PixelsDebeziumConsumerTest.java | 204 +++++++++++++ .../sink/util/EtcdFileRegistryTest.java | 3 +- .../pixels/sink/util/TestDateUtil.java | 57 ++++ .../pixels/sink/writer/TestRetinaWriter.java | 16 +- .../resources/pixels-sink-test.properties | 3 + 111 files changed, 4413 insertions(+), 3357 deletions(-) create mode 100644 conf/pixels-sink.mysql.properties create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumAvroRowConverter.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumAvroTransactionConverter.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizer.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumJsonRowConverter.java rename src/main/java/io/pixelsdb/pixels/sink/{event/deserializer/DeserializerUtil.java => conversion/debezium/DebeziumRecordUtil.java} (69%) create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowRecordAssembler.java rename src/main/java/io/pixelsdb/pixels/sink/{event/deserializer/RowDataParser.java => conversion/debezium/DebeziumRowValueConverter.java} (54%) create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventAvroDeserializer.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventJsonDeserializer.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventStructConverter.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/TransactionMetadataAvroDeserializer.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/TransactionMetadataJsonDeserializer.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceAdapter.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceAdapterRegistry.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceMetadata.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/MySqlSourceAdapter.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/PostgresSourceAdapter.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/kafka/KafkaRecordConverter.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverter.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/TransactionMetadataConverter.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactory.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventAvroDeserializer.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventJsonDeserializer.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventStructDeserializer.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/event/deserializer/SchemaDeserializer.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionAvroMessageDeserializer.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionJsonMessageDeserializer.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionStructMessageDeserializer.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipeline.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipelineManager.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/pipeline/TransactionPipeline.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/processor/TopicProcessor.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/provider/BlockingEventProvider.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/provider/EventProvider.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/provider/RowEventProvider.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/provider/TableEventEngineProvider.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/provider/TableEventKafkaProvider.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/provider/TableEventProvider.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/provider/TableEventStorageLoopProvider.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/provider/TableEventStorageProvider.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/provider/TableProviderAndProcessorPipelineManager.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventEngineProvider.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventKafkaProvider.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventStorageLoopProvider.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventStorageProvider.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumSourceAdapterSelector.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumStructAdapter.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaRowSource.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaTransactionSource.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/kafka/TopicProcessor.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverter.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowChangeEventAvroDeserializer.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowChangeEventJsonDeserializer.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowRecordDeserializer.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataAvroDeserializer.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataDeserializer.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataJsonDeserializer.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractReaderSinkStorageSource.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/storage/LegacySinkStorageSource.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/util/LatencySimulator.java create mode 100644 src/main/resources/META-INF/services/io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter create mode 100644 src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizerTest.java create mode 100644 src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverterTest.java create mode 100644 src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventJsonDeserializerTest.java create mode 100644 src/test/java/io/pixelsdb/pixels/sink/conversion/kafka/KafkaRecordConverterTest.java create mode 100644 src/test/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverterTest.java rename src/test/java/io/pixelsdb/pixels/sink/event/{deserializer => }/RowBatchTest.java (97%) create mode 100644 src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactoryTest.java delete mode 100644 src/test/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventDeserializerTest.java delete mode 100644 src/test/java/io/pixelsdb/pixels/sink/event/deserializer/RowDataParserTest.java delete mode 100644 src/test/java/io/pixelsdb/pixels/sink/event/deserializer/SchemaDeserializerTest.java create mode 100644 src/test/java/io/pixelsdb/pixels/sink/provider/BlockingEventProviderTest.java create mode 100644 src/test/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumerTest.java create mode 100644 src/test/java/io/pixelsdb/pixels/sink/util/TestDateUtil.java create mode 100644 src/test/resources/pixels-sink-test.properties diff --git a/conf/pixels-sink.aws.properties b/conf/pixels-sink.aws.properties index cd9f98d..012b60f 100644 --- a/conf/pixels-sink.aws.properties +++ b/conf/pixels-sink.aws.properties @@ -41,8 +41,8 @@ bootstrap.servers=realtime-kafka-2:29092 group.id=3078 auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -#value.deserializer=io.pixelsdb.pixels.writer.deserializer.RowChangeEventAvroDeserializer -value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventJsonDeserializer +#value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventAvroDeserializer +value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer # Topic & Database Config topic.prefix=postgresql.oltp_server consumer.capture_database=pixels_bench_sf1x @@ -78,8 +78,8 @@ sink.flink.server.port=9091 sink.registry.url=http://localhost:8080/apis/registry/v2 # Transaction Config transaction.topic.suffix=transaction -#transaction.topic.value.deserializer=io.pixelsdb.pixels.writer.deserializer.TransactionAvroMessageDeserializer -transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.TransactionJsonMessageDeserializer +#transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataAvroDeserializer +transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer sink.trans.batch.size=100 # Sink Metrics @@ -90,7 +90,6 @@ sink.monitor.freshness.interval=1000 # Interact with other rpc sink.rpc.enable=true -sink.rpc.mock.delay=20 # debezium engine config debezium.name=testEngine debezium.connector.class=io.debezium.connector.postgresql.PostgresConnector diff --git a/conf/pixels-sink.ch.properties b/conf/pixels-sink.ch.properties index c9a4fbe..c979389 100644 --- a/conf/pixels-sink.ch.properties +++ b/conf/pixels-sink.ch.properties @@ -42,8 +42,8 @@ bootstrap.servers=realtime-kafka-2:29092 group.id=3078 auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -#value.deserializer=io.pixelsdb.pixels.writer.deserializer.RowChangeEventAvroDeserializer -value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventJsonDeserializer +#value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventAvroDeserializer +value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer # Topic & Database Config topic.prefix=postgresql.oltp_server consumer.capture_database=pixels_bench_sf1x @@ -75,8 +75,8 @@ sink.flink.server.port=9091 sink.registry.url=http://localhost:8080/apis/registry/v2 # Transaction Config transaction.topic.suffix=transaction -#transaction.topic.value.deserializer=io.pixelsdb.pixels.writer.deserializer.TransactionAvroMessageDeserializer -transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.TransactionJsonMessageDeserializer +#transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataAvroDeserializer +transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer sink.trans.batch.size=100 # Sink Metrics @@ -87,7 +87,6 @@ sink.monitor.freshness.interval=1000 # Interact with other rpc sink.rpc.enable=true -sink.rpc.mock.delay=20 # debezium engine config debezium.name=testEngine debezium.connector.class=io.debezium.connector.postgresql.PostgresConnector diff --git a/conf/pixels-sink.mysql.properties b/conf/pixels-sink.mysql.properties new file mode 100644 index 0000000..4b8bc97 --- /dev/null +++ b/conf/pixels-sink.mysql.properties @@ -0,0 +1,99 @@ +# engine | kafka | storage +sink.datasource=engine +# -1 means no limit, Only implement in retina sink mode yet +sink.datasource.rate.limit=100000 +# Sink Config: retina | csv | proto | flink | none +sink.mode=retina +sink.retina.client=8 +sink.retina.log.queue=false +## batch or single or record, batch is recommend. record is faster, but doesn't have ACID feature +sink.trans.mode=batch +sink.monitor.report.enable=true +sink.monitor.report.file=/home/ubuntu/pixels-sink/resulti7i/100k_rate_mysql.csv +sink.monitor.freshness.file=/home/ubuntu/pixels-sink/resulti7i/100k_freshness_mysql.csv +# trino for freshness query +trino.url=jdbc:trino://realtime-kafka-2:8080/pixels/pixels_bench_sf10x +trino.user=pixels +trino.password=password +trino.parallel=8 +# row or txn or embed +sink.monitor.freshness.level=embed +sink.monitor.freshness.embed.warmup=10 +sink.monitor.freshness.embed.static=false +sink.monitor.freshness.embed.snapshot=true +sink.monitor.freshness.embed.tablelist=binlog_test +sink.monitor.freshness.verbose=true +sink.monitor.freshness.timestamp=true +sink.storage.loop=true +# Kafka Config +bootstrap.servers=realtime-kafka-2:29092 +group.id=3078 +auto.offset.reset=earliest +key.deserializer=org.apache.kafka.common.serialization.StringDeserializer +value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer +# Topic & Database Config +topic.prefix=mysql-cdc +consumer.capture_database=cdc_verify +consumer.include_tables= +sink.csv.path=./data +sink.csv.enable_header=false +## Retina Config +sink.retina.embedded=false +# stub or stream +sink.retina.mode=stream +sink.remote.host=localhost +sink.remote.port=29422 +sink.timeout.ms=5000 +sink.flush.interval.ms=50 +sink.flush.batch.size=10 +sink.max.retries=3 +## writer commit +# sync or async +sink.commit.method=sync +sink.commit.batch.size=10 +sink.commit.batch.worker=32 +sink.commit.batch.delay=3000 +## Proto Config +sink.proto.dir=file:///home/ubuntu/disk1/hybench/ +sink.proto.data=hybench10_10 +sink.proto.maxRecords=100000 +## Flink Config +sink.flink.server.port=9091 +## Schema Registry +sink.registry.url=http://localhost:8080/apis/registry/v2 +# Transaction Config +transaction.topic.suffix=transaction +transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer +sink.trans.batch.size=100 + +# Sink Metrics +sink.monitor.enable=true +sink.monitor.port=9464 +sink.monitor.report.interval=10000 +sink.monitor.freshness.interval=1000 + +# Interact with other rpc +sink.rpc.enable=true +# debezium engine config +debezium.name=mysql-cdc-engine +debezium.connector.class=io.debezium.connector.mysql.MySqlConnector +debezium.provide.transaction.metadata=true +debezium.offset.storage=org.apache.kafka.connect.storage.FileOffsetBackingStore +debezium.offset.storage.file.filename=/tmp/pixels-sink-mysql-offsets.dat +debezium.offset.flush.interval.ms=60000 +debezium.schema.history.internal=io.debezium.storage.file.history.FileSchemaHistory +debezium.schema.history.internal.file.filename=/tmp/pixels-sink-mysql-schema-history.dat +debezium.database.hostname=tdsql +debezium.database.port=6060 +debezium.database.user=tdsql +debezium.database.password=tdsql123 +debezium.database.server.id=5401 +debezium.database.include.list=cdc_verify +debezium.snapshot.mode=never +debezium.key.converter=org.apache.kafka.connect.json.JsonConverter +debezium.value.converter=org.apache.kafka.connect.json.JsonConverter +debezium.topic.prefix=mysql-cdc +debezium.transforms=topicRouting +debezium.transforms.topicRouting.type=org.apache.kafka.connect.transforms.RegexRouter +debezium.transforms.topicRouting.regex=mysql-cdc\\.cdc_verify\\.(.*) +debezium.transforms.topicRouting.replacement=mysql-cdc.cdc_verify.$1 diff --git a/conf/pixels-sink.pg.properties b/conf/pixels-sink.pg.properties index effc171..868c64d 100644 --- a/conf/pixels-sink.pg.properties +++ b/conf/pixels-sink.pg.properties @@ -31,8 +31,8 @@ bootstrap.servers=realtime-kafka-2:29092 group.id=3078 auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -#value.deserializer=io.pixelsdb.pixels.writer.deserializer.RowChangeEventAvroDeserializer -value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventJsonDeserializer +#value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventAvroDeserializer +value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer # Topic & Database Config topic.prefix=postgresql.oltp_server consumer.capture_database=pixels_bench_sf1x @@ -66,8 +66,8 @@ sink.flink.server.port=9091 sink.registry.url=http://localhost:8080/apis/registry/v2 # Transaction Config transaction.topic.suffix=transaction -#transaction.topic.value.deserializer=io.pixelsdb.pixels.writer.deserializer.TransactionAvroMessageDeserializer -transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.TransactionJsonMessageDeserializer +#transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataAvroDeserializer +transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer sink.trans.batch.size=100 # Sink Metrics @@ -78,7 +78,6 @@ sink.monitor.freshness.interval=1000 # Interact with other rpc sink.rpc.enable=true -sink.rpc.mock.delay=20 # debezium engine config debezium.name=testEngine debezium.connector.class=io.debezium.connector.postgresql.PostgresConnector diff --git a/docs/configuration.md b/docs/configuration.md index 845af65..6355e23 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -20,6 +20,8 @@ Values are loaded by `PixelsSinkConfig` and mapped from keys in the properties f - `engine` reads CDC logs directly from Debezium Engine. - `storage` reads CDC logs from files dumped by `sink.proto` output; schema reference: [sink.proto](https://github.com/pixelsdb/pixels/blob/master/proto/sink.proto). - `kafka` reads from a set of Kafka topics; this mode is deprecated and not actively tested. +- Engine and Kafka records are normalized to the canonical `SinkProto` contract before reaching writers. +- Storage row records use canonical `SinkProto`; `SourceInfo.schema` may be empty for MySQL. ### Notes on `sink.mode` @@ -51,9 +53,13 @@ Notes on `sink.trans.mode`: | Key | Default | Notes | | --- | --- | --- | | `debezium.name` | none | Engine name. | -| `debezium.connector.class` | none | Connector class, e.g. PostgreSQL connector. | +| `debezium.connector.class` | none | Connector class, e.g. PostgreSQL or MySQL connector. | | `debezium.*` | none | Standard Debezium engine properties. | +See `conf/pixels-sink.mysql.properties` for a TDSQL MySQL CDC example. + +The Debezium Connector reads the database Binlog or WAL. The local `conversion.debezium` package only converts envelopes already emitted by Debezium. + ### Retina Sink | Key | Default | Notes | @@ -108,12 +114,12 @@ Kafka source is deprecated. | `group.id` | required | Consumer group id. | | `auto.offset.reset` | none | Standard Kafka consumer property. | | `key.deserializer` | `org.apache.kafka.common.serialization.StringDeserializer` | Kafka key deserializer. | -| `value.deserializer` | `io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventJsonDeserializer` | Kafka value deserializer for row events. | +| `value.deserializer` | `io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer` | Kafka value deserializer for row events. | | `topic.prefix` | required | Topic prefix for table events. | | `consumer.capture_database` | required | Database name used to build topic names. | | `consumer.include_tables` | empty | Comma-separated table list, empty means all. | | `transaction.topic.suffix` | `transaction` | Suffix appended to transaction topics. | -| `transaction.topic.value.deserializer` | `io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventJsonDeserializer` | Deserializer for transaction messages. | +| `transaction.topic.value.deserializer` | `io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer` | Deserializer for transaction messages. | | `transaction.topic.group_id` | `transaction_consumer` | Consumer group for transaction topic. | | `sink.registry.url` | required | Avro Schema registry endpoint. | @@ -124,7 +130,6 @@ Kafka source is deprecated. | `sink.remote.host` | `localhost` | Sink server host. | | `sink.remote.port` | `9090` | Sink server port. | | `sink.rpc.enable` | `false` | Enable RPC simulation (for development). | -| `sink.rpc.mock.delay` | `0` | Artificial delay in ms. | **Monitoring and Metrics** diff --git a/docs/overview.md b/docs/overview.md index c321e20..e60b0b7 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -15,7 +15,7 @@ Pixels Sink uses a multi-stage pipeline. Each stage communicates via producer/co - Receives a `ConfigFactory` directly and builds the same sink pipeline. **Source** -The source stage pulls events and forwards raw payloads to providers. +The source stage owns input lifecycle and invokes the configured converter. **Source Inputs** | Source Type | Description | Related Config | @@ -25,76 +25,54 @@ The source stage pulls events and forwards raw payloads to providers. | `storage` | Reads from Pixels storage files containing serialized sink proto records | `sink.proto.*`, `sink.storage.loop` | **Source Outputs** -- The source does not parse events. It forwards raw records to providers. +- Kafka, Engine, and Storage sources publish only canonical + `RowChangeEvent` or `SinkProto.TransactionMetadata` objects. +- Protocol conversion is implemented in `conversion`; providers never receive + Kafka bytes, Engine `SourceRecord`, or Storage `ByteBuffer`. **Provider** -Providers convert source records into Pixels events. +Providers are bounded canonical-event channels. They provide backpressure, +ordered delivery, and lifecycle management without knowing the source +protocol. ```mermaid classDiagram direction TB - class EventProvider~SOURCE_RECORD_T, TARGET_RECORD_T~ { - +run() + class BlockingEventProvider~T~ { + +publish(T event) + +take() +close() - +processLoop() - +convertToTargetRecord() - +recordSerdEvent() - +putRawEvent() - +getRawEvent() - +pollRawEvent() - +putTargetEvent() - +getTargetEvent() } - class TableEventProvider~SOURCE_RECORD_T~ { + class RowEventProvider { } - class TableEventEngineProvider~T~ { + class TransactionEventProvider { } - class TableEventKafkaProvider~T~ { + class TablePipeline { + +publish(RowChangeEvent) } - class TableEventStorageProvider~T~ { + class TransactionPipeline { + +publish(TransactionMetadata) } - - class TransactionEventProvider~SOURCE_RECORD_T~ { - } - class TransactionEventEngineProvider~T~ { - } - class TransactionEventKafkaProvider~T~ { + class TablePipelineManager { + +route(RowChangeEvent) } - class TransactionEventStorageProvider~T~ { - } - - EventProvider <|-- TableEventProvider - EventProvider <|-- TransactionEventProvider - TableEventProvider <|-- TableEventEngineProvider - TableEventProvider <|-- TableEventKafkaProvider - TableEventProvider <|-- TableEventStorageProvider - - TransactionEventProvider <|-- TransactionEventEngineProvider - TransactionEventProvider <|-- TransactionEventKafkaProvider - TransactionEventProvider <|-- TransactionEventStorageProvider + BlockingEventProvider <|-- RowEventProvider + BlockingEventProvider <|-- TransactionEventProvider + TablePipelineManager --> TablePipeline + TablePipeline --> RowEventProvider + TransactionPipeline --> TransactionEventProvider ``` -Example mappings: - -| Provider | Source Type | Target Type | -| --- | --- | --- | -| `TableEventEngineProvider` | Debezium Struct | `RowChangeEvent` | -| `TableEventKafkaProvider` | Kafka topic | `RowChangeEvent` | -| `TableEventStorageProvider` | Proto bytes | `RowChangeEvent` | -| `TransactionEventEngineProvider` | Debezium Struct | `SinkProto.TransactionMetadata` | -| `TransactionEventKafkaProvider` | Kafka topic | `SinkProto.TransactionMetadata` | -| `TransactionEventStorageProvider` | Proto bytes | `SinkProto.TransactionMetadata` | - **Processor** Processors pull events from providers and write to the sink writers. -- `TableProcessor` instances are created by `TableProviderAndProcessorPipelineManager`. +- `TableProcessor` instances are created by `TablePipelineManager`. - There is typically one `TableProcessor` per table to maintain per-table ordering. -- `TransactionProcessor` is a singleton. +- `TransactionPipeline` owns the transaction provider and `TransactionProcessor`. **Writer** Writers implement `PixelsSinkWriter`: diff --git a/pom.xml b/pom.xml index 2b965f2..e3038ae 100644 --- a/pom.xml +++ b/pom.xml @@ -63,6 +63,12 @@ org.apache.hive hive-jdbc 2.3.9 + + + jdk.tools + jdk.tools + + io.etcd @@ -185,6 +191,11 @@ debezium-connector-postgres ${dep.debezium.version} + + io.debezium + debezium-connector-mysql + ${dep.debezium.version} + junit diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java index 4f75eeb..297b852 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java @@ -21,7 +21,8 @@ package io.pixelsdb.pixels.sink.config; import io.pixelsdb.pixels.common.utils.ConfigFactory; -import io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventJsonDeserializer; +import io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer; +import io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer; import io.pixelsdb.pixels.sink.writer.PixelsSinkMode; import io.pixelsdb.pixels.sink.writer.retina.RetinaServiceProxy; import io.pixelsdb.pixels.sink.writer.retina.TransactionMode; @@ -117,9 +118,6 @@ public class PixelsSinkConfig @ConfigKey(value = "sink.rpc.enable", defaultValue = "false") private boolean rpcEnable; - @ConfigKey(value = "sink.rpc.mock.delay", defaultValue = "0") - private int mockRpcDelay; - @ConfigKey(value = "sink.trans.batch.size", defaultValue = "100") private int transBatchSize; @@ -136,6 +134,9 @@ public class PixelsSinkConfig @ConfigKey("debezium.topic.prefix") private String debeziumTopicPrefix; + @ConfigKey(value = "debezium.connector.class", defaultValue = "") + private String debeziumConnectorClass; + @ConfigKey("consumer.capture_database") private String captureDatabase; @@ -161,7 +162,7 @@ public class PixelsSinkConfig private String transactionTopicSuffix; @ConfigKey(value = "transaction.topic.value.deserializer", - defaultClass = RowChangeEventJsonDeserializer.class) + defaultClass = TransactionMetadataJsonDeserializer.class) private String transactionTopicValueDeserializer; @ConfigKey(value = "transaction.topic.group_id", diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java index 034ae78..b5d7477 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java @@ -24,7 +24,10 @@ public final class PixelsSinkConstants { public static final String ROW_RECORD_KAFKA_PROP_FACTORY = "row-record"; public static final String TRANSACTION_KAFKA_PROP_FACTORY = "transaction"; - public static final int MONITOR_NUM = 2; + public static final String ROW_RECORD_CONVERTER_CLASS = "pixels.sink.row.converter.class"; + public static final String TRANSACTION_CONVERTER_CLASS = "pixels.sink.transaction.converter.class"; + public static final String DEBEZIUM_CONNECTOR_CLASS = "debezium.connector.class"; + public static final int MONITOR_NUM = 3; public static final int MAX_QUEUE_SIZE = 1_000; public static final String SNAPSHOT_TX_PREFIX = "SNAPSHOT-"; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkDefaultConfig.java b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkDefaultConfig.java index 1a56ddd..a0667ad 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkDefaultConfig.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkDefaultConfig.java @@ -52,6 +52,5 @@ public class PixelsSinkDefaultConfig // Mock RPC public static final boolean SINK_RPC_ENABLED = true; - public static final int MOCK_RPC_DELAY = 100; public static final String MAX_RECORDS_PER_FILE = "100000"; } diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/TransactionConfig.java b/src/main/java/io/pixelsdb/pixels/sink/config/TransactionConfig.java index f516b3a..88c0c5f 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/TransactionConfig.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/TransactionConfig.java @@ -20,12 +20,12 @@ package io.pixelsdb.pixels.sink.config; -import io.pixelsdb.pixels.sink.event.deserializer.TransactionJsonMessageDeserializer; +import io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer; public class TransactionConfig { public static final String DEFAULT_TRANSACTION_TOPIC_SUFFIX = "transaction"; - public static final String DEFAULT_TRANSACTION_TOPIC_VALUE_DESERIALIZER = TransactionJsonMessageDeserializer.class.getName(); + public static final String DEFAULT_TRANSACTION_TOPIC_VALUE_DESERIALIZER = TransactionMetadataJsonDeserializer.class.getName(); public static final String DEFAULT_TRANSACTION_TOPIC_GROUP_ID = "transaction_consumer"; public static final String DEFAULT_TRANSACTION_TIME_OUT = "300"; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/factory/RowRecordKafkaPropFactory.java b/src/main/java/io/pixelsdb/pixels/sink/config/factory/RowRecordKafkaPropFactory.java index 6c1fbc0..2a82521 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/factory/RowRecordKafkaPropFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/factory/RowRecordKafkaPropFactory.java @@ -21,7 +21,10 @@ package io.pixelsdb.pixels.sink.config.factory; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; +import io.apicurio.registry.serde.SerdeConfig; import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; import java.util.Properties; @@ -40,7 +43,19 @@ static Properties getCommonKafkaProperties(PixelsSinkConfig config) public Properties createKafkaProperties(PixelsSinkConfig config) { Properties kafkaProperties = getCommonKafkaProperties(config); - kafkaProperties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, config.getValueDeserializer()); + kafkaProperties.put(PixelsSinkConstants.ROW_RECORD_CONVERTER_CLASS, config.getValueDeserializer()); + if (config.getDebeziumConnectorClass() != null && + !config.getDebeziumConnectorClass().isBlank()) + { + kafkaProperties.put( + PixelsSinkConstants.DEBEZIUM_CONNECTOR_CLASS, + config.getDebeziumConnectorClass()); + } + if (config.getRegistryUrl() != null && !config.getRegistryUrl().isBlank()) + { + kafkaProperties.put(SerdeConfig.REGISTRY_URL, config.getRegistryUrl()); + } + kafkaProperties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class); kafkaProperties.put(ConsumerConfig.GROUP_ID_CONFIG, config.getGroupId()); kafkaProperties.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "30000"); diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/factory/TransactionKafkaPropFactory.java b/src/main/java/io/pixelsdb/pixels/sink/config/factory/TransactionKafkaPropFactory.java index 52949b7..49c0592 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/factory/TransactionKafkaPropFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/factory/TransactionKafkaPropFactory.java @@ -20,8 +20,11 @@ package io.pixelsdb.pixels.sink.config.factory; +import io.apicurio.registry.serde.SerdeConfig; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; import java.util.Properties; @@ -33,7 +36,20 @@ public class TransactionKafkaPropFactory implements KafkaPropFactory public Properties createKafkaProperties(PixelsSinkConfig config) { Properties kafkaProperties = getCommonKafkaProperties(config); - kafkaProperties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, config.getTransactionTopicValueDeserializer()); + kafkaProperties.put(PixelsSinkConstants.TRANSACTION_CONVERTER_CLASS, + config.getTransactionTopicValueDeserializer()); + if (config.getDebeziumConnectorClass() != null && + !config.getDebeziumConnectorClass().isBlank()) + { + kafkaProperties.put( + PixelsSinkConstants.DEBEZIUM_CONNECTOR_CLASS, + config.getDebeziumConnectorClass()); + } + if (config.getRegistryUrl() != null && !config.getRegistryUrl().isBlank()) + { + kafkaProperties.put(SerdeConfig.REGISTRY_URL, config.getRegistryUrl()); + } + kafkaProperties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class); kafkaProperties.put(ConsumerConfig.GROUP_ID_CONFIG, config.getTransactionTopicGroupId() + "-" + config.getGroupId()); return kafkaProperties; } diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumAvroRowConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumAvroRowConverter.java new file mode 100644 index 0000000..d577956 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumAvroRowConverter.java @@ -0,0 +1,106 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.conversion.debezium; + +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import org.apache.avro.generic.GenericRecord; + +public final class DebeziumAvroRowConverter +{ + private final TableMetadataRegistry tableMetadataRegistry; + private final DebeziumSourceAdapter configuredAdapter; + private final DebeziumRowRecordAssembler rowRecordAssembler = + new DebeziumRowRecordAssembler(); + + public DebeziumAvroRowConverter(TableMetadataRegistry tableMetadataRegistry) + { + this(tableMetadataRegistry, null); + } + + public DebeziumAvroRowConverter( + TableMetadataRegistry tableMetadataRegistry, + DebeziumSourceAdapter configuredAdapter) + { + this.tableMetadataRegistry = tableMetadataRegistry; + this.configuredAdapter = configuredAdapter; + } + + public RowChangeEvent convert(GenericRecord record) throws SinkException + { + SinkProto.OperationType operation = DebeziumRecordUtil.getOperationType( + DebeziumRecordUtil.getStringSafely(record, "op")); + Object sourceObject = record.get("source"); + if (!(sourceObject instanceof GenericRecord source)) + { + throw new SinkException("Missing source field in row record"); + } + DebeziumSourceAdapter adapter = configuredAdapter == null + ? DebeziumEnvelopeNormalizer.adapterForSource(source) + : configuredAdapter; + SinkProto.SourceInfo sourceInfo = + DebeziumEnvelopeNormalizer.normalizeSource(source, adapter); + TableMetadata metadata = tableMetadataRegistry.getMetadata( + sourceInfo.getDb(), sourceInfo.getTable()); + TypeDescription schema = metadata.getTypeDescription(); + DebeziumRowValueConverter rowValueConverter = + new DebeziumRowValueConverter(schema); + + SinkProto.RowValue before = parseRowValue( + record.get("before"), operation, true, rowValueConverter); + SinkProto.RowValue after = parseRowValue( + record.get("after"), operation, false, rowValueConverter); + SinkProto.TransactionInfo transaction = record.get("transaction") == null + ? null + : DebeziumEnvelopeNormalizer.normalizeTransaction( + record.get("transaction"), adapter); + return rowRecordAssembler.assemble( + operation, sourceInfo, transaction, before, after, schema, metadata); + } + + private SinkProto.RowValue parseRowValue( + Object value, + SinkProto.OperationType operation, + boolean before, + DebeziumRowValueConverter converter) throws SinkException + { + boolean required = before + ? DebeziumRecordUtil.hasBeforeValue(operation) + : DebeziumRecordUtil.hasAfterValue(operation); + if (!required) + { + return null; + } + if (!(value instanceof GenericRecord row)) + { + throw new SinkException( + "Missing " + (before ? "before" : "after") + " image for " + operation); + } + SinkProto.RowValue.Builder builder = SinkProto.RowValue.newBuilder(); + converter.parse(row, builder); + return builder.build(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumAvroTransactionConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumAvroTransactionConverter.java new file mode 100644 index 0000000..1a47c12 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumAvroTransactionConverter.java @@ -0,0 +1,48 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.conversion.debezium; + +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; +import org.apache.avro.generic.GenericRecord; + +public final class DebeziumAvroTransactionConverter +{ + private final DebeziumSourceAdapter configuredAdapter; + + public DebeziumAvroTransactionConverter() + { + this(null); + } + + public DebeziumAvroTransactionConverter(DebeziumSourceAdapter configuredAdapter) + { + this.configuredAdapter = configuredAdapter; + } + + public SinkProto.TransactionMetadata convert(GenericRecord record) + { + return configuredAdapter == null + ? DebeziumEnvelopeNormalizer.normalizeTransactionMetadata(record) + : DebeziumEnvelopeNormalizer.normalizeTransactionMetadata( + record, configuredAdapter); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizer.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizer.java new file mode 100644 index 0000000..f1e5dcd --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizer.java @@ -0,0 +1,113 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + */ +package io.pixelsdb.pixels.sink.conversion.debezium; + +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapterRegistry; +import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceMetadata; + +public final class DebeziumEnvelopeNormalizer +{ + private DebeziumEnvelopeNormalizer() + { + } + + public static DebeziumSourceMetadata extractSource(Object source) + { + return new DebeziumSourceMetadata( + DebeziumRecordUtil.getStringSafely(source, "connector"), + DebeziumRecordUtil.getStringSafely(source, "db"), + DebeziumRecordUtil.getStringSafely(source, "schema"), + DebeziumRecordUtil.getStringSafely(source, "table")); + } + + public static DebeziumSourceAdapter adapterForSource(Object source) + { + String payloadConnector = DebeziumRecordUtil.getStringSafely(source, "connector"); + return DebeziumSourceAdapterRegistry.forSource(payloadConnector); + } + + public static DebeziumSourceAdapter adapterForTransaction(Object transaction) + { + Object source = DebeziumRecordUtil.getFieldSafely(transaction, "source"); + if (source == null) + { + throw new IllegalArgumentException( + "Transaction metadata requires an explicit source adapter"); + } + return adapterForSource(source); + } + + public static SinkProto.SourceInfo normalizeSource(Object source) + { + return normalizeSource(source, adapterForSource(source)); + } + + public static SinkProto.SourceInfo normalizeSource( + Object source, DebeziumSourceAdapter adapter) + { + DebeziumSourceMetadata metadata = extractSource(source); + if (!metadata.connector().isBlank() && + !adapter.connector().equalsIgnoreCase(metadata.connector())) + { + throw new IllegalArgumentException( + "Configured Debezium connector " + adapter.connector() + + " does not match source.connector " + metadata.connector()); + } + + return adapter.normalizeSource(metadata); + } + + public static SinkProto.TransactionInfo normalizeTransaction( + Object transaction, DebeziumSourceAdapter adapter) + { + String rawTransactionId = DebeziumRecordUtil.getStringSafely(transaction, "id"); + return SinkProto.TransactionInfo.newBuilder() + .setId(adapter.normalizeTransactionId(rawTransactionId)) + .setTotalOrder(DebeziumRecordUtil.getLongSafely(transaction, "total_order")) + .setDataCollectionOrder( + DebeziumRecordUtil.getLongSafely(transaction, "data_collection_order")) + .build(); + } + + public static SinkProto.TransactionMetadata normalizeTransactionMetadata( + Object transaction, DebeziumSourceAdapter adapter) + { + String rawTransactionId = DebeziumRecordUtil.getStringSafely(transaction, "id"); + SinkProto.TransactionMetadata.Builder builder = SinkProto.TransactionMetadata.newBuilder() + .setStatus(DebeziumRecordUtil.getStatusSafely(transaction, "status")) + .setId(adapter.normalizeTransactionId(rawTransactionId)) + .setEventCount(DebeziumRecordUtil.getLongSafely(transaction, "event_count")) + .setTimestamp(DebeziumRecordUtil.getLongSafely(transaction, "ts_ms")); + + Object collections = DebeziumRecordUtil.getFieldSafely(transaction, "data_collections"); + if (collections instanceof Iterable iterable) + { + for (Object item : iterable) + { + String sourceDataCollection = + DebeziumRecordUtil.getStringSafely(item, "data_collection"); + builder.addDataCollections(SinkProto.DataCollection.newBuilder() + .setDataCollection( + adapter.normalizeDataCollection(sourceDataCollection)) + .setEventCount( + DebeziumRecordUtil.getLongSafely(item, "event_count"))); + } + } + return builder.build(); + } + + public static SinkProto.TransactionMetadata normalizeTransactionMetadata(Object transaction) + { + return normalizeTransactionMetadata(transaction, adapterForTransaction(transaction)); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumJsonRowConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumJsonRowConverter.java new file mode 100644 index 0000000..5f04ca9 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumJsonRowConverter.java @@ -0,0 +1,128 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.conversion.debezium; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; + +public final class DebeziumJsonRowConverter +{ + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private final TableMetadataRegistry tableMetadataRegistry; + private final DebeziumSourceAdapter configuredAdapter; + private final DebeziumRowRecordAssembler rowRecordAssembler = + new DebeziumRowRecordAssembler(); + + public DebeziumJsonRowConverter(TableMetadataRegistry tableMetadataRegistry) + { + this(tableMetadataRegistry, null); + } + + public DebeziumJsonRowConverter( + TableMetadataRegistry tableMetadataRegistry, + DebeziumSourceAdapter configuredAdapter) + { + this.tableMetadataRegistry = tableMetadataRegistry; + this.configuredAdapter = configuredAdapter; + } + + public RowChangeEvent convert(byte[] data) throws Exception + { + JsonNode rootNode = OBJECT_MAPPER.readTree(data); + JsonNode payloadNode = rootNode.path("payload"); + SinkProto.OperationType operation = DebeziumRecordUtil.getOperationType( + payloadNode.path("op").asText("")); + + if (!payloadNode.hasNonNull("source")) + { + throw new SinkException("Missing source field in row record"); + } + + JsonNode sourceNode = payloadNode.get("source"); + DebeziumSourceAdapter adapter = configuredAdapter == null + ? DebeziumEnvelopeNormalizer.adapterForSource(sourceNode) + : configuredAdapter; + SinkProto.SourceInfo sourceInfo = + DebeziumEnvelopeNormalizer.normalizeSource(sourceNode, adapter); + TableMetadata metadata = tableMetadataRegistry.getMetadata( + sourceInfo.getDb(), sourceInfo.getTable()); + TypeDescription schema = metadata.getTypeDescription(); + DebeziumRowValueConverter rowValueConverter = + new DebeziumRowValueConverter(schema); + + SinkProto.TransactionInfo transaction = payloadNode.hasNonNull("transaction") + ? DebeziumEnvelopeNormalizer.normalizeTransaction( + payloadNode.get("transaction"), adapter) + : null; + SinkProto.RowValue before = parseBefore( + payloadNode, operation, rowValueConverter); + SinkProto.RowValue after = parseAfter( + payloadNode, operation, rowValueConverter); + return rowRecordAssembler.assemble( + operation, sourceInfo, transaction, before, after, schema, metadata); + } + + private SinkProto.RowValue parseBefore( + JsonNode payload, + SinkProto.OperationType operation, + DebeziumRowValueConverter converter) throws SinkException + { + if (!DebeziumRecordUtil.hasBeforeValue(operation)) + { + return null; + } + JsonNode before = payload.get("before"); + if (before == null || before.isNull()) + { + throw new SinkException("Missing before image for " + operation); + } + SinkProto.RowValue.Builder builder = SinkProto.RowValue.newBuilder(); + converter.parse(before, builder); + return builder.build(); + } + + private SinkProto.RowValue parseAfter( + JsonNode payload, + SinkProto.OperationType operation, + DebeziumRowValueConverter converter) throws SinkException + { + if (!DebeziumRecordUtil.hasAfterValue(operation)) + { + return null; + } + JsonNode after = payload.get("after"); + if (after == null || after.isNull()) + { + throw new SinkException("Missing after image for " + operation); + } + SinkProto.RowValue.Builder builder = SinkProto.RowValue.newBuilder(); + converter.parse(after, builder); + return builder.build(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/DeserializerUtil.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRecordUtil.java similarity index 69% rename from src/main/java/io/pixelsdb/pixels/sink/event/deserializer/DeserializerUtil.java rename to src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRecordUtil.java index 81ef25d..31ba018 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/DeserializerUtil.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRecordUtil.java @@ -7,27 +7,22 @@ * it under the terms of the Affero GNU General Public License as * published by the Free Software Foundation, either version 3 of * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . */ +package io.pixelsdb.pixels.sink.conversion.debezium; -package io.pixelsdb.pixels.sink.event.deserializer; - +import com.fasterxml.jackson.databind.JsonNode; import io.pixelsdb.pixels.sink.SinkProto; import org.apache.avro.generic.GenericRecord; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.source.SourceRecord; -public class DeserializerUtil +public final class DebeziumRecordUtil { - static public SinkProto.TransactionStatus getStatusSafely(T record, String field) + private DebeziumRecordUtil() + { + } + + public static SinkProto.TransactionStatus getStatusSafely(T record, String field) { String statusString = getStringSafely(record, field); if (statusString.equals("BEGIN")) @@ -52,6 +47,10 @@ public static Object getFieldSafely(T record, String field) } else if (record instanceof Struct struct) { return struct.get(field); + } else if (record instanceof JsonNode json) + { + JsonNode value = json.get(field); + return value == null || value.isNull() ? null : value; } else if (record instanceof SourceRecord sourceRecord) { return ((Struct) sourceRecord.value()).get(field); @@ -66,49 +65,53 @@ public static Object getFieldSafely(T record, String field) public static String getStringSafely(T record, String field) { Object value = getFieldSafely(record, field); + if (value instanceof JsonNode json) + { + return json.asText(""); + } return value != null ? value.toString() : ""; } public static Long getLongSafely(T record, String field) { Object value = getFieldSafely(record, field); + if (value instanceof JsonNode json) + { + return json.asLong(); + } return value instanceof Number ? ((Number) value).longValue() : 0L; } public static Integer getIntSafely(T record, String field) { Object value = getFieldSafely(record, field); + if (value instanceof JsonNode json) + { + return json.asInt(); + } return value instanceof Number ? ((Number) value).intValue() : 0; } - static public SinkProto.OperationType getOperationType(String op) + public static SinkProto.OperationType getOperationType(String op) { - op = op.toLowerCase(); - return switch (op) + return switch (op.toLowerCase()) { case "c" -> SinkProto.OperationType.INSERT; case "u" -> SinkProto.OperationType.UPDATE; case "d" -> SinkProto.OperationType.DELETE; case "r" -> SinkProto.OperationType.SNAPSHOT; - default -> throw new IllegalArgumentException(String.format("Can't convert %s to operation type", op)); + default -> throw new IllegalArgumentException( + String.format("Can't convert %s to operation type", op)); }; } - static public boolean hasBeforeValue(SinkProto.OperationType op) + public static boolean hasBeforeValue(SinkProto.OperationType op) { return op == SinkProto.OperationType.DELETE || op == SinkProto.OperationType.UPDATE; } - static public boolean hasAfterValue(SinkProto.OperationType op) + public static boolean hasAfterValue(SinkProto.OperationType op) { return op != SinkProto.OperationType.DELETE; } - - static public String getTransIdPrefix(String originTransID) - { - return originTransID.contains(":") - ? originTransID.substring(0, originTransID.indexOf(":")) - : originTransID; - } - } diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowRecordAssembler.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowRecordAssembler.java new file mode 100644 index 0000000..9b72956 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowRecordAssembler.java @@ -0,0 +1,71 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.conversion.debezium; + +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.event.RowChangeEventFactory; +import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; + +public final class DebeziumRowRecordAssembler +{ + public RowChangeEvent assemble( + SinkProto.OperationType operation, + SinkProto.SourceInfo source, + SinkProto.TransactionInfo transaction, + SinkProto.RowValue before, + SinkProto.RowValue after, + TypeDescription schema) throws SinkException + { + return assemble(operation, source, transaction, before, after, schema, null); + } + + public RowChangeEvent assemble( + SinkProto.OperationType operation, + SinkProto.SourceInfo source, + SinkProto.TransactionInfo transaction, + SinkProto.RowValue before, + SinkProto.RowValue after, + TypeDescription schema, + TableMetadata metadata) throws SinkException + { + SinkProto.RowRecord.Builder builder = SinkProto.RowRecord.newBuilder() + .setOp(operation) + .setSource(source); + if (transaction != null) + { + builder.setTransaction(transaction); + } + if (before != null) + { + builder.setBefore(before); + } + if (after != null) + { + builder.setAfter(after); + } + return metadata == null + ? RowChangeEventFactory.create(builder.build(), schema) + : RowChangeEventFactory.create(builder.build(), schema, metadata); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowDataParser.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverter.java similarity index 54% rename from src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowDataParser.java rename to src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverter.java index 5c85d0a..cf8928b 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowDataParser.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverter.java @@ -18,32 +18,26 @@ * . */ -package io.pixelsdb.pixels.sink.event.deserializer; +package io.pixelsdb.pixels.sink.conversion.debezium; import com.fasterxml.jackson.databind.JsonNode; import com.google.protobuf.ByteString; import io.pixelsdb.pixels.core.TypeDescription; import io.pixelsdb.pixels.sink.SinkProto; import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.connect.data.Field; -import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.Struct; -import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.time.LocalDate; import java.util.Base64; -import java.util.Collections; -import java.util.Map; -class RowDataParser +public class DebeziumRowValueConverter { private final TypeDescription schema; - public RowDataParser(TypeDescription schema) + public DebeziumRowValueConverter(TypeDescription schema) { this.schema = schema; } @@ -55,8 +49,15 @@ private static void buildFloat32(float value, SinkProto.ColumnValue.Builder colu private static void buildInt32(int value, int capacity, SinkProto.ColumnValue.Builder columnValueBuilder) { - int intBits = value; - byte[] bytes = ByteBuffer.allocate(capacity).putInt(intBits).array(); + ByteBuffer buffer = ByteBuffer.allocate(capacity); + switch (capacity) + { + case Byte.BYTES -> buffer.put((byte) value); + case Short.BYTES -> buffer.putShort((short) value); + case Integer.BYTES -> buffer.putInt(value); + default -> throw new IllegalArgumentException("Unsupported integer width: " + capacity); + } + byte[] bytes = buffer.array(); columnValueBuilder.setValue(ByteString.copyFrom(bytes)); } @@ -66,7 +67,7 @@ public void parse(GenericRecord record, SinkProto.RowValue.Builder builder) { String fieldName = schema.getFieldNames().get(i); TypeDescription fieldType = schema.getChildren().get(i); - builder.addValues(parseValue(record, fieldName, fieldType).build()); + builder.addValues(parseCanonicalValue(record.get(fieldName), fieldType).build()); } } @@ -85,9 +86,12 @@ public void parse(Struct record, SinkProto.RowValue.Builder builder) for (int i = 0; i < schema.getFieldNames().size(); i++) { String fieldName = schema.getFieldNames().get(i); - Field field = record.schema().field(fieldName); - Schema.Type fieldType = field.schema().type(); - builder.addValues(parseValue(record.get(fieldName), fieldName, fieldType).build()); + TypeDescription fieldType = schema.getChildren().get(i); + if (record.schema().field(fieldName) == null) + { + throw new IllegalArgumentException("Missing field in Debezium row schema: " + fieldName); + } + builder.addValues(parseCanonicalValue(record.get(fieldName), fieldType).build()); } } @@ -97,13 +101,24 @@ private SinkProto.ColumnValue.Builder parseValue(JsonNode valueNode, String fiel { return SinkProto.ColumnValue.newBuilder() // .setName(fieldName) - .setValue(ByteString.EMPTY); + .setValue(ByteString.EMPTY) + .setIsNull(true); } SinkProto.ColumnValue.Builder columnValueBuilder = SinkProto.ColumnValue.newBuilder(); switch (type.getCategory()) { + case BYTE: + { + buildInt32(valueNode.asInt(), Byte.BYTES, columnValueBuilder); + break; + } + case SHORT: + { + buildInt32(valueNode.asInt(), Short.BYTES, columnValueBuilder); + break; + } case INT: { buildInt32(valueNode.asInt(), Integer.BYTES, columnValueBuilder); @@ -131,7 +146,7 @@ private SinkProto.ColumnValue.Builder parseValue(JsonNode valueNode, String fiel case STRING: case VARBINARY: { - String value = valueNode.asText().trim(); + String value = valueNode.asText(); columnValueBuilder.setValue(ByteString.copyFrom(value, StandardCharsets.UTF_8)); // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.STRING)); break; @@ -175,6 +190,7 @@ private SinkProto.ColumnValue.Builder parseValue(JsonNode valueNode, String fiel break; } case DATE: + case TIME: { buildInt32(valueNode.asInt(), Integer.BYTES, columnValueBuilder); // columnValueBuilder.setType(PixelsProto.Type.newBuilder() @@ -197,168 +213,113 @@ private SinkProto.ColumnValue.Builder parseValue(JsonNode valueNode, String fiel return columnValueBuilder; } - @Deprecated // TODO: use bit - private SinkProto.ColumnValue.Builder parseValue(GenericRecord record, String fieldName, TypeDescription fieldType) + private SinkProto.ColumnValue.Builder parseCanonicalValue(Object raw, TypeDescription type) { - SinkProto.ColumnValue.Builder columnValueBuilder = SinkProto.ColumnValue.newBuilder(); - // columnValueBuilder.setName(fieldName); - - Object raw = record.get(fieldName); if (raw == null) { - columnValueBuilder.setValue(ByteString.EMPTY); - return columnValueBuilder; + return SinkProto.ColumnValue.newBuilder() + .setValue(ByteString.EMPTY) + .setIsNull(true); + } + if (raw instanceof JsonNode jsonNode) + { + return parseValue(jsonNode, "", type); } - switch (fieldType.getCategory()) + SinkProto.ColumnValue.Builder builder = SinkProto.ColumnValue.newBuilder(); + switch (type.getCategory()) { + case BYTE: + buildInt32(((Number) raw).intValue(), Byte.BYTES, builder); + break; + case SHORT: + buildInt32(((Number) raw).intValue(), Short.BYTES, builder); + break; case INT: - { - int value = (int) raw; - columnValueBuilder.setValue(ByteString.copyFrom(Integer.toString(value), StandardCharsets.UTF_8)); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.INT)); + buildInt32(((Number) raw).intValue(), Integer.BYTES, builder); break; - } - case LONG: + case TIMESTAMP: { - long value = (long) raw; - columnValueBuilder.setValue(ByteString.copyFrom(Long.toString(value), StandardCharsets.UTF_8)); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.LONG)); + byte[] bytes = ByteBuffer.allocate(Long.BYTES) + .putLong(((Number) raw).longValue()).array(); + builder.setValue(ByteString.copyFrom(bytes)); break; } - + case DATE: + case TIME: + buildInt32(((Number) raw).intValue(), Integer.BYTES, builder); + break; + case CHAR: + case VARCHAR: case STRING: - { - String value = raw.toString(); - columnValueBuilder.setValue(ByteString.copyFrom(value, StandardCharsets.UTF_8)); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.STRING)); + builder.setValue(ByteString.copyFrom(raw.toString(), StandardCharsets.UTF_8)); break; - } - case DECIMAL: { - ByteBuffer buffer = (ByteBuffer) raw; - String decimalStr = new String(buffer.array(), StandardCharsets.UTF_8).trim(); - columnValueBuilder.setValue(ByteString.copyFrom(decimalStr, StandardCharsets.UTF_8)); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder() -// .setKind(PixelsProto.Type.Kind.DECIMAL) -// .setDimension(fieldType.getPrecision()) -// .setScale(fieldType.getScale())); + BigDecimal decimal = toBigDecimal(raw, type); + builder.setValue(ByteString.copyFrom( + decimal.toPlainString(), StandardCharsets.UTF_8)); break; } - - case DATE: - { - int epochDay = (int) raw; - String isoDate = LocalDate.ofEpochDay(epochDay).toString(); // e.g., "2025-07-03" - columnValueBuilder.setValue(ByteString.copyFrom(isoDate, StandardCharsets.UTF_8)); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.DATE)); - break; - } - case BINARY: - { - ByteBuffer buffer = (ByteBuffer) raw; - // encode as hex or base64 if needed, otherwise just dump as UTF-8 string if it's meant to be readable - String base64 = Base64.getEncoder().encodeToString(buffer.array()); - columnValueBuilder.setValue(ByteString.copyFrom(base64, StandardCharsets.UTF_8)); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.BINARY)); - break; - } - default: - throw new IllegalArgumentException("Unsupported type: " + fieldType.getCategory()); - } - - return columnValueBuilder; - } - - private SinkProto.ColumnValue.Builder parseValue(Object record, String fieldName, Schema.Type type) - { - // TODO(AntiO2) support pixels type - if (record == null) - { - return SinkProto.ColumnValue.newBuilder() - // .setName(fieldName) - .setValue(ByteString.EMPTY); - } - - SinkProto.ColumnValue.Builder columnValueBuilder = SinkProto.ColumnValue.newBuilder(); - switch (type) - { - case INT8: - case INT16: - { - buildInt32((Short) record, Integer.BYTES, columnValueBuilder); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.INT)); + case VARBINARY: + builder.setValue(ByteString.copyFrom(toBytes(raw))); break; - } - case INT32: + case DOUBLE: { - buildInt32((Integer) record, Integer.BYTES, columnValueBuilder); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.INT)); + long bits = Double.doubleToLongBits(((Number) raw).doubleValue()); + builder.setValue(ByteString.copyFrom( + ByteBuffer.allocate(Long.BYTES).putLong(bits).array())); break; } - case INT64: - { - long value = (Long) record; - byte[] bytes = ByteBuffer.allocate(Long.BYTES).putLong(value).array(); - columnValueBuilder.setValue(ByteString.copyFrom(bytes)); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.LONG)); - break; - } - case BYTES: - { - if (record instanceof BigDecimal) - { - float value = ((BigDecimal) record).floatValue(); - buildFloat32(value, columnValueBuilder); - break; - } - byte[] bytes = (byte[]) record; - columnValueBuilder.setValue(ByteString.copyFrom(bytes)); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.BYTE)); + case FLOAT: + buildFloat32(((Number) raw).floatValue(), builder); break; - } case BOOLEAN: - case STRING: - { - String value = (String) record; - columnValueBuilder.setValue(ByteString.copyFrom(value, StandardCharsets.UTF_8)); -// columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.STRING)); + builder.setValue(ByteString.copyFrom( + new byte[]{(byte) ((Boolean) raw ? 1 : 0)})); break; - } case STRUCT: - { - // You can recursively parse fields in a struct here throw new UnsupportedOperationException("STRUCT parsing not yet implemented"); - } - case FLOAT64: - { - double value = (double) record; - long doubleBits = Double.doubleToLongBits(value); - byte[] bytes = ByteBuffer.allocate(Long.BYTES).putLong(doubleBits).array(); - columnValueBuilder.setValue(ByteString.copyFrom(bytes)); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.DOUBLE)); - break; - } - case FLOAT32: - { - float value = (float) record; - buildFloat32(value, columnValueBuilder); - // columnValueBuilder.setType(PixelsProto.Type.newBuilder().setKind(PixelsProto.Type.Kind.FLOAT)); - break; - } default: - throw new IllegalArgumentException("Unsupported type: " + type); + throw new IllegalArgumentException( + "Unsupported canonical type: " + type.getCategory()); } + return builder; + } - return columnValueBuilder; + private BigDecimal toBigDecimal(Object raw, TypeDescription type) + { + if (raw instanceof BigDecimal decimal) + { + return decimal; + } + if (raw instanceof ByteBuffer buffer) + { + return new BigDecimal(new BigInteger(toBytes(buffer)), type.getScale()); + } + if (raw instanceof byte[] bytes) + { + return new BigDecimal(new BigInteger(bytes), type.getScale()); + } + return new BigDecimal(raw.toString()); } - private Map parseDeleteRecord() + private byte[] toBytes(Object raw) { - return Collections.singletonMap("__deleted", true); + if (raw instanceof byte[] bytes) + { + return bytes; + } + if (raw instanceof ByteBuffer buffer) + { + ByteBuffer copy = buffer.duplicate(); + byte[] bytes = new byte[copy.remaining()]; + copy.get(bytes); + return bytes; + } + return raw.toString().getBytes(StandardCharsets.UTF_8); } BigDecimal parseDecimal(JsonNode node, TypeDescription type) @@ -368,19 +329,4 @@ BigDecimal parseDecimal(JsonNode node, TypeDescription type) return new BigDecimal(new BigInteger(bytes), scale); } - private LocalDate parseDate(JsonNode node) - { - return LocalDate.ofEpochDay(node.asLong()); - } - - private byte[] parseBinary(JsonNode node) - { - try - { - return node.binaryValue(); - } catch (IOException e) - { - throw new RuntimeException("Binary parsing failed", e); - } - } -} \ No newline at end of file +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventAvroDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventAvroDeserializer.java new file mode 100644 index 0000000..53ee7d1 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventAvroDeserializer.java @@ -0,0 +1,31 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.conversion.debezium; + +/** + * @deprecated Use the Kafka-boundary facade in + * {@code source.kafka.serde.RowChangeEventAvroDeserializer}. + */ +@Deprecated +public class RowChangeEventAvroDeserializer + extends io.pixelsdb.pixels.sink.source.kafka.serde.RowChangeEventAvroDeserializer +{ +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventJsonDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventJsonDeserializer.java new file mode 100644 index 0000000..a605720 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventJsonDeserializer.java @@ -0,0 +1,31 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.conversion.debezium; + +/** + * @deprecated Use the Kafka-boundary facade in + * {@code source.kafka.serde.RowChangeEventJsonDeserializer}. + */ +@Deprecated +public class RowChangeEventJsonDeserializer + extends io.pixelsdb.pixels.sink.source.kafka.serde.RowChangeEventJsonDeserializer +{ +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventStructConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventStructConverter.java new file mode 100644 index 0000000..5ef8e1f --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventStructConverter.java @@ -0,0 +1,157 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.conversion.debezium; + + +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.errors.DataException; +import org.apache.kafka.connect.source.SourceRecord; + +import java.util.logging.Logger; + +/** + * @package: io.pixelsdb.pixels.sink.conversion.debezium + * @className: RowChangeEventStructConverter + * @author: AntiO2 + * @date: 2025/9/26 12:00 + */ +public class RowChangeEventStructConverter +{ + private static final Logger LOGGER = Logger.getLogger(RowChangeEventStructConverter.class.getName()); + private final TableMetadataRegistry tableMetadataRegistry; + private final DebeziumSourceAdapter configuredAdapter; + + public RowChangeEventStructConverter() + { + this(TableMetadataRegistry.Instance(), null); + } + + public RowChangeEventStructConverter(TableMetadataRegistry tableMetadataRegistry) + { + this(tableMetadataRegistry, null); + } + + public RowChangeEventStructConverter( + TableMetadataRegistry tableMetadataRegistry, + DebeziumSourceAdapter configuredAdapter) + { + this.tableMetadataRegistry = tableMetadataRegistry; + this.configuredAdapter = configuredAdapter; + } + + public static RowChangeEvent convertToRowChangeEvent(SourceRecord sourceRecord) throws SinkException + { + return new RowChangeEventStructConverter().convert(sourceRecord); + } + + public RowChangeEvent convert(SourceRecord sourceRecord) throws SinkException + { + if (!(sourceRecord.value() instanceof Struct value)) + { + throw new SinkException("Debezium row value must be a Struct"); + } + String op = DebeziumRecordUtil.getStringSafely(value, "op"); + SinkProto.OperationType operationType = DebeziumRecordUtil.getOperationType(op); + return buildRowRecord(value, operationType); + } + + private RowChangeEvent buildRowRecord(Struct value, + SinkProto.OperationType opType) throws SinkException + { + + String schemaName; + String tableName; + DebeziumSourceAdapter adapter; + SinkProto.SourceInfo sourceInfo; + try + { + Struct source = value.getStruct("source"); + if (source == null) + { + throw new SinkException("Missing source field in row record"); + } + adapter = configuredAdapter == null + ? DebeziumEnvelopeNormalizer.adapterForSource(source) + : configuredAdapter; + sourceInfo = DebeziumEnvelopeNormalizer.normalizeSource(source, adapter); + schemaName = sourceInfo.getDb(); + tableName = sourceInfo.getTable(); + } catch (DataException | IllegalArgumentException e) + { + LOGGER.warning("Missing source field in row record"); + throw new SinkException(e); + } + + TableMetadata metadata = tableMetadataRegistry.getMetadata(schemaName, tableName); + TypeDescription typeDescription = metadata.getTypeDescription(); + DebeziumRowValueConverter rowDataParser = new DebeziumRowValueConverter(typeDescription); + + SinkProto.TransactionInfo transactionInfo = null; + try + { + Struct transaction = value.getStruct("transaction"); + if (transaction != null) + { + transactionInfo = DebeziumEnvelopeNormalizer.normalizeTransaction(transaction, adapter); + } + } catch (DataException e) + { + LOGGER.warning("Missing transaction field in row record"); + } + + SinkProto.RowValue beforeValue = null; + if (DebeziumRecordUtil.hasBeforeValue(opType)) + { + Struct before = value.getStruct("before"); + if (before == null) + { + throw new SinkException("Missing before image for " + opType); + } + SinkProto.RowValue.Builder beforeBuilder = SinkProto.RowValue.newBuilder(); + rowDataParser.parse(before, beforeBuilder); + beforeValue = beforeBuilder.build(); + } + + SinkProto.RowValue afterValue = null; + if (DebeziumRecordUtil.hasAfterValue(opType)) + { + Struct after = value.getStruct("after"); + if (after == null) + { + throw new SinkException("Missing after image for " + opType); + } + SinkProto.RowValue.Builder afterBuilder = SinkProto.RowValue.newBuilder(); + rowDataParser.parse(after, afterBuilder); + afterValue = afterBuilder.build(); + } + + return new DebeziumRowRecordAssembler().assemble( + opType, sourceInfo, transactionInfo, beforeValue, afterValue, + typeDescription, metadata); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/TransactionMetadataAvroDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/TransactionMetadataAvroDeserializer.java new file mode 100644 index 0000000..ac3ca38 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/TransactionMetadataAvroDeserializer.java @@ -0,0 +1,31 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.conversion.debezium; + +/** + * @deprecated Use the Kafka-boundary facade in + * {@code source.kafka.serde.TransactionMetadataAvroDeserializer}. + */ +@Deprecated +public class TransactionMetadataAvroDeserializer + extends io.pixelsdb.pixels.sink.source.kafka.serde.TransactionMetadataAvroDeserializer +{ +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/TransactionMetadataJsonDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/TransactionMetadataJsonDeserializer.java new file mode 100644 index 0000000..2287be1 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/TransactionMetadataJsonDeserializer.java @@ -0,0 +1,31 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.conversion.debezium; + +/** + * @deprecated Use the Kafka-boundary facade in + * {@code source.kafka.serde.TransactionMetadataJsonDeserializer}. + */ +@Deprecated +public class TransactionMetadataJsonDeserializer + extends io.pixelsdb.pixels.sink.source.kafka.serde.TransactionMetadataJsonDeserializer +{ +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceAdapter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceAdapter.java new file mode 100644 index 0000000..26c13e4 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceAdapter.java @@ -0,0 +1,39 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.source; + +import io.pixelsdb.pixels.sink.SinkProto; + +import java.util.Locale; + +public interface DebeziumSourceAdapter +{ + String connector(); + + default boolean supports(String connectorName) + { + if (connectorName == null || connectorName.isBlank()) + { + return false; + } + String normalized = connectorName.toLowerCase(Locale.ROOT); + return normalized.equals(connector()) || normalized.contains(connector()); + } + + String normalizeTransactionId(String sourceId); + + SinkProto.SourceInfo normalizeSource(DebeziumSourceMetadata source); + + default String normalizeDataCollection(String sourceDataCollection) + { + return sourceDataCollection; + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceAdapterRegistry.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceAdapterRegistry.java new file mode 100644 index 0000000..1f841f5 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceAdapterRegistry.java @@ -0,0 +1,57 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.source; + +import java.util.List; +import java.util.ServiceLoader; + +public final class DebeziumSourceAdapterRegistry +{ + private static final List ADAPTERS = ServiceLoader + .load(DebeziumSourceAdapter.class) + .stream() + .map(ServiceLoader.Provider::get) + .toList(); + + private DebeziumSourceAdapterRegistry() + { + } + + public static DebeziumSourceAdapter forSource(String connector) + { + if (connector != null && !connector.isBlank()) + { + return resolve(connector); + } + return configured(); + } + + public static DebeziumSourceAdapter resolve(String connector) + { + if (connector == null || connector.isBlank()) + { + throw new IllegalArgumentException("Unsupported Debezium connector: " + connector); + } + + List matches = ADAPTERS.stream() + .filter(adapter -> adapter.supports(connector)) + .toList(); + if (matches.size() == 1) + { + return matches.get(0); + } + if (matches.size() > 1) + { + throw new IllegalStateException("Multiple Debezium connector adapters support: " + connector); + } + throw new IllegalArgumentException("Unsupported Debezium connector: " + connector); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceMetadata.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceMetadata.java new file mode 100644 index 0000000..2ff3c92 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceMetadata.java @@ -0,0 +1,19 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.source; + +public record DebeziumSourceMetadata( + String connector, + String db, + String schema, + String table) +{ +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/MySqlSourceAdapter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/MySqlSourceAdapter.java new file mode 100644 index 0000000..4aaa359 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/MySqlSourceAdapter.java @@ -0,0 +1,43 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.source; + +import io.pixelsdb.pixels.sink.SinkProto; + +public final class MySqlSourceAdapter implements DebeziumSourceAdapter +{ + public static final MySqlSourceAdapter INSTANCE = new MySqlSourceAdapter(); + + public MySqlSourceAdapter() + { + } + + @Override + public String connector() + { + return "mysql"; + } + + @Override + public String normalizeTransactionId(String sourceId) + { + return sourceId; + } + + @Override + public SinkProto.SourceInfo normalizeSource(DebeziumSourceMetadata source) + { + return SinkProto.SourceInfo.newBuilder() + .setDb(source.db()) + .setTable(source.table()) + .build(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/PostgresSourceAdapter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/PostgresSourceAdapter.java new file mode 100644 index 0000000..12e3ed3 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/PostgresSourceAdapter.java @@ -0,0 +1,49 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + */ +package io.pixelsdb.pixels.sink.conversion.debezium.source; + +import io.pixelsdb.pixels.sink.SinkProto; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public final class PostgresSourceAdapter implements DebeziumSourceAdapter +{ + public static final PostgresSourceAdapter INSTANCE = new PostgresSourceAdapter(); + private static final Pattern TRANSACTION_WITH_LSN = Pattern.compile("^(\\d+):\\d+$"); + + public PostgresSourceAdapter() + { + } + + @Override + public String connector() + { + return "postgresql"; + } + + @Override + public String normalizeTransactionId(String sourceId) + { + Matcher matcher = TRANSACTION_WITH_LSN.matcher(sourceId); + return matcher.matches() ? matcher.group(1) : sourceId; + } + + @Override + public SinkProto.SourceInfo normalizeSource(DebeziumSourceMetadata source) + { + return SinkProto.SourceInfo.newBuilder() + .setDb(source.db()) + .setSchema(source.schema()) + .setTable(source.table()) + .build(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/kafka/KafkaRecordConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/kafka/KafkaRecordConverter.java new file mode 100644 index 0000000..3474be8 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/kafka/KafkaRecordConverter.java @@ -0,0 +1,58 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.conversion.kafka; + +import java.util.Properties; + +/** + * @deprecated Kafka source wiring belongs to {@code source.kafka.serde}. + * This facade keeps the old SPI name usable by existing deployments. + */ +@Deprecated +public final class KafkaRecordConverter implements AutoCloseable +{ + private final io.pixelsdb.pixels.sink.source.kafka.serde.KafkaRecordConverter delegate; + + private KafkaRecordConverter( + io.pixelsdb.pixels.sink.source.kafka.serde.KafkaRecordConverter delegate) + { + this.delegate = delegate; + } + + public static KafkaRecordConverter create( + Properties properties, String converterClassKey) + { + return new KafkaRecordConverter<>( + io.pixelsdb.pixels.sink.source.kafka.serde.KafkaRecordConverter.create( + properties, converterClassKey)); + } + + public T convert(String topic, byte[] data) + { + return delegate.convert(topic, data); + } + + @Override + public void close() + { + delegate.close(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverter.java new file mode 100644 index 0000000..79715be --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverter.java @@ -0,0 +1,56 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.conversion.sinkproto; + +import com.google.protobuf.InvalidProtocolBufferException; +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.event.RowChangeEventFactory; +import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; + +import java.nio.ByteBuffer; + +public final class RowRecordConverter +{ + private final TableMetadataRegistry tableMetadataRegistry; + + public RowRecordConverter(TableMetadataRegistry tableMetadataRegistry) + { + this.tableMetadataRegistry = tableMetadataRegistry; + } + + public RowChangeEvent convert(SinkProto.RowRecord rowRecord) throws SinkException + { + SinkProto.SourceInfo source = rowRecord.getSource(); + TableMetadata metadata = tableMetadataRegistry.getMetadata( + source.getDb(), source.getTable()); + TypeDescription schema = metadata.getTypeDescription(); + return RowChangeEventFactory.create(rowRecord, schema, metadata); + } + + public SinkProto.RowRecord parse(ByteBuffer buffer) throws InvalidProtocolBufferException + { + return SinkProto.RowRecord.parseFrom(buffer.duplicate()); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/TransactionMetadataConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/TransactionMetadataConverter.java new file mode 100644 index 0000000..c3d5341 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/TransactionMetadataConverter.java @@ -0,0 +1,39 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.conversion.sinkproto; + +import com.google.protobuf.InvalidProtocolBufferException; +import io.pixelsdb.pixels.sink.SinkProto; + +import java.nio.ByteBuffer; + +public final class TransactionMetadataConverter +{ + public SinkProto.TransactionMetadata convert(ByteBuffer buffer, int loopId) + throws InvalidProtocolBufferException + { + SinkProto.TransactionMetadata metadata = + SinkProto.TransactionMetadata.parseFrom(buffer.duplicate()); + return metadata.toBuilder() + .setId(metadata.getId() + "_" + loopId) + .build(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEvent.java b/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEvent.java index b3588fd..8800ad9 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEvent.java +++ b/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEvent.java @@ -30,8 +30,6 @@ import io.pixelsdb.pixels.sink.exception.SinkException; import io.pixelsdb.pixels.sink.metadata.TableMetadata; import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; -import io.pixelsdb.pixels.sink.util.MetricsFacade; -import io.prometheus.client.Summary; import lombok.Getter; import lombok.Setter; @@ -47,7 +45,6 @@ public class RowChangeEvent @Getter private final SinkProto.RowRecord rowRecord; - private final MetricsFacade metricsFacade = MetricsFacade.getInstance(); @Getter private final TypeDescription schema; /** @@ -60,7 +57,6 @@ public class RowChangeEvent private String topic; @Getter private TableMetadata tableMetadata = null; - private Summary.Timer latencyTimer; private Map beforeValueMap; private Map afterValueMap; @Getter @@ -78,20 +74,31 @@ public class RowChangeEvent public RowChangeEvent(SinkProto.RowRecord rowRecord) throws SinkException { - this.rowRecord = rowRecord; TableMetadataRegistry tableMetadataRegistry = TableMetadataRegistry.Instance(); - this.schema = tableMetadataRegistry.getTypeDescription(getSchemaName(), getTable()); + this.rowRecord = rowRecord; + this.schema = tableMetadataRegistry.getTypeDescription( + rowRecord.getSource().getDb(), rowRecord.getSource().getTable()); + this.tableMetadata = tableMetadataRegistry.getMetadata( + rowRecord.getSource().getDb(), rowRecord.getSource().getTable()); init(); initIndexKey(); } public RowChangeEvent(SinkProto.RowRecord rowRecord, TypeDescription schema) throws SinkException + { + this(rowRecord, schema, TableMetadataRegistry.Instance().getMetadata( + rowRecord.getSource().getDb(), rowRecord.getSource().getTable())); + } + + public RowChangeEvent( + SinkProto.RowRecord rowRecord, + TypeDescription schema, + TableMetadata tableMetadata) throws SinkException { this.rowRecord = rowRecord; this.schema = schema; - + this.tableMetadata = tableMetadata; init(); - // initIndexKey(); } protected static int getBucketFromIndexKey(IndexProto.IndexKey indexKey) @@ -106,8 +113,7 @@ protected static int getBucketIdFromByteBuffer(ByteString byteString) private void init() throws SinkException { - TableMetadataRegistry tableMetadataRegistry = TableMetadataRegistry.Instance(); - this.tableId = tableMetadataRegistry.getTableId(getSchemaName(), getTable()); + this.tableId = tableMetadata == null ? 0 : tableMetadata.getTableId(); this.schemaTableName = new SchemaTableName(getSchemaName(), getTable()); initColumnValueMap(); @@ -141,9 +147,10 @@ public void initIndexKey() throws SinkException return; } - this.tableMetadata = TableMetadataRegistry.Instance().getMetadata( - this.rowRecord.getSource().getDb(), - this.rowRecord.getSource().getTable()); + if (this.tableMetadata == null) + { + throw new SinkException("Row change table metadata is missing"); + } if (!this.tableMetadata.hasPrimaryIndex()) { @@ -258,16 +265,14 @@ public String getTable() public String getFullTableName() { - // TODO(AntiO2): In postgresql, data collection uses schemaName as prefix, while MySQL uses DB as prefix. - return rowRecord.getSource().getSchema() + "." + rowRecord.getSource().getTable(); - // return getSchemaName() + "." + getTable(); + SinkProto.SourceInfo source = rowRecord.getSource(); + String namespace = source.getSchema().isBlank() ? source.getDb() : source.getSchema(); + return namespace + "." + source.getTable(); } - // TODO(AntiO2): How to Map Schema Names Between Source DB and Pixels public String getSchemaName() { return rowRecord.getSource().getDb(); - // return rowRecord.getSource().getSchema(); } public boolean hasError() @@ -310,20 +315,6 @@ public boolean hasAfterData() return isUpdate() || isInsert() || isSnapshot(); } - public void startLatencyTimer() - { - this.latencyTimer = metricsFacade.startProcessLatencyTimer(); - } - - public void endLatencyTimer() - { - if (latencyTimer != null) - { - this.latencyTimer.close(); - } - - } - public SinkProto.OperationType getOp() { return rowRecord.getOp(); diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactory.java b/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactory.java new file mode 100644 index 0000000..f69dec0 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactory.java @@ -0,0 +1,106 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.event; + +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; + +public final class RowChangeEventFactory +{ + private RowChangeEventFactory() + { + } + + public static RowChangeEvent create( + SinkProto.OperationType operation, + SinkProto.SourceInfo source, + SinkProto.TransactionInfo transaction, + SinkProto.RowValue before, + SinkProto.RowValue after, + TypeDescription schema) throws SinkException + { + if (operation == null) + { + throw new SinkException("Row change operation is missing"); + } + if (source == null) + { + throw new SinkException("Row change source is missing"); + } + + SinkProto.RowRecord.Builder builder = SinkProto.RowRecord.newBuilder() + .setOp(operation) + .setSource(source); + if (transaction != null) + { + builder.setTransaction(transaction); + } + if (before != null) + { + builder.setBefore(before); + } + if (after != null) + { + builder.setAfter(after); + } + return create(builder.build(), schema); + } + + public static RowChangeEvent create( + SinkProto.RowRecord rowRecord, + TypeDescription schema) throws SinkException + { + if (rowRecord == null) + { + throw new SinkException("Row change record is missing"); + } + if (schema == null) + { + throw new SinkException("Row change schema is missing"); + } + TableMetadataRegistry registry = TableMetadataRegistry.Instance(); + TableMetadata metadata = registry.getMetadata( + rowRecord.getSource().getDb(), rowRecord.getSource().getTable()); + return create(rowRecord, schema, metadata); + } + + public static RowChangeEvent create( + SinkProto.RowRecord rowRecord, + TypeDescription schema, + TableMetadata metadata) throws SinkException + { + if (rowRecord == null) + { + throw new SinkException("Row change record is missing"); + } + if (schema == null) + { + throw new SinkException("Row change schema is missing"); + } + + RowChangeEvent event = new RowChangeEvent(rowRecord, schema, metadata); + event.initIndexKey(); + return event; + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventAvroDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventAvroDeserializer.java deleted file mode 100644 index 7a2711a..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventAvroDeserializer.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - -import io.apicurio.registry.serde.SerdeConfig; -import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; -import io.pixelsdb.pixels.core.TypeDescription; -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.exception.SinkException; -import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; -import io.pixelsdb.pixels.sink.util.MetricsFacade; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.common.serialization.Deserializer; - -import java.util.HashMap; -import java.util.Map; - -public class RowChangeEventAvroDeserializer implements Deserializer -{ - - private final AvroKafkaDeserializer avroDeserializer = new AvroKafkaDeserializer<>(); - private final TableMetadataRegistry tableMetadataRegistry = TableMetadataRegistry.Instance(); - private final PixelsSinkConfig config = PixelsSinkConfigFactory.getInstance(); - - @Override - public void configure(Map configs, boolean isKey) - { - Map enrichedConfig = new HashMap<>(configs); - enrichedConfig.put(SerdeConfig.REGISTRY_URL, config.getRegistryUrl()); - enrichedConfig.put(SerdeConfig.CHECK_PERIOD_MS, SerdeConfig.CHECK_PERIOD_MS_DEFAULT); - avroDeserializer.configure(enrichedConfig, isKey); - } - - @Override - public RowChangeEvent deserialize(String topic, byte[] data) - { - try - { - MetricsFacade.getInstance().addRawData(data.length); - GenericRecord avroRecord = avroDeserializer.deserialize(topic, data); - Schema avroSchema = avroRecord.getSchema(); - RowChangeEvent rowChangeEvent = convertToRowChangeEvent(avroRecord, avroSchema); - return rowChangeEvent; - } catch (Exception e) - { - e.printStackTrace(); - return null; - // throw new SerializationException("Avro deserialization failed", e); - // TODO: 这里有些Row Change Event 不是完整的结构。 - } - } - - private void registerSchema(String topic, Schema avroSchema) - { - - } - - private RowChangeEvent convertToRowChangeEvent(GenericRecord avroRecord, Schema schema) throws SinkException - { - SinkProto.OperationType op = parseOperationType(avroRecord); - SinkProto.RowRecord.Builder recordBuilder = SinkProto.RowRecord.newBuilder() - .setOp(op) -// .setTsMs(DeserializerUtil.getLongSafely(avroRecord, "ts_ms")); - ; - if (avroRecord.get("source") != null) - { - //TODO: 这里看下怎么处理，如果没有source信息，其实可以通过topic推出schema和table信息。 - parseSourceInfo((GenericRecord) avroRecord.get("source"), recordBuilder.getSourceBuilder()); - } - - String sourceSchema = recordBuilder.getSource().getDb(); - String sourceTable = recordBuilder.getSource().getTable(); - TypeDescription typeDescription = null; - try - { - typeDescription = tableMetadataRegistry.getTypeDescription(sourceSchema, sourceTable); - } catch (SinkException e) - { - throw new RuntimeException(e); - } - // TableMetadata tableMetadata = tableMetadataRegistry.loadTableMetadata(sourceSchema, sourceTable); - - recordBuilder.setBefore(parseRowData(avroRecord.get("before"), typeDescription)); - recordBuilder.setAfter(parseRowData(avroRecord.get("after"), typeDescription)); - - if (avroRecord.get("transaction") != null) - { - parseTransactionInfo((GenericRecord) avroRecord.get("transaction"), - recordBuilder.getTransactionBuilder()); - } - - return new RowChangeEvent(recordBuilder.build(), typeDescription); - } - - private SinkProto.OperationType parseOperationType(GenericRecord record) - { - String op = DeserializerUtil.getStringSafely(record, "op"); - try - { - return DeserializerUtil.getOperationType(op); - } catch (IllegalArgumentException e) - { - return SinkProto.OperationType.UNRECOGNIZED; - } - } - - private SinkProto.RowValue.Builder parseRowData(Object data, TypeDescription typeDescription) - { - SinkProto.RowValue.Builder builder = SinkProto.RowValue.newBuilder(); - if (data instanceof GenericRecord rowData) - { - RowDataParser rowDataParser = new RowDataParser(typeDescription); // TODO make it static? - rowDataParser.parse(rowData, builder); - } - return builder; - } - - private void parseSourceInfo(GenericRecord source, SinkProto.SourceInfo.Builder builder) - { - - builder - .setDb(DeserializerUtil.getStringSafely(source, "db")) - .setSchema(DeserializerUtil.getStringSafely(source, "schema")) - .setTable(DeserializerUtil.getStringSafely(source, "table")) -// .setVersion(DeserializerUtil.getStringSafely(source, "version")) -// .setConnector(DeserializerUtil.getStringSafely(source, "connector")) -// .setName(DeserializerUtil.getStringSafely(source, "name")) -// .setTsMs(DeserializerUtil.getLongSafely(source, "ts_ms")) -// .setSnapshot(DeserializerUtil.getStringSafely(source, "snapshot")) -// -// .setSequence(DeserializerUtil.getStringSafely(source, "sequence")) - -// .setTxId(DeserializerUtil.getLongSafely(source, "tx_id")) -// .setLsn(DeserializerUtil.getLongSafely(source, "lsn")) -// .setXmin(DeserializerUtil.getLongSafely(source, "xmin")) - ; - } - - private void parseTransactionInfo(GenericRecord transaction, - SinkProto.TransactionInfo.Builder builder) - { - builder.setId(DeserializerUtil.getTransIdPrefix(DeserializerUtil.getStringSafely(transaction, "id"))) - .setTotalOrder(DeserializerUtil.getLongSafely(transaction, "total_order")) - .setDataCollectionOrder(DeserializerUtil.getLongSafely(transaction, "data_collection_order")); - } - -} \ No newline at end of file diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventJsonDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventJsonDeserializer.java deleted file mode 100644 index 6d55297..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventJsonDeserializer.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.pixelsdb.pixels.core.TypeDescription; -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.exception.SinkException; -import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; -import io.pixelsdb.pixels.sink.util.MetricsFacade; -import org.apache.kafka.common.serialization.Deserializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -public class RowChangeEventJsonDeserializer implements Deserializer -{ - private static final Logger logger = LoggerFactory.getLogger(RowChangeEventJsonDeserializer.class); - private static final ObjectMapper objectMapper = new ObjectMapper(); - private final TableMetadataRegistry tableMetadataRegistry = TableMetadataRegistry.Instance(); - - @Override - public RowChangeEvent deserialize(String topic, byte[] data) - { - if (data == null || data.length == 0) - { - logger.debug("Received empty message from topic: {}", topic); - return null; - } - MetricsFacade.getInstance().addRawData(data.length); - try - { - JsonNode rootNode = objectMapper.readTree(data); - JsonNode payloadNode = rootNode.path("payload"); - - SinkProto.OperationType opType = parseOperationType(payloadNode); - - return buildRowRecord(payloadNode, opType); - } catch (Exception e) - { - logger.error("Failed to deserialize message from topic {}: {}", topic, e.getMessage()); - return null; - } - } - - private SinkProto.OperationType parseOperationType(JsonNode payloadNode) - { - String opCode = payloadNode.path("op").asText(""); - return DeserializerUtil.getOperationType(opCode); - } - - @Deprecated - private TypeDescription getSchema(JsonNode schemaNode, SinkProto.OperationType opType) - { - return switch (opType) - { - case DELETE -> SchemaDeserializer.parseFromBeforeOrAfter(schemaNode, "before"); - case INSERT, UPDATE, SNAPSHOT -> SchemaDeserializer.parseFromBeforeOrAfter(schemaNode, "after"); - case UNRECOGNIZED -> throw new IllegalArgumentException("Operation type is unknown. Check op"); - }; - } - - private RowChangeEvent buildRowRecord(JsonNode payloadNode, - SinkProto.OperationType opType) throws SinkException - { - - SinkProto.RowRecord.Builder builder = SinkProto.RowRecord.newBuilder(); - - builder.setOp(parseOperationType(payloadNode)); -// .setTsMs(payloadNode.path("ts_ms").asLong()) -// .setTsUs(payloadNode.path("ts_us").asLong()) -// .setTsNs(payloadNode.path("ts_ns").asLong()); - - String schemaName; - String tableName; - if (payloadNode.has("source")) - { - SinkProto.SourceInfo.Builder sourceInfoBuilder = parseSourceInfo(payloadNode.get("source")); - schemaName = sourceInfoBuilder.getDb(); // Notice we use the schema - tableName = sourceInfoBuilder.getTable(); - builder.setSource(sourceInfoBuilder); - } else - { - throw new IllegalArgumentException("Missing source field in row record"); - } - - TypeDescription typeDescription = tableMetadataRegistry.getTypeDescription(schemaName, tableName); - RowDataParser rowDataParser = new RowDataParser(typeDescription); - if (payloadNode.hasNonNull("transaction")) - { - builder.setTransaction(parseTransactionInfo(payloadNode.get("transaction"))); - } - - if (DeserializerUtil.hasBeforeValue(opType)) - { - SinkProto.RowValue.Builder beforeBuilder = builder.getBeforeBuilder(); - rowDataParser.parse(payloadNode.get("before"), beforeBuilder); - builder.setBefore(beforeBuilder); - } - - if (DeserializerUtil.hasAfterValue(opType)) - { - - SinkProto.RowValue.Builder afterBuilder = builder.getAfterBuilder(); - rowDataParser.parse(payloadNode.get("after"), afterBuilder); - builder.setAfter(afterBuilder); - } - - RowChangeEvent event = new RowChangeEvent(builder.build(), typeDescription); - try - { - event.initIndexKey(); - } catch (SinkException e) - { - logger.warn("Row change event {}: Init index key failed", event); - } - - return event; - } - - private SinkProto.SourceInfo.Builder parseSourceInfo(JsonNode sourceNode) - { - return SinkProto.SourceInfo.newBuilder() - .setDb(sourceNode.path("db").asText()) - .setSchema(sourceNode.path("schema").asText()) - .setTable(sourceNode.path("table").asText()) -// .setVersion(sourceNode.path("version").asText()) -// .setConnector(sourceNode.path("connector").asText()) -// .setName(sourceNode.path("name").asText()) -// .setTsMs(sourceNode.path("ts_ms").asLong()) -// .setSnapshot(sourceNode.path("snapshot").asText()) - -// .setSequence(sourceNode.path("sequence").asText()) -// .setTsUs(sourceNode.path("ts_us").asLong()) -// .setTsNs(sourceNode.path("ts_ns").asLong()) - -// .setTxId(sourceNode.path("txId").asLong()) -// .setLsn(sourceNode.path("lsn").asLong()) -// .setXmin(sourceNode.path("xmin").asLong()) - ; - } - - private SinkProto.TransactionInfo parseTransactionInfo(JsonNode txNode) - { - return SinkProto.TransactionInfo.newBuilder() - .setId(DeserializerUtil.getTransIdPrefix(txNode.path("id").asText())) - .setTotalOrder(txNode.path("total_order").asLong()) - .setDataCollectionOrder(txNode.path("data_collection_order").asLong()) - .build(); - } - -} - diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventStructDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventStructDeserializer.java deleted file mode 100644 index 498d382..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventStructDeserializer.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - - -import io.pixelsdb.pixels.core.TypeDescription; -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.exception.SinkException; -import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; -import org.apache.kafka.connect.data.Struct; -import org.apache.kafka.connect.errors.DataException; -import org.apache.kafka.connect.source.SourceRecord; - -import java.util.logging.Logger; - -/** - * @package: io.pixelsdb.pixels.sink.event.deserializer - * @className: RowChangeEventStructDeserializer - * @author: AntiO2 - * @date: 2025/9/26 12:00 - */ -public class RowChangeEventStructDeserializer -{ - private static final Logger LOGGER = Logger.getLogger(RowChangeEventStructDeserializer.class.getName()); - private static final TableMetadataRegistry tableMetadataRegistry = TableMetadataRegistry.Instance(); - - public static RowChangeEvent convertToRowChangeEvent(SourceRecord sourceRecord) throws SinkException - { - Struct value = (Struct) sourceRecord.value(); - String op = value.getString("op"); - SinkProto.OperationType operationType = DeserializerUtil.getOperationType(op); - return buildRowRecord(value, operationType); - } - - public static RowChangeEvent convertToRowChangeEvent(SinkProto.RowRecord rowRecord) throws SinkException - { - String schemaName = rowRecord.getSource().getDb(); - String tableName = rowRecord.getSource().getTable(); - TypeDescription typeDescription = tableMetadataRegistry.getTypeDescription(schemaName, tableName); - return new RowChangeEvent(rowRecord, typeDescription); - } - - private static RowChangeEvent buildRowRecord(Struct value, - SinkProto.OperationType opType) throws SinkException - { - - SinkProto.RowRecord.Builder builder = SinkProto.RowRecord.newBuilder(); - - builder.setOp(opType); - - String schemaName; - String tableName; - try - { - Struct source = value.getStruct("source"); - SinkProto.SourceInfo.Builder sourceInfoBuilder = parseSourceInfo(source); - schemaName = sourceInfoBuilder.getDb(); // Notice we use the schema - tableName = sourceInfoBuilder.getTable(); - builder.setSource(sourceInfoBuilder); - } catch (DataException e) - { - LOGGER.warning("Missing source field in row record"); - throw new SinkException(e); - } - - TypeDescription typeDescription = tableMetadataRegistry.getTypeDescription(schemaName, tableName); - RowDataParser rowDataParser = new RowDataParser(typeDescription); - - try - { - Struct transaction = value.getStruct("transaction"); - SinkProto.TransactionInfo transactionInfo = parseTransactionInfo(transaction); - builder.setTransaction(transactionInfo); - } catch (DataException e) - { - LOGGER.warning("Missing transaction field in row record"); - } - - if (DeserializerUtil.hasBeforeValue(opType)) - { - SinkProto.RowValue.Builder beforeBuilder = builder.getBeforeBuilder(); - rowDataParser.parse(value.getStruct("before"), beforeBuilder); - builder.setBefore(beforeBuilder); - } - - if (DeserializerUtil.hasAfterValue(opType)) - { - - SinkProto.RowValue.Builder afterBuilder = builder.getAfterBuilder(); - rowDataParser.parse(value.getStruct("after"), afterBuilder); - builder.setAfter(afterBuilder); - } - - RowChangeEvent event = new RowChangeEvent(builder.build(), typeDescription); - return event; - } - - private static SinkProto.SourceInfo.Builder parseSourceInfo(T source) - { - return SinkProto.SourceInfo.newBuilder() - // .setVersion(DeserializerUtil.getStringSafely(source, "version")) - // .setConnector(DeserializerUtil.getStringSafely(source, "connector")) -// .setName(DeserializerUtil.getStringSafely(source, "name")) -// .setTsMs(DeserializerUtil.getLongSafely(source, "ts_ms")) -// .setSnapshot(DeserializerUtil.getStringSafely(source, "snapshot")) - .setDb(DeserializerUtil.getStringSafely(source, "db")) -// .setSequence(DeserializerUtil.getStringSafely(source, "sequence")) -// .setTsUs(DeserializerUtil.getLongSafely(source, "ts_us")) -// .setTsNs(DeserializerUtil.getLongSafely(source, "ts_ns")) - .setSchema(DeserializerUtil.getStringSafely(source, "schema")) - .setTable(DeserializerUtil.getStringSafely(source, "table")); -// .setTxId(DeserializerUtil.getLongSafely(source, "txId")) -// .setLsn(DeserializerUtil.getLongSafely(source, "lsn")) -// .setXmin(DeserializerUtil.getLongSafely(source, "xmin")); - } - - private static SinkProto.TransactionInfo parseTransactionInfo(T txNode) - { - return SinkProto.TransactionInfo.newBuilder() - .setId(DeserializerUtil.getTransIdPrefix( - DeserializerUtil.getStringSafely(txNode, "id"))) - .setTotalOrder(DeserializerUtil.getLongSafely(txNode, "total_order")) - .setDataCollectionOrder(DeserializerUtil.getLongSafely(txNode, "data_collection_order")) - .build(); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/SchemaDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/SchemaDeserializer.java deleted file mode 100644 index d33f3fe..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/SchemaDeserializer.java +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - -import com.fasterxml.jackson.databind.JsonNode; -import io.pixelsdb.pixels.core.TypeDescription; -import org.apache.avro.Schema; - -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - -public class SchemaDeserializer -{ - public static TypeDescription parseFromBeforeOrAfter(JsonNode schemaNode, String fieldName) - { - JsonNode beforeAfterSchema = findSchemaField(schemaNode, fieldName); - if (beforeAfterSchema == null) - { - throw new IllegalArgumentException("Field '" + fieldName + "' not found in schema"); - } - return parseStruct(beforeAfterSchema.get("fields")); - } - - private static JsonNode findSchemaField(JsonNode schemaNode, String targetField) - { - Iterator fields = schemaNode.get("fields").elements(); - while (fields.hasNext()) - { - JsonNode field = fields.next(); - if (targetField.equals(field.get("field").asText())) - { - return field; - } - } - return null; - } - - public static TypeDescription parseStruct(JsonNode fields) - { - TypeDescription structType = TypeDescription.createStruct(); - fields.forEach(field -> - { - String name = field.get("field").asText(); - TypeDescription fieldType = parseFieldType(field); - structType.addField(name, fieldType); - }); - return structType; - } - - static TypeDescription parseFieldType(JsonNode fieldNode) - { - if (!fieldNode.has("type")) - { - throw new IllegalArgumentException("Field is missing required 'type' property"); - } - String typeName = fieldNode.get("type").asText(); - String logicalType = fieldNode.has("name") ? fieldNode.get("name").asText() : null; - - if (logicalType != null) - { - switch (logicalType) - { - case "org.apache.kafka.connect.data.Decimal": - int precision = Integer.parseInt(fieldNode.get("parameters").get("connect.decimal.precision").asText()); - int scale = Integer.parseInt(fieldNode.get("parameters").get("scale").asText()); - return TypeDescription.createDecimal(precision, scale); - case "io.debezium.time.Date": - return TypeDescription.createDate(); - } - } - - switch (typeName) - { - case "int64": - return TypeDescription.createLong(); - case "int32": - return TypeDescription.createInt(); - case "string": - return TypeDescription.createString(); - case "struct": - return parseStruct(fieldNode.get("fields")); - default: - throw new IllegalArgumentException("Unsupported type: " + typeName); - } - } - - public static TypeDescription parseFromBeforeOrAfter(Schema schemaNode, String fieldName) - { - Schema.Field filed = schemaNode.getField(fieldName); - if (filed == null) - { - throw new IllegalArgumentException("Can't find field in avro schema: " + fieldName); - } - - Schema valueSchema = filed.schema(); - return parseFromAvroSchema(valueSchema); - } - - - public static TypeDescription parseFromAvroSchema(Schema avroSchema) - { - return parseAvroType(avroSchema, new HashMap<>()); - } - - private static TypeDescription parseAvroType(Schema schema, Map cache) - { - String schemaKey = schema.getFullName() + ":" + schema.hashCode(); - if (cache.containsKey(schemaKey)) - { - return cache.get(schemaKey); - } - - TypeDescription typeDesc; - switch (schema.getType()) - { - case RECORD: - typeDesc = parseAvroRecord(schema, cache); - break; - case UNION: - typeDesc = parseAvroUnion(schema, cache); - break; - case ARRAY: - typeDesc = parseAvroArray(schema, cache); - break; - case MAP: - typeDesc = parseAvroMap(schema, cache); - break; - default: - typeDesc = parseAvroPrimitive(schema); - } - - cache.put(schemaKey, typeDesc); - return typeDesc; - } - - private static TypeDescription parseAvroRecord(Schema schema, Map cache) - { - TypeDescription structType = TypeDescription.createStruct(); - for (Schema.Field field : schema.getFields()) - { - TypeDescription fieldType = parseAvroType(field.schema(), cache); - structType.addField(field.name(), fieldType); - } - return structType; - } - - private static TypeDescription parseAvroUnion(Schema schema, Map cache) - { - for (Schema type : schema.getTypes()) - { - if (type.getType() != Schema.Type.NULL) - { - return parseAvroType(type, cache); - } - } - throw new IllegalArgumentException("Invalid union type: " + schema); - } - - private static TypeDescription parseAvroArray(Schema schema, Map cache) - { - throw new RuntimeException("Doesn't support Array"); - } - - private static TypeDescription parseAvroMap(Schema schema, Map cache) - { - throw new RuntimeException("Doesn't support Map"); - } - - private static TypeDescription parseAvroPrimitive(Schema schema) - { - String logicalType = schema.getLogicalType() != null ? - schema.getLogicalType().getName() : null; - - if (logicalType != null) - { - switch (logicalType) - { - case "decimal": - return TypeDescription.createDecimal( - (Integer) (schema.getObjectProp("precision")), - (Integer) (schema.getObjectProp("scale")) - ); - case "date": - return TypeDescription.createDate(); - case "timestamp-millis": - return TypeDescription.createTimestamp((Integer) (schema.getObjectProp("precision"))); - case "uuid": - return TypeDescription.createString(); - } - } - - switch (schema.getType()) - { - case LONG: - return TypeDescription.createLong(); - case INT: - return TypeDescription.createInt(); - case STRING: - return TypeDescription.createString(); - case BOOLEAN: - return TypeDescription.createBoolean(); - case FLOAT: - return TypeDescription.createFloat(); - case DOUBLE: - return TypeDescription.createDouble(); - case BYTES: - // return TypeDescription.createBinary(); - default: - throw new IllegalArgumentException("Unsupported Avro type: " + schema); - } - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionAvroMessageDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionAvroMessageDeserializer.java deleted file mode 100644 index ffdf3f6..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionAvroMessageDeserializer.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - -import io.apicurio.registry.serde.SerdeConfig; -import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import io.pixelsdb.pixels.sink.util.MetricsFacade; -import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.common.errors.SerializationException; -import org.apache.kafka.common.serialization.Deserializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.HashMap; -import java.util.Map; - -public class TransactionAvroMessageDeserializer implements Deserializer -{ - private static final Logger logger = LoggerFactory.getLogger(TransactionAvroMessageDeserializer.class); - private final AvroKafkaDeserializer avroDeserializer = new AvroKafkaDeserializer<>(); - private final PixelsSinkConfig config = PixelsSinkConfigFactory.getInstance(); - - @Override - public void configure(Map configs, boolean isKey) - { - Map enrichedConfig = new HashMap<>(configs); - enrichedConfig.put(SerdeConfig.REGISTRY_URL, config.getRegistryUrl()); - enrichedConfig.put(SerdeConfig.CHECK_PERIOD_MS, SerdeConfig.CHECK_PERIOD_MS_DEFAULT); - avroDeserializer.configure(enrichedConfig, isKey); - } - - @Override - public SinkProto.TransactionMetadata deserialize(String topic, byte[] bytes) - { - if (bytes == null || bytes.length == 0) - { - return null; - } - try - { - MetricsFacade.getInstance().addRawData(bytes.length); - GenericRecord avroRecord = avroDeserializer.deserialize(topic, bytes); - return TransactionStructMessageDeserializer.convertToTransactionMetadata(avroRecord); - } catch (Exception e) - { - logger.error("Avro deserialization failed for topic {}: {}", topic, e.getMessage()); - throw new SerializationException("Failed to deserialize Avro message", e); - } - } - - @Override - public void close() - { - Deserializer.super.close(); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionJsonMessageDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionJsonMessageDeserializer.java deleted file mode 100644 index c09ba3e..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionJsonMessageDeserializer.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.protobuf.util.JsonFormat; -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.util.MetricsFacade; -import org.apache.kafka.common.serialization.Deserializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Map; - -public class TransactionJsonMessageDeserializer implements Deserializer -{ - - private static final Logger LOGGER = LoggerFactory.getLogger(TransactionJsonMessageDeserializer.class); - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static final JsonFormat.Parser PROTO_PARSER = JsonFormat.parser().ignoringUnknownFields(); - - @Override - public SinkProto.TransactionMetadata deserialize(String topic, byte[] data) - { - if (data == null || data.length == 0) - { - return null; - } - MetricsFacade.getInstance().addRawData(data.length); - try - { - Map rawMessage = OBJECT_MAPPER.readValue(data, Map.class); - return parseTransactionMetadata(rawMessage); - } catch (IOException e) - { - LOGGER.error("Failed to deserialize transaction message", e); - throw new RuntimeException("Deserialization error", e); - } - } - - private SinkProto.TransactionMetadata parseTransactionMetadata(Map rawMessage) throws IOException - { - SinkProto.TransactionMetadata.Builder builder = SinkProto.TransactionMetadata.newBuilder(); - String json = OBJECT_MAPPER.writeValueAsString(rawMessage.get("payload")); - PROTO_PARSER.merge(json, builder); - - builder.setId(DeserializerUtil.getTransIdPrefix(builder.getId())); - - return builder.build(); - } -} \ No newline at end of file diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionStructMessageDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionStructMessageDeserializer.java deleted file mode 100644 index dd521cb..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/event/deserializer/TransactionStructMessageDeserializer.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - - -import io.pixelsdb.pixels.sink.SinkProto; -import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.connect.data.Struct; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @package: io.pixelsdb.pixels.sink.event.deserializer - * @className: TransactionStructMessageDeserializer - * @author: AntiO2 - * @date: 2025/9/26 12:42 - */ -public class TransactionStructMessageDeserializer -{ - private static final Logger LOGGER = LoggerFactory.getLogger(TransactionStructMessageDeserializer.class); - - @SuppressWarnings("unchecked") - public static SinkProto.TransactionMetadata convertToTransactionMetadata(T record) - { - SinkProto.TransactionMetadata.Builder builder = SinkProto.TransactionMetadata.newBuilder(); - - builder.setStatus(DeserializerUtil.getStatusSafely(record, "status")) - .setId(DeserializerUtil.getTransIdPrefix( - DeserializerUtil.getStringSafely(record, "id"))) - .setEventCount(DeserializerUtil.getLongSafely(record, "event_count")) - .setTimestamp(DeserializerUtil.getLongSafely(record, "ts_ms")); - - Object collections = DeserializerUtil.getFieldSafely(record, "data_collections"); - if (collections instanceof Iterable) - { - for (Object item : (Iterable) collections) - { - if (item instanceof GenericRecord collectionRecord) - { - SinkProto.DataCollection.Builder collectionBuilder = SinkProto.DataCollection.newBuilder(); - collectionBuilder.setDataCollection( - DeserializerUtil.getStringSafely(collectionRecord, "data_collection")); - collectionBuilder.setEventCount( - DeserializerUtil.getLongSafely(collectionRecord, "event_count")); - builder.addDataCollections(collectionBuilder); - } else if (item instanceof Struct collectionRecord) - { - SinkProto.DataCollection.Builder collectionBuilder = SinkProto.DataCollection.newBuilder(); - collectionBuilder.setDataCollection( - DeserializerUtil.getStringSafely(collectionRecord, "data_collection")); - collectionBuilder.setEventCount( - DeserializerUtil.getLongSafely(collectionRecord, "event_count")); - builder.addDataCollections(collectionBuilder); - } - } - } - - return builder.build(); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipeline.java b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipeline.java new file mode 100644 index 0000000..6704391 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipeline.java @@ -0,0 +1,58 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.pipeline; + +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.processor.TableProcessor; +import io.pixelsdb.pixels.sink.provider.RowEventProvider; +import io.pixelsdb.pixels.sink.writer.PixelsSinkWriter; +import io.pixelsdb.pixels.sink.writer.PixelsSinkWriterFactory; + +public final class TablePipeline implements AutoCloseable +{ + private final RowEventProvider provider; + private final PixelsSinkWriter writer; + private final TableProcessor processor; + + public TablePipeline() + { + this.provider = new RowEventProvider(); + this.writer = PixelsSinkWriterFactory.getWriter(); + this.processor = new TableProcessor(provider, writer); + } + + public void start() + { + processor.run(); + } + + public void publish(RowChangeEvent event) + { + provider.publishRowChangeEvent(event); + } + + @Override + public void close() + { + processor.stopProcessor(); + provider.close(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipelineManager.java b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipelineManager.java new file mode 100644 index 0000000..f4e40cc --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipelineManager.java @@ -0,0 +1,54 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.pipeline; + +import io.pixelsdb.pixels.common.metadata.SchemaTableName; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public final class TablePipelineManager implements AutoCloseable +{ + private final Map pipelines = new ConcurrentHashMap<>(); + + public void route(RowChangeEvent event) + { + if (event == null) + { + return; + } + SchemaTableName table = new SchemaTableName(event.getSchemaName(), event.getTable()); + pipelines.computeIfAbsent(table, ignored -> + { + TablePipeline pipeline = new TablePipeline(); + pipeline.start(); + return pipeline; + }).publish(event); + } + + @Override + public void close() + { + pipelines.values().forEach(TablePipeline::close); + pipelines.clear(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TransactionPipeline.java b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TransactionPipeline.java new file mode 100644 index 0000000..3ecb604 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TransactionPipeline.java @@ -0,0 +1,64 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.pipeline; + +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.processor.TransactionProcessor; +import io.pixelsdb.pixels.sink.provider.TransactionEventProvider; +import io.pixelsdb.pixels.sink.writer.PixelsSinkWriter; +import io.pixelsdb.pixels.sink.writer.PixelsSinkWriterFactory; + +public final class TransactionPipeline implements AutoCloseable +{ + private final TransactionEventProvider provider; + private final PixelsSinkWriter writer; + private final TransactionProcessor processor; + private final Thread processorThread; + + public TransactionPipeline() + { + this.provider = new TransactionEventProvider(); + this.writer = PixelsSinkWriterFactory.getWriter(); + this.processor = new TransactionProcessor(provider, writer); + this.processorThread = new Thread(processor, "transaction-processor"); + } + + public void start() + { + if (!processorThread.isAlive()) + { + processorThread.start(); + } + } + + public void publish(SinkProto.TransactionMetadata transaction) + { + provider.publishTransaction(transaction); + } + + @Override + public void close() + { + processor.stopProcessor(); + provider.close(); + processorThread.interrupt(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/processor/TableProcessor.java b/src/main/java/io/pixelsdb/pixels/sink/processor/TableProcessor.java index 134dab7..c13c6b3 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/processor/TableProcessor.java +++ b/src/main/java/io/pixelsdb/pixels/sink/processor/TableProcessor.java @@ -22,7 +22,7 @@ import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.provider.TableEventProvider; +import io.pixelsdb.pixels.sink.provider.RowEventProvider; import io.pixelsdb.pixels.sink.util.MetricsFacade; import io.pixelsdb.pixels.sink.writer.PixelsSinkWriter; import io.pixelsdb.pixels.sink.writer.PixelsSinkWriterFactory; @@ -42,15 +42,20 @@ public class TableProcessor implements StoppableProcessor, Runnable private static final Logger LOGGER = LoggerFactory.getLogger(TableProcessor.class); private final AtomicBoolean running = new AtomicBoolean(true); private final PixelsSinkWriter pixelsSinkWriter; - private final TableEventProvider tableEventProvider; + private final RowEventProvider rowEventProvider; private final MetricsFacade metricsFacade = MetricsFacade.getInstance(); private Thread processorThread; private boolean tableAdded = false; - public TableProcessor(TableEventProvider tableEventProvider) + public TableProcessor(RowEventProvider rowEventProvider) { - this.pixelsSinkWriter = PixelsSinkWriterFactory.getWriter(); - this.tableEventProvider = tableEventProvider; + this(rowEventProvider, PixelsSinkWriterFactory.getWriter()); + } + + public TableProcessor(RowEventProvider rowEventProvider, PixelsSinkWriter pixelsSinkWriter) + { + this.pixelsSinkWriter = pixelsSinkWriter; + this.rowEventProvider = rowEventProvider; } @Override @@ -64,7 +69,7 @@ private void processLoop() { while (running.get()) { - RowChangeEvent event = tableEventProvider.getRowChangeEvent(); + RowChangeEvent event = rowEventProvider.takeRowChangeEvent(); if (event == null) { continue; @@ -79,6 +84,9 @@ public void stopProcessor() { LOGGER.info("Stopping transaction monitor"); running.set(false); - processorThread.interrupt(); + if (processorThread != null) + { + processorThread.interrupt(); + } } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/processor/TopicProcessor.java b/src/main/java/io/pixelsdb/pixels/sink/processor/TopicProcessor.java deleted file mode 100644 index 905c87e..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/processor/TopicProcessor.java +++ /dev/null @@ -1,285 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.processor; - -import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; -import io.pixelsdb.pixels.sink.provider.TableEventKafkaProvider; -import org.apache.kafka.clients.admin.AdminClient; -import org.apache.kafka.clients.admin.AdminClientConfig; -import org.apache.kafka.clients.admin.ListTopicsResult; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.time.Duration; -import java.util.*; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - -public class TopicProcessor implements StoppableProcessor, Runnable -{ - - private static final Logger log = LoggerFactory.getLogger(TopicProcessor.class); - private final Properties kafkaProperties; - private final PixelsSinkConfig pixelsSinkConfig; - private final String[] includeTables; - private final Set subscribedTopics = ConcurrentHashMap.newKeySet(); - private final String bootstrapServers; - private final String baseTopic; - - private final AtomicBoolean running = new AtomicBoolean(true); - private final Map activeTasks = new ConcurrentHashMap<>(); // track row event consumer - private final ExecutorService executorService = Executors.newCachedThreadPool(); - private AdminClient adminClient; - private Timer timer; - - public TopicProcessor(PixelsSinkConfig pixelsSinkConfig, Properties kafkaProperties) - { - this.pixelsSinkConfig = pixelsSinkConfig; - this.kafkaProperties = kafkaProperties; - this.baseTopic = pixelsSinkConfig.getTopicPrefix() + "." + pixelsSinkConfig.getCaptureDatabase(); - this.includeTables = pixelsSinkConfig.getIncludeTables(); - this.bootstrapServers = pixelsSinkConfig.getBootstrapServers(); - } - - private static Set filterTopics(Set topics, String prefix) - { - return topics.stream() - .filter(t -> t.startsWith(prefix)) - .collect(Collectors.toSet()); - } - - @Override - public void run() - { - try - { - initializeResources(); - startMonitoringCycle(); - } finally - { - cleanupResources(); - log.info("Topic monitor stopped"); - } - } - - @Override - public void stopProcessor() - { - log.info("Initiating topic monitor shutdown..."); - running.set(false); - interruptMonitoring(); - shutdownConsumerTasks(); - awaitTermination(); - } - - private void shutdownConsumerTasks() - { - log.info("Shutting down {} active consumer tasks", activeTasks.size()); - activeTasks.forEach((topic, task) -> - { - log.info("Stopping consumer for topic: {}", topic); - task.close(); - }); - activeTasks.clear(); - } - - private void awaitTermination() - { - try - { - if (executorService != null && !executorService.awaitTermination(30, TimeUnit.SECONDS)) - { - log.warn("Forcing shutdown of remaining tasks"); - executorService.shutdownNow(); - } - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - } - } - - private void initializeResources() - { - Properties props = new Properties(); - props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, this.bootstrapServers); - this.adminClient = AdminClient.create(props); - this.timer = new Timer("TopicProcessor-Timer", true); - log.info("Started topic monitor for base topic: {}", baseTopic); - } - - private void startMonitoringCycle() - { - String topicPrefix = baseTopic + "."; - timer.scheduleAtFixedRate(new TopicMonitorTask(), 0, 5000); - - while (running.get()) - { - try - { - TimeUnit.SECONDS.sleep(1); - } catch (InterruptedException e) - { - if (running.get()) - { - log.warn("Monitoring thread interrupted unexpectedly", e); - } - Thread.currentThread().interrupt(); - } - } - } - - private void interruptMonitoring() - { - if (timer != null) - { - timer.cancel(); - timer.purge(); - } - if (adminClient != null) - { - adminClient.close(Duration.ofSeconds(5)); - } - shutdownExecutorService(); - } - - private void shutdownExecutorService() - { - executorService.shutdown(); - try - { - if (!executorService.awaitTermination(10, TimeUnit.SECONDS)) - { - executorService.shutdownNow(); - } - } catch (InterruptedException e) - { - executorService.shutdownNow(); - Thread.currentThread().interrupt(); - } - } - - private void cleanupResources() - { - try - { - if (adminClient != null) - { - adminClient.close(Duration.ofSeconds(5)); - } - } catch (Exception e) - { - log.warn("Error closing admin client", e); - } - } - - private Set detectNewTopics(Set currentTopics) - { - return currentTopics.stream() - .filter(t -> !subscribedTopics.contains(t)) - .collect(Collectors.toSet()); - } - - private String extractTableName(String topic) - { - int lastDotIndex = topic.lastIndexOf('.'); - return lastDotIndex != -1 ? topic.substring(lastDotIndex + 1) : topic; - } - - private void launchConsumerTask(String topic) - { - try - { - TableEventKafkaProvider task = new TableEventKafkaProvider(kafkaProperties, topic); - executorService.submit(task); - } catch (Exception e) - { - log.error("Failed to start consumer for topic {}: {}", topic, e.getMessage()); - } - } - - private class TopicMonitorTask extends TimerTask - { - @Override - public void run() - { - if (!running.get()) - { - cancel(); - return; - } - - try - { - processTopicChanges(); - } catch (Exception e) - { - e.printStackTrace(); - log.error("Error processing topic changes: {}", e.getMessage()); - } - } - - private void processTopicChanges() - { - try - { - ListTopicsResult listTopicsResult = adminClient.listTopics(); - Set currentTopics = listTopicsResult.names().get(5, TimeUnit.SECONDS); - Set filteredTopics = filterTopics(currentTopics, baseTopic + "."); - - Set newTopics = detectNewTopics(filteredTopics); - handleNewTopics(newTopics); - } catch (TimeoutException | ExecutionException | InterruptedException ignored) - { - - } - } - - private void handleNewTopics(Set newTopics) - { - newTopics.stream() - .filter(this::shouldProcessTable) - .forEach(topic -> - { - try - { - TableEventKafkaProvider task = new TableEventKafkaProvider(kafkaProperties, topic); - executorService.submit(task); - activeTasks.put(topic, task); - subscribedTopics.add(topic); - } catch (IOException e) - { - log.error("Failed to create consumer for {}: {}", topic, e.getMessage()); - } - }); - } - - private boolean shouldProcessTable(String topic) - { - String tableName = extractTableName(topic); - return includeTables.length == 0 || - Arrays.stream(includeTables).anyMatch(t -> t.equals(tableName)); - } - } - -} - diff --git a/src/main/java/io/pixelsdb/pixels/sink/processor/TransactionProcessor.java b/src/main/java/io/pixelsdb/pixels/sink/processor/TransactionProcessor.java index 1d28fb1..a18da8a 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/processor/TransactionProcessor.java +++ b/src/main/java/io/pixelsdb/pixels/sink/processor/TransactionProcessor.java @@ -37,9 +37,16 @@ public class TransactionProcessor implements Runnable, StoppableProcessor private final TransactionEventProvider transactionEventProvider; public TransactionProcessor(TransactionEventProvider transactionEventProvider) + { + this(transactionEventProvider, PixelsSinkWriterFactory.getWriter()); + } + + public TransactionProcessor( + TransactionEventProvider transactionEventProvider, + PixelsSinkWriter sinkWriter) { this.transactionEventProvider = transactionEventProvider; - this.sinkWriter = PixelsSinkWriterFactory.getWriter(); + this.sinkWriter = sinkWriter; } @Override @@ -47,7 +54,7 @@ public void run() { while (running.get()) { - SinkProto.TransactionMetadata transaction = transactionEventProvider.getTransaction(); + SinkProto.TransactionMetadata transaction = transactionEventProvider.takeTransaction(); if (transaction == null) { LOGGER.warn("Received null transaction"); diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/BlockingEventProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/BlockingEventProvider.java new file mode 100644 index 0000000..d41db4e --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/provider/BlockingEventProvider.java @@ -0,0 +1,88 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.provider; + +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; + +import java.io.Closeable; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +/** + * Bounded channel for already converted sink events. + * + *

The provider deliberately does not know how an event was read or + * converted. Source adapters publish canonical objects and processors consume + * them from this channel.

+ */ +public class BlockingEventProvider implements Closeable +{ + private static final Object POISON_PILL = new Object(); + + private final BlockingQueue queue = + new LinkedBlockingQueue<>(PixelsSinkConstants.MAX_QUEUE_SIZE); + private volatile boolean closed; + + public void publish(T event) + { + if (event == null || closed) + { + return; + } + try + { + queue.put(event); + } catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + } + } + + @SuppressWarnings("unchecked") + public T take() + { + try + { + Object value = queue.take(); + if (value == POISON_PILL) + { + return null; + } + return (T) value; + } catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + return null; + } + } + + @Override + public void close() + { + if (closed) + { + return; + } + closed = true; + queue.clear(); + queue.offer(POISON_PILL); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/EventProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/EventProvider.java deleted file mode 100644 index 8039405..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/EventProvider.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - -import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; -import io.pixelsdb.pixels.sink.util.MetricsFacade; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.Closeable; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.*; - -public abstract class EventProvider implements Runnable, Closeable -{ - private static final Logger LOGGER = LoggerFactory.getLogger(EventProvider.class); - - private static final int BATCH_SIZE = 64; - private static final int THREAD_NUM = 4; - private static final long MAX_WAIT_MS = 5; // configurable - protected final MetricsFacade metricsFacade = MetricsFacade.getInstance(); - private final BlockingQueue rawEventQueue = new LinkedBlockingQueue<>(PixelsSinkConstants.MAX_QUEUE_SIZE); - private final BlockingQueue eventQueue = new LinkedBlockingQueue<>(PixelsSinkConstants.MAX_QUEUE_SIZE); - private final ExecutorService decodeExecutor = Executors.newFixedThreadPool(THREAD_NUM); - - private Thread providerThread; - - - @Override - public void run() - { - providerThread = new Thread(this::processLoop); - providerThread.start(); - } - - @Override - public void close() - { - this.providerThread.interrupt(); - decodeExecutor.shutdown(); - } - - protected void processLoop() - { - List sourceBatch = new ArrayList<>(BATCH_SIZE); - while (true) - { - try - { - sourceBatch.clear(); - // take first element (blocking) - SOURCE_RECORD_T first = getRawEvent(); - sourceBatch.add(first); - long startTime = System.nanoTime(); - - // keep polling until sourceBatch full or timeout - while (sourceBatch.size() < BATCH_SIZE) - { - long elapsedMs = (System.nanoTime() - startTime) / 1_000_000; - long remainingMs = MAX_WAIT_MS - elapsedMs; - if (remainingMs <= 0) - { - break; - } - - SOURCE_RECORD_T next = pollRawEvent(remainingMs); - if (next == null) - { - break; - } - sourceBatch.add(next); - } - - // parallel decode - List> futures = new ArrayList<>(sourceBatch.size()); - for (SOURCE_RECORD_T data : sourceBatch) - { - futures.add(decodeExecutor.submit(() -> - convertToTargetRecord(data))); - } - - // ordered put into queue - for (Future future : futures) - { - try - { - TARGET_RECORD_T event = future.get(); - if (event != null) - { - recordSerdEvent(); - putTargetEvent(event); - } - } catch (ExecutionException e) - { - LOGGER.warn("Decode failed: {}", String.valueOf(e.getCause())); - } - } - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - break; - } - } - } - - abstract TARGET_RECORD_T convertToTargetRecord(SOURCE_RECORD_T record); - - protected TARGET_RECORD_T getTargetEvent() - { - try - { - return eventQueue.take(); - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - } - return null; - } - - protected void putTargetEvent(TARGET_RECORD_T event) - { - try - { - eventQueue.put(event); - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - } - } - - protected void putRawEvent(SOURCE_RECORD_T record) - { - try - { - rawEventQueue.put(record); - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - } - } - - protected SOURCE_RECORD_T getRawEvent() - { - try - { - return rawEventQueue.take(); - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - return null; - } - } - - protected SOURCE_RECORD_T pollRawEvent(long remainingMs) - { - try - { - return rawEventQueue.poll(remainingMs, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) - { - return null; - } - } - - abstract protected void recordSerdEvent(); -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/RowEventProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/RowEventProvider.java new file mode 100644 index 0000000..595a63b --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/provider/RowEventProvider.java @@ -0,0 +1,36 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.provider; + +import io.pixelsdb.pixels.sink.event.RowChangeEvent; + +public final class RowEventProvider extends BlockingEventProvider +{ + public void publishRowChangeEvent(RowChangeEvent event) + { + publish(event); + } + + public RowChangeEvent takeRowChangeEvent() + { + return take(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventEngineProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventEngineProvider.java deleted file mode 100644 index 2a3f0da..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventEngineProvider.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - - -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventStructDeserializer; -import io.pixelsdb.pixels.sink.exception.SinkException; -import org.apache.kafka.connect.source.SourceRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -/** - * @package: io.pixelsdb.pixels.sink.provider - * @className: TableEventEngineProvider - * @author: AntiO2 - * @date: 2025/9/26 10:45 - */ -public class TableEventEngineProvider extends TableEventProvider -{ - private final Logger LOGGER = LoggerFactory.getLogger(TableEventEngineProvider.class.getName()); - - @Override - RowChangeEvent convertToTargetRecord(T record) - { - SourceRecord sourceRecord = (SourceRecord) record; - try - { - return RowChangeEventStructDeserializer.convertToRowChangeEvent(sourceRecord); - } catch (SinkException e) - { - LOGGER.warn("Failed to convert RowChangeEvent to RowChangeEventStruct {}", e.getMessage()); - return null; - } - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventKafkaProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventKafkaProvider.java deleted file mode 100644 index b582b2c..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventKafkaProvider.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - -import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.util.DataTransform; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.errors.InterruptException; -import org.apache.kafka.common.errors.WakeupException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.time.Duration; -import java.util.Collections; -import java.util.Properties; -import java.util.concurrent.atomic.AtomicBoolean; - -public class TableEventKafkaProvider extends TableEventProvider -{ - private static final Logger log = LoggerFactory.getLogger(TableEventKafkaProvider.class); - private final Properties kafkaProperties; - private final String topic; - private final AtomicBoolean running = new AtomicBoolean(true); - private final String tableName; - private KafkaConsumer consumer; - - public TableEventKafkaProvider(Properties kafkaProperties, String topic) throws IOException - { - PixelsSinkConfig config = PixelsSinkConfigFactory.getInstance(); - this.kafkaProperties = kafkaProperties; - this.topic = topic; - this.kafkaProperties.put(ConsumerConfig.GROUP_ID_CONFIG, config.getGroupId() + "-" + topic); - this.kafkaProperties.put(ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, "false"); - this.kafkaProperties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 500); - this.tableName = DataTransform.extractTableName(topic); - } - - @Override - protected void processLoop() - { - try - { - consumer = new KafkaConsumer<>(kafkaProperties); - consumer.subscribe(Collections.singleton(topic)); - - while (running.get()) - { - try - { - ConsumerRecords records = consumer.poll(Duration.ofSeconds(5)); - if (!records.isEmpty()) - { - log.info("{} Consumer poll returned {} records", tableName, records.count()); - records.forEach(record -> - { - if (record.value() == null) - { - return; - } - metricsFacade.recordSerdRowChange(); - putRowChangeEvent(record.value()); - }); - } - } catch (InterruptException ignored) - { - Thread.currentThread().interrupt(); - break; - } - } - } catch (WakeupException e) - { - log.info("Consumer wakeup triggered for {}", tableName); - } catch (Exception e) - { - log.info("Exception: {}", e.getMessage()); - } finally - { - if (consumer != null) - { - consumer.close(Duration.ofSeconds(5)); - log.info("Kafka consumer closed for {}", tableName); - } - } - } - - @Override - RowChangeEvent convertToTargetRecord(Void record) - { - throw new UnsupportedOperationException(); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventProvider.java deleted file mode 100644 index 1dc5111..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventProvider.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - - -import io.pixelsdb.pixels.sink.event.RowChangeEvent; - -/** - * @package: io.pixelsdb.pixels.sink.provider - * @className: TableEventProvider - * @author: AntiO2 - * @date: 2025/9/26 07:47 - */ -public abstract class TableEventProvider extends EventProvider -{ - - protected void putRowChangeEvent(RowChangeEvent rowChangeEvent) - { - putTargetEvent(rowChangeEvent); - } - - public RowChangeEvent getRowChangeEvent() - { - return getTargetEvent(); - } - - protected void putRawRowChangeEvent(SOURCE_RECORD_T record) - { - putRawEvent(record); - } - - final protected void recordSerdEvent() - { - metricsFacade.recordSerdRowChange(); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventStorageLoopProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventStorageLoopProvider.java deleted file mode 100644 index b628876..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventStorageLoopProvider.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - -import com.google.protobuf.InvalidProtocolBufferException; -import io.pixelsdb.pixels.core.utils.Pair; -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventStructDeserializer; -import io.pixelsdb.pixels.sink.exception.SinkException; -import io.pixelsdb.pixels.sink.util.DataTransform; - -import java.nio.ByteBuffer; -import java.util.logging.Logger; - -public class TableEventStorageLoopProvider extends TableEventProvider -{ - private final Logger LOGGER = Logger.getLogger(TableEventStorageProvider.class.getName()); - private final boolean freshness_embed; - private final boolean freshness_timestamp; - - protected TableEventStorageLoopProvider() - { - super(); - PixelsSinkConfig config = PixelsSinkConfigFactory.getInstance(); - String sinkMonitorFreshnessLevel = config.getSinkMonitorFreshnessLevel(); - if (sinkMonitorFreshnessLevel.equals("embed")) - { - freshness_embed = true; - } else - { - freshness_embed = false; - } - freshness_timestamp = config.isSinkMonitorFreshnessTimestamp(); - } - - @Override - RowChangeEvent convertToTargetRecord(T record) - { - Pair pairRecord = (Pair) record; - ByteBuffer sourceRecord = pairRecord.getLeft(); - sourceRecord.rewind(); - Integer loopId = pairRecord.getRight(); - try - { - SinkProto.RowRecord rowRecord = SinkProto.RowRecord.parseFrom(sourceRecord); - - SinkProto.RowRecord.Builder rowRecordBuilder = rowRecord.toBuilder(); - if (freshness_timestamp) - { - DataTransform.updateRecordTimestamp(rowRecordBuilder, System.currentTimeMillis() * 1000); - } - -// if(rowRecord.getSource().getTable().equals("transfer")) -// { -// DataTransform.transIdToBigint(rowRecordBuilder); -// } - - SinkProto.TransactionInfo.Builder transactionBuilder = rowRecordBuilder.getTransactionBuilder(); - String id = transactionBuilder.getId(); - transactionBuilder.setId(id + "_" + loopId); - rowRecordBuilder.setTransaction(transactionBuilder); - return RowChangeEventStructDeserializer.convertToRowChangeEvent(rowRecordBuilder.build()); - } catch (InvalidProtocolBufferException | SinkException e) - { - LOGGER.warning(e.getMessage()); - return null; - } - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventStorageProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventStorageProvider.java deleted file mode 100644 index 8ad49d3..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TableEventStorageProvider.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - - -import com.google.protobuf.InvalidProtocolBufferException; -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventStructDeserializer; -import io.pixelsdb.pixels.sink.exception.SinkException; - -import java.nio.ByteBuffer; -import java.util.logging.Logger; - -/** - * @package: io.pixelsdb.pixels.sink.event - * @className: TableEventStorageProvider - * @author: AntiO2 - * @date: 2025/9/26 10:45 - */ -public class TableEventStorageProvider extends TableEventProvider -{ - private final Logger LOGGER = Logger.getLogger(TableEventStorageProvider.class.getName()); - - protected TableEventStorageProvider() - { - super(); - } - - @Override - RowChangeEvent convertToTargetRecord(T record) - { - ByteBuffer sourceRecord = (ByteBuffer) record; - try - { - SinkProto.RowRecord rowRecord = SinkProto.RowRecord.parseFrom(sourceRecord); - return RowChangeEventStructDeserializer.convertToRowChangeEvent(rowRecord); - } catch (InvalidProtocolBufferException | SinkException e) - { - LOGGER.warning(e.getMessage()); - return null; - } - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TableProviderAndProcessorPipelineManager.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TableProviderAndProcessorPipelineManager.java deleted file mode 100644 index 128f70b..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TableProviderAndProcessorPipelineManager.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - - -import io.pixelsdb.pixels.common.metadata.SchemaTableName; -import io.pixelsdb.pixels.core.utils.Pair; -import io.pixelsdb.pixels.sink.processor.TableProcessor; -import org.apache.kafka.connect.source.SourceRecord; - -import java.nio.ByteBuffer; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * @package: io.pixelsdb.pixels.sink.provider - * @className: TableProviderAndProcessorPipelineManager - * @author: AntiO2 - * @date: 2025/9/26 10:44 - */ -public class TableProviderAndProcessorPipelineManager -{ - protected final Map activeTableProcessors = new ConcurrentHashMap<>(); - protected final Map tableIds = new ConcurrentHashMap<>(); - private final Map> tableProviders = new ConcurrentHashMap<>(); - private final AtomicInteger nextTableId = new AtomicInteger(); - - - public void routeRecord(SchemaTableName schemaTableName, SOURCE_RECORD_T record) - { - routeRecord(getTableId(schemaTableName), record); - } - - public void routeRecord(Integer tableId, SOURCE_RECORD_T record) - { - TableEventProvider pipeline = tableProviders.computeIfAbsent(tableId, k -> - { - TableEventProvider newPipeline = createProvider(record); - TableProcessor tableProcessor = activeTableProcessors.computeIfAbsent(tableId, k2 -> - new TableProcessor(newPipeline) - ); - tableProcessor.run(); - newPipeline.run(); - return newPipeline; - }); - pipeline.putRawEvent(record); - } - - private TableEventProvider createProvider(SOURCE_RECORD_T record) - { - Class recordType = record.getClass(); - if (recordType == Pair.class) - { - return new TableEventStorageLoopProvider<>(); - } - if (recordType == SourceRecord.class) - { - return new TableEventEngineProvider<>(); - } else if (ByteBuffer.class.isAssignableFrom(recordType)) - { - return new TableEventStorageProvider<>(); - } else - { - throw new IllegalArgumentException("Unsupported record type: " + recordType.getName()); - } - } - - private Integer getTableId(SchemaTableName schemaTableName) - { - return tableIds.computeIfAbsent(schemaTableName, k -> allocateTableId()); - } - - private Integer allocateTableId() - { - return nextTableId.getAndIncrement(); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventEngineProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventEngineProvider.java deleted file mode 100644 index e39d853..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventEngineProvider.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - - -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.event.deserializer.TransactionStructMessageDeserializer; -import org.apache.kafka.connect.data.Struct; -import org.apache.kafka.connect.source.SourceRecord; - -/** - * @package: io.pixelsdb.pixels.sink.provider - * @className: TransactionEventEngineProvider - * @author: AntiO2 - * @date: 2025/9/25 13:20 - */ -public class TransactionEventEngineProvider extends TransactionEventProvider -{ - - public static final TransactionEventEngineProvider INSTANCE = new TransactionEventEngineProvider<>(); - - public static TransactionEventEngineProvider getInstance() - { - return INSTANCE; - } - - @Override - SinkProto.TransactionMetadata convertToTargetRecord(T record) - { - SourceRecord sourceRecord = (SourceRecord) record; - Struct value = (Struct) sourceRecord.value(); - return TransactionStructMessageDeserializer.convertToTransactionMetadata(value); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventKafkaProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventKafkaProvider.java deleted file mode 100644 index 081e0cf..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventKafkaProvider.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - - -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.common.errors.WakeupException; - -import java.time.Duration; -import java.util.Collections; -import java.util.Properties; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * @package: io.pixelsdb.pixels.sink.provider - * @className: TransactionEventKafkaProvider - * @author: AntiO2 - * @date: 2025/9/25 13:40 - */ -public class TransactionEventKafkaProvider extends TransactionEventProvider -{ - private final AtomicBoolean running = new AtomicBoolean(true); - private final String transactionTopic; - private final KafkaConsumer consumer; - - private TransactionEventKafkaProvider() - { - Properties kafkaProperties = new Properties(); - PixelsSinkConfig pixelsSinkConfig = PixelsSinkConfigFactory.getInstance(); - this.transactionTopic = pixelsSinkConfig.getTopicPrefix() + "." + pixelsSinkConfig.getTransactionTopicSuffix(); - this.consumer = new KafkaConsumer<>(kafkaProperties); - } - - - @Override - public void processLoop() - { - consumer.subscribe(Collections.singletonList(transactionTopic)); - while (running.get()) - { - try - { - - ConsumerRecords records = - consumer.poll(Duration.ofMillis(1000)); - - for (ConsumerRecord record : records) - { - if (record.value() == null) - { - continue; - } - putTargetEvent(record.value()); - } - } catch (WakeupException e) - { - if (running.get()) - { - // LOGGER.warn("Consumer wakeup unexpectedly", e); - } - } catch (Exception e) - { - e.printStackTrace(); - throw new RuntimeException(e); - } - } - } - - @Override - SinkProto.TransactionMetadata convertToTargetRecord(T record) - { - throw new UnsupportedOperationException(); - } - -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventProvider.java index 9239ba1..9c47856 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventProvider.java +++ b/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventProvider.java @@ -22,20 +22,16 @@ import io.pixelsdb.pixels.sink.SinkProto; -public abstract class TransactionEventProvider extends EventProvider +public final class TransactionEventProvider + extends BlockingEventProvider { - public void putTransRawEvent(SOURCE_RECORD_T record) + public void publishTransaction(SinkProto.TransactionMetadata transaction) { - putRawEvent(record); + publish(transaction); } - public SinkProto.TransactionMetadata getTransaction() + public SinkProto.TransactionMetadata takeTransaction() { - return getTargetEvent(); - } - - final protected void recordSerdEvent() - { - metricsFacade.recordSerdTxChange(); + return take(); } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventStorageLoopProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventStorageLoopProvider.java deleted file mode 100644 index 956de39..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventStorageLoopProvider.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - -import com.google.protobuf.InvalidProtocolBufferException; -import io.pixelsdb.pixels.core.utils.Pair; -import io.pixelsdb.pixels.sink.SinkProto; - -import java.nio.ByteBuffer; - -public class TransactionEventStorageLoopProvider extends TransactionEventProvider -{ - @Override - SinkProto.TransactionMetadata convertToTargetRecord(T record) - { - Pair buffer = (Pair) record; - try - { - SinkProto.TransactionMetadata tx = SinkProto.TransactionMetadata.parseFrom(buffer.getLeft()); - Integer loopId = buffer.getRight(); - SinkProto.TransactionMetadata.Builder builder = tx.toBuilder(); - builder.setId(builder.getId() + "_" + loopId); - return builder.build(); - } catch (InvalidProtocolBufferException e) - { - throw new RuntimeException(e); - } - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventStorageProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventStorageProvider.java deleted file mode 100644 index 59198c3..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventStorageProvider.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - -import com.google.protobuf.InvalidProtocolBufferException; -import io.pixelsdb.pixels.sink.SinkProto; - -import java.nio.ByteBuffer; - -public class TransactionEventStorageProvider extends TransactionEventProvider -{ - @Override - SinkProto.TransactionMetadata convertToTargetRecord(T record) - { - ByteBuffer buffer = (ByteBuffer) record; - try - { - SinkProto.TransactionMetadata tx = SinkProto.TransactionMetadata.parseFrom(buffer); - return tx; - } catch (InvalidProtocolBufferException e) - { - throw new RuntimeException(e); - } - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumer.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumer.java index 96785a7..97bb27e 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumer.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumer.java @@ -23,18 +23,26 @@ import io.debezium.engine.DebeziumEngine; import io.debezium.engine.RecordChangeEvent; -import io.pixelsdb.pixels.common.metadata.SchemaTableName; +import io.pixelsdb.pixels.sink.SinkProto; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumRecordUtil; +import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.sink.pipeline.TablePipelineManager; +import io.pixelsdb.pixels.sink.pipeline.TransactionPipeline; import io.pixelsdb.pixels.sink.processor.StoppableProcessor; -import io.pixelsdb.pixels.sink.processor.TransactionProcessor; -import io.pixelsdb.pixels.sink.provider.TableProviderAndProcessorPipelineManager; -import io.pixelsdb.pixels.sink.provider.TransactionEventEngineProvider; +import io.pixelsdb.pixels.sink.source.engine.adapter.DebeziumStructAdapter; +import io.pixelsdb.pixels.sink.source.engine.adapter.DebeziumSourceAdapterSelector; import io.pixelsdb.pixels.sink.util.MetricsFacade; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.source.SourceRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.List; +import java.util.Locale; /** * @package: io.pixelsdb.pixels.source @@ -44,23 +52,34 @@ */ public class PixelsDebeziumConsumer implements DebeziumEngine.ChangeConsumer>, StoppableProcessor { + private static final Logger LOGGER = LoggerFactory.getLogger(PixelsDebeziumConsumer.class); + + public enum RecordType + { + ROW, + TRANSACTION, + TOMBSTONE, + UNKNOWN_CONTROL + } + private final String checkTransactionTopic; - private final TransactionEventEngineProvider transactionEventProvider = TransactionEventEngineProvider.INSTANCE; - private final TableProviderAndProcessorPipelineManager tableProvidersManagerImpl = new TableProviderAndProcessorPipelineManager<>(); - private final TransactionProcessor processor = new TransactionProcessor(transactionEventProvider); - private final Thread transactionProviderThread; - private final Thread transactionProcessorThread; + private final DebeziumSourceAdapter connectorAdapter; + private final DebeziumStructAdapter structAdapter; + private final TransactionPipeline transactionPipeline = new TransactionPipeline(); + private final TablePipelineManager tablePipelineManager = new TablePipelineManager(); private final MetricsFacade metricsFacade = MetricsFacade.getInstance(); - PixelsSinkConfig pixelsSinkConfig = PixelsSinkConfigFactory.getInstance(); + private final PixelsSinkConfig pixelsSinkConfig = PixelsSinkConfigFactory.getInstance(); public PixelsDebeziumConsumer() { this.checkTransactionTopic = pixelsSinkConfig.getDebeziumTopicPrefix() + ".transaction"; - this.transactionProviderThread = new Thread(this.transactionEventProvider, "transaction-adapter"); - this.transactionProcessorThread = new Thread(this.processor, "transaction-processor"); + this.connectorAdapter = DebeziumSourceAdapterSelector.configured(); + this.structAdapter = new DebeziumStructAdapter(connectorAdapter); + } - this.transactionProcessorThread.start(); - this.transactionProviderThread.start(); + public void start() + { + transactionPipeline.start(); } @@ -78,12 +97,22 @@ public void handleBatch(List> event, } metricsFacade.recordDebeziumEvent(); - if (isTransactionEvent(sourceRecord)) + RecordType recordType = classify(sourceRecord, checkTransactionTopic); + logSourceRecord(sourceRecord, recordType); + try { - handleTransactionSourceRecord(sourceRecord); - } else + switch (recordType) + { + case ROW -> handleRowChangeSourceRecord(sourceRecord); + case TRANSACTION -> handleTransactionSourceRecord(sourceRecord); + case TOMBSTONE, UNKNOWN_CONTROL -> + LOGGER.debug("Skipping Debezium {} event from topic {}", + recordType, sourceRecord.topic()); + } + } catch (RuntimeException e) { - handleRowChangeSourceRecord(sourceRecord); + LOGGER.warn("Skipping invalid Debezium {} event from topic {}: {}", + recordType, sourceRecord.topic(), e.getMessage()); } } finally { @@ -96,34 +125,117 @@ public void handleBatch(List> event, private void handleTransactionSourceRecord(SourceRecord sourceRecord) throws InterruptedException { - transactionEventProvider.putTransRawEvent(sourceRecord); + SinkProto.TransactionMetadata transaction = structAdapter.toTransactionMetadata(sourceRecord); + metricsFacade.recordSerdTxChange(); + transactionPipeline.publish(transaction); } private void handleRowChangeSourceRecord(SourceRecord sourceRecord) { - Struct value = (Struct) sourceRecord.value(); - if (value == null) + try + { + RowChangeEvent event = structAdapter.toRowEvent(sourceRecord); + metricsFacade.recordSerdRowChange(); + tablePipelineManager.route(event); + } catch (SinkException e) { - return; // Delete Record, We will handle it in next record + throw new IllegalArgumentException("Failed to convert Debezium row event", e); } - Object sourceObject = value.get("source"); - Struct source = (Struct) sourceObject; - String schemaName = source.get("db").toString(); - String tableName = source.get("table").toString(); - SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName); - tableProvidersManagerImpl.routeRecord(schemaTableName, sourceRecord); } - private boolean isTransactionEvent(SourceRecord sourceRecord) + public static RecordType classify(SourceRecord sourceRecord, String transactionTopic) + { + if (sourceRecord == null || sourceRecord.value() == null) + { + return RecordType.TOMBSTONE; + } + if (!(sourceRecord.value() instanceof Struct value)) + { + return RecordType.UNKNOWN_CONTROL; + } + + if (transactionTopic != null && transactionTopic.equals(sourceRecord.topic())) + { + String status = DebeziumRecordUtil.getStringSafely(value, "status"); + String id = DebeziumRecordUtil.getStringSafely(value, "id"); + return (status.equals("BEGIN") || status.equals("END")) && !id.isBlank() + ? RecordType.TRANSACTION + : RecordType.UNKNOWN_CONTROL; + } + + return isRowChange(value) ? RecordType.ROW : RecordType.UNKNOWN_CONTROL; + } + + private static boolean isRowChange(Struct value) + { + String op = DebeziumRecordUtil.getStringSafely(value, "op").toLowerCase(Locale.ROOT); + if (!(op.equals("c") || op.equals("u") || op.equals("d") || op.equals("r"))) + { + return false; + } + + Object sourceObject = DebeziumRecordUtil.getFieldSafely(value, "source"); + if (!(sourceObject instanceof Struct source) || + DebeziumRecordUtil.getStringSafely(source, "db").isBlank() || + DebeziumRecordUtil.getStringSafely(source, "table").isBlank()) + { + return false; + } + + Object before = DebeziumRecordUtil.getFieldSafely(value, "before"); + Object after = DebeziumRecordUtil.getFieldSafely(value, "after"); + return switch (op) + { + case "c", "r" -> after instanceof Struct; + case "u" -> before instanceof Struct && after instanceof Struct; + case "d" -> before instanceof Struct; + default -> false; + }; + } + + private void logSourceRecord(SourceRecord record, RecordType recordType) + { + if (!LOGGER.isDebugEnabled()) + { + return; + } + Struct value = record.value() instanceof Struct struct ? struct : null; + Struct source = value == null ? null : + asStruct(DebeziumRecordUtil.getFieldSafely(value, "source")); + Struct transaction = value == null ? null : + asStruct(DebeziumRecordUtil.getFieldSafely(value, "transaction")); + String rawTransactionId = recordType == RecordType.TRANSACTION + ? DebeziumRecordUtil.getStringSafely(value, "id") + : DebeziumRecordUtil.getStringSafely(transaction, "id"); + String canonicalTransactionId = + connectorAdapter.normalizeTransactionId(rawTransactionId); + + LOGGER.debug("Debezium SourceRecord topic={}, sourcePartition={}, sourceOffset={}, " + + "keySchema={}, key={}, valueSchema={}, category={}, transaction.id={}, " + + "source.gtid={}, source.file={}, source.pos={}, source.row={}", + record.topic(), record.sourcePartition(), record.sourceOffset(), + schemaName(record.keySchema()), record.key(), schemaName(record.valueSchema()), + recordType, canonicalTransactionId, + DebeziumRecordUtil.getStringSafely(source, "gtid"), + DebeziumRecordUtil.getStringSafely(source, "file"), + DebeziumRecordUtil.getStringSafely(source, "pos"), + DebeziumRecordUtil.getStringSafely(source, "row")); + } + + private static Struct asStruct(Object value) + { + return value instanceof Struct struct ? struct : null; + } + + private static String schemaName(org.apache.kafka.connect.data.Schema schema) { - return checkTransactionTopic.equals(sourceRecord.topic()); + return schema == null ? "" : String.valueOf(schema.name()); } @Override public void stopProcessor() { - transactionProviderThread.interrupt(); - processor.stopProcessor(); - transactionProcessorThread.interrupt(); + tablePipelineManager.close(); + transactionPipeline.close(); } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/SinkEngineSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/SinkEngineSource.java index 47d2735..cfb69ef 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/engine/SinkEngineSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/SinkEngineSource.java @@ -46,6 +46,7 @@ public SinkEngineSource() public void start() { + consumer.start(); Properties debeziumProps = PixelsSinkConfigFactory.getInstance() .getConfig().extractPropertiesByPrefix("debezium.", true); diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumSourceAdapterSelector.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumSourceAdapterSelector.java new file mode 100644 index 0000000..097a8c5 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumSourceAdapterSelector.java @@ -0,0 +1,47 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.source.engine.adapter; + +import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapterRegistry; + +public final class DebeziumSourceAdapterSelector +{ + private DebeziumSourceAdapterSelector() + { + } + + public static DebeziumSourceAdapter configured() + { + return DebeziumSourceAdapterRegistry.resolve( + PixelsSinkConfigFactory.getInstance().getDebeziumConnectorClass()); + } + + public static DebeziumSourceAdapter configuredIfPresent() + { + String connector = + PixelsSinkConfigFactory.getInstance().getDebeziumConnectorClass(); + return connector == null || connector.isBlank() + ? null + : DebeziumSourceAdapterRegistry.resolve(connector); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumStructAdapter.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumStructAdapter.java new file mode 100644 index 0000000..46eb9b0 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumStructAdapter.java @@ -0,0 +1,58 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.source.engine.adapter; + +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumEnvelopeNormalizer; +import io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventStructConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.source.SourceRecord; + +public final class DebeziumStructAdapter +{ + private final DebeziumSourceAdapter sourceAdapter; + private final RowChangeEventStructConverter rowConverter; + + public DebeziumStructAdapter(DebeziumSourceAdapter sourceAdapter) + { + this.sourceAdapter = sourceAdapter; + this.rowConverter = new RowChangeEventStructConverter( + TableMetadataRegistry.Instance(), sourceAdapter); + } + + public RowChangeEvent toRowEvent(SourceRecord sourceRecord) throws SinkException + { + return rowConverter.convert(sourceRecord); + } + + public SinkProto.TransactionMetadata toTransactionMetadata(SourceRecord sourceRecord) + { + if (!(sourceRecord.value() instanceof Struct value)) + { + throw new IllegalArgumentException("Debezium transaction value must be a Struct"); + } + return DebeziumEnvelopeNormalizer.normalizeTransactionMetadata(value, sourceAdapter); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaRowSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaRowSource.java new file mode 100644 index 0000000..ee391a3 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaRowSource.java @@ -0,0 +1,144 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.source.kafka; + +import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; +import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.pipeline.TablePipelineManager; +import io.pixelsdb.pixels.sink.processor.StoppableProcessor; +import io.pixelsdb.pixels.sink.source.kafka.serde.KafkaRecordConverter; +import io.pixelsdb.pixels.sink.util.DataTransform; +import io.pixelsdb.pixels.sink.util.MetricsFacade; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.errors.InterruptException; +import org.apache.kafka.common.errors.WakeupException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.Collections; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicBoolean; + +public final class KafkaRowSource implements Runnable, StoppableProcessor +{ + private static final Logger LOGGER = LoggerFactory.getLogger(KafkaRowSource.class); + + private final Properties kafkaProperties; + private final String topic; + private final String tableName; + private final TablePipelineManager tablePipelineManager; + private final KafkaRecordConverter converter; + private final MetricsFacade metricsFacade = MetricsFacade.getInstance(); + private final AtomicBoolean running = new AtomicBoolean(true); + private KafkaConsumer consumer; + + public KafkaRowSource( + Properties kafkaProperties, + String topic, + TablePipelineManager tablePipelineManager) + { + PixelsSinkConfig config = PixelsSinkConfigFactory.getInstance(); + this.kafkaProperties = new Properties(); + this.kafkaProperties.putAll(kafkaProperties); + this.kafkaProperties.put( + ConsumerConfig.GROUP_ID_CONFIG, config.getGroupId() + "-" + topic); + this.kafkaProperties.put(ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, "false"); + this.kafkaProperties.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 500); + this.topic = topic; + this.tableName = DataTransform.extractTableName(topic); + this.tablePipelineManager = tablePipelineManager; + this.converter = KafkaRecordConverter.create( + this.kafkaProperties, PixelsSinkConstants.ROW_RECORD_CONVERTER_CLASS); + } + + @Override + public void run() + { + try + { + consumer = new KafkaConsumer<>(kafkaProperties); + consumer.subscribe(Collections.singleton(topic)); + while (running.get()) + { + try + { + ConsumerRecords records = + consumer.poll(Duration.ofSeconds(5)); + records.forEach(record -> + { + byte[] value = record.value(); + if (value == null) + { + return; + } + metricsFacade.addRawData(value.length); + try + { + RowChangeEvent event = converter.convert(topic, value); + if (event != null) + { + metricsFacade.recordSerdRowChange(); + tablePipelineManager.route(event); + } + } catch (RuntimeException e) + { + LOGGER.warn( + "Failed to convert Kafka record from topic {}: {}", + topic, e.getMessage()); + } + }); + } catch (InterruptException e) + { + Thread.currentThread().interrupt(); + break; + } + } + } catch (WakeupException e) + { + LOGGER.debug("Consumer wakeup triggered for {}", tableName); + } catch (Exception e) + { + LOGGER.error("Kafka source failed for {}", tableName, e); + } finally + { + converter.close(); + if (consumer != null) + { + consumer.close(Duration.ofSeconds(5)); + } + } + } + + @Override + public void stopProcessor() + { + running.set(false); + if (consumer != null) + { + consumer.wakeup(); + } + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaTransactionSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaTransactionSource.java new file mode 100644 index 0000000..2e4ea69 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaTransactionSource.java @@ -0,0 +1,121 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.source.kafka; + +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; +import io.pixelsdb.pixels.sink.pipeline.TransactionPipeline; +import io.pixelsdb.pixels.sink.processor.StoppableProcessor; +import io.pixelsdb.pixels.sink.source.kafka.serde.KafkaRecordConverter; +import io.pixelsdb.pixels.sink.util.MetricsFacade; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.errors.WakeupException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.Collections; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicBoolean; + +public final class KafkaTransactionSource implements Runnable, StoppableProcessor +{ + private static final Logger LOGGER = LoggerFactory.getLogger(KafkaTransactionSource.class); + + private final String transactionTopic; + private final KafkaConsumer consumer; + private final KafkaRecordConverter converter; + private final TransactionPipeline transactionPipeline; + private final MetricsFacade metricsFacade = MetricsFacade.getInstance(); + private final AtomicBoolean running = new AtomicBoolean(true); + + public KafkaTransactionSource( + Properties kafkaProperties, + String transactionTopic, + TransactionPipeline transactionPipeline) + { + this.transactionTopic = transactionTopic; + Properties consumerProperties = new Properties(); + consumerProperties.putAll(kafkaProperties); + this.converter = KafkaRecordConverter.create( + consumerProperties, PixelsSinkConstants.TRANSACTION_CONVERTER_CLASS); + this.consumer = new KafkaConsumer<>(consumerProperties); + this.transactionPipeline = transactionPipeline; + } + + @Override + public void run() + { + try + { + consumer.subscribe(Collections.singletonList(transactionTopic)); + while (running.get()) + { + try + { + ConsumerRecords records = + consumer.poll(Duration.ofMillis(1000)); + for (ConsumerRecord record : records) + { + byte[] value = record.value(); + if (value == null) + { + continue; + } + metricsFacade.addRawData(value.length); + SinkProto.TransactionMetadata transaction = + converter.convert(transactionTopic, value); + if (transaction != null) + { + metricsFacade.recordSerdTxChange(); + transactionPipeline.publish(transaction); + } + } + } catch (WakeupException e) + { + if (running.get()) + { + throw e; + } + } + } + } catch (Exception e) + { + if (running.get()) + { + LOGGER.error("Kafka transaction source failed for {}", transactionTopic, e); + } + } finally + { + converter.close(); + consumer.close(Duration.ofSeconds(5)); + } + } + + @Override + public void stopProcessor() + { + running.set(false); + consumer.wakeup(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/SinkKafkaSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/SinkKafkaSource.java index 423e1d1..da9a790 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/SinkKafkaSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/SinkKafkaSource.java @@ -25,8 +25,8 @@ import io.pixelsdb.pixels.sink.config.factory.KafkaPropFactorySelector; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; import io.pixelsdb.pixels.sink.processor.MonitorThreadManager; -import io.pixelsdb.pixels.sink.processor.TopicProcessor; -import io.pixelsdb.pixels.sink.processor.TransactionProcessor; +import io.pixelsdb.pixels.sink.pipeline.TablePipelineManager; +import io.pixelsdb.pixels.sink.pipeline.TransactionPipeline; import io.pixelsdb.pixels.sink.source.SinkSource; import java.util.Properties; @@ -34,7 +34,9 @@ public class SinkKafkaSource implements SinkSource { private MonitorThreadManager manager; - private volatile boolean running = true; + private TablePipelineManager tablePipelineManager; + private TransactionPipeline transactionPipeline; + private volatile boolean running; @Override public void start() @@ -45,23 +47,43 @@ public void start() Properties transactionKafkaProperties = kafkaPropFactorySelector .getFactory(PixelsSinkConstants.TRANSACTION_KAFKA_PROP_FACTORY) .createKafkaProperties(pixelsSinkConfig); - TransactionProcessor transactionProcessor = null; // TODO: new TransactionProcessor(); + String transactionTopic = pixelsSinkConfig.getTopicPrefix() + "." + + pixelsSinkConfig.getTransactionTopicSuffix(); + tablePipelineManager = new TablePipelineManager(); + transactionPipeline = new TransactionPipeline(); + KafkaTransactionSource transactionSource = + new KafkaTransactionSource( + transactionKafkaProperties, transactionTopic, transactionPipeline); Properties topicKafkaProperties = kafkaPropFactorySelector .getFactory(PixelsSinkConstants.ROW_RECORD_KAFKA_PROP_FACTORY) .createKafkaProperties(pixelsSinkConfig); - TopicProcessor topicMonitor = new TopicProcessor(pixelsSinkConfig, topicKafkaProperties); + TopicProcessor topicMonitor = new TopicProcessor( + pixelsSinkConfig, topicKafkaProperties, tablePipelineManager); + transactionPipeline.start(); manager = new MonitorThreadManager(); - manager.startMonitor(transactionProcessor); + manager.startMonitor(transactionSource); manager.startMonitor(topicMonitor); + running = true; } @Override public void stopProcessor() { - manager.shutdown(); + if (manager != null) + { + manager.shutdown(); + } + if (transactionPipeline != null) + { + transactionPipeline.close(); + } + if (tablePipelineManager != null) + { + tablePipelineManager.close(); + } running = false; } diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/TopicProcessor.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/TopicProcessor.java new file mode 100644 index 0000000..7867b3c --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/TopicProcessor.java @@ -0,0 +1,184 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.source.kafka; + +import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; +import io.pixelsdb.pixels.sink.pipeline.TablePipelineManager; +import io.pixelsdb.pixels.sink.processor.StoppableProcessor; +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +public final class TopicProcessor implements Runnable, StoppableProcessor +{ + private static final Logger LOGGER = LoggerFactory.getLogger(TopicProcessor.class); + + private final Properties kafkaProperties; + private final PixelsSinkConfig config; + private final String[] includeTables; + private final String bootstrapServers; + private final String baseTopic; + private final TablePipelineManager tablePipelineManager; + private final Set subscribedTopics = ConcurrentHashMap.newKeySet(); + private final Map activeSources = new ConcurrentHashMap<>(); + private final ExecutorService executor = Executors.newCachedThreadPool(); + private final AtomicBoolean running = new AtomicBoolean(true); + private AdminClient adminClient; + private Timer timer; + + public TopicProcessor( + PixelsSinkConfig config, + Properties kafkaProperties, + TablePipelineManager tablePipelineManager) + { + this.config = config; + this.kafkaProperties = kafkaProperties; + this.includeTables = config.getIncludeTables(); + this.bootstrapServers = config.getBootstrapServers(); + this.baseTopic = config.getTopicPrefix() + "." + config.getCaptureDatabase(); + this.tablePipelineManager = tablePipelineManager; + } + + @Override + public void run() + { + try + { + Properties adminProperties = new Properties(); + adminProperties.put( + AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + adminClient = AdminClient.create(adminProperties); + timer = new Timer("TopicProcessor-Timer", true); + timer.scheduleAtFixedRate(new TopicMonitorTask(), 0, 5000); + while (running.get()) + { + try + { + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) + { + if (running.get()) + { + LOGGER.warn("Kafka topic monitor interrupted", e); + } + Thread.currentThread().interrupt(); + break; + } + } + } finally + { + stopProcessor(); + } + } + + @Override + public void stopProcessor() + { + if (!running.compareAndSet(true, false)) + { + return; + } + if (timer != null) + { + timer.cancel(); + } + if (adminClient != null) + { + adminClient.close(Duration.ofSeconds(5)); + } + activeSources.values().forEach(KafkaRowSource::stopProcessor); + activeSources.clear(); + executor.shutdown(); + try + { + if (!executor.awaitTermination(10, TimeUnit.SECONDS)) + { + executor.shutdownNow(); + } + } catch (InterruptedException e) + { + executor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + + private class TopicMonitorTask extends TimerTask + { + @Override + public void run() + { + if (!running.get()) + { + cancel(); + return; + } + try + { + Set topics = adminClient.listTopics().names().get(5, TimeUnit.SECONDS) + .stream() + .filter(topic -> topic.startsWith(baseTopic + ".")) + .collect(Collectors.toSet()); + topics.stream() + .filter(topic -> !subscribedTopics.contains(topic)) + .filter(TopicProcessor.this::shouldProcessTable) + .forEach(this::startTopic); + } catch (Exception e) + { + if (running.get()) + { + LOGGER.warn("Failed to inspect Kafka topics", e); + } + } + } + + private void startTopic(String topic) + { + KafkaRowSource source = new KafkaRowSource( + kafkaProperties, topic, tablePipelineManager); + activeSources.put(topic, source); + subscribedTopics.add(topic); + executor.submit(source); + } + } + + private boolean shouldProcessTable(String topic) + { + String tableName = topic.substring(topic.lastIndexOf('.') + 1); + return includeTables.length == 0 || + Arrays.stream(includeTables).anyMatch(tableName::equals); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverter.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverter.java new file mode 100644 index 0000000..4fe7e42 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverter.java @@ -0,0 +1,86 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.source.kafka.serde; + +import org.apache.kafka.common.serialization.Deserializer; + +import java.lang.reflect.InvocationTargetException; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +public final class KafkaRecordConverter implements AutoCloseable +{ + private final Deserializer deserializer; + + private KafkaRecordConverter(Deserializer deserializer) + { + this.deserializer = deserializer; + } + + public static KafkaRecordConverter create( + Properties properties, String converterClassKey) + { + Object configuredClass = properties.get(converterClassKey); + if (configuredClass == null) + { + throw new IllegalArgumentException( + "Missing Kafka record converter: " + converterClassKey); + } + + try + { + Class converterClass = configuredClass instanceof Class type + ? type + : Class.forName(configuredClass.toString()); + if (!Deserializer.class.isAssignableFrom(converterClass)) + { + throw new IllegalArgumentException( + "Kafka record converter must implement Deserializer: " + + converterClass.getName()); + } + + @SuppressWarnings("unchecked") + Deserializer deserializer = + (Deserializer) converterClass.getDeclaredConstructor().newInstance(); + Map configuration = new HashMap<>(); + properties.forEach((key, value) -> configuration.put(key.toString(), value)); + deserializer.configure(configuration, false); + return new KafkaRecordConverter<>(deserializer); + } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | + IllegalAccessException | InvocationTargetException e) + { + throw new IllegalArgumentException( + "Failed to create Kafka record converter: " + configuredClass, e); + } + } + + public T convert(String topic, byte[] data) + { + return deserializer.deserialize(topic, data); + } + + @Override + public void close() + { + deserializer.close(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowChangeEventAvroDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowChangeEventAvroDeserializer.java new file mode 100644 index 0000000..82490a7 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowChangeEventAvroDeserializer.java @@ -0,0 +1,79 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.source.kafka.serde; + +import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumAvroRowConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapterRegistry; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import io.pixelsdb.pixels.sink.source.engine.adapter.DebeziumSourceAdapterSelector; +import org.apache.avro.generic.GenericRecord; +import org.apache.kafka.common.serialization.Deserializer; + +import java.util.HashMap; +import java.util.Map; + +public class RowChangeEventAvroDeserializer implements Deserializer +{ + private final AvroKafkaDeserializer avroDeserializer = + new AvroKafkaDeserializer<>(); + private DebeziumAvroRowConverter converter = + new DebeziumAvroRowConverter( + TableMetadataRegistry.Instance(), + DebeziumSourceAdapterSelector.configuredIfPresent()); + + @Override + public void configure(Map configs, boolean isKey) + { + Map enrichedConfig = new HashMap<>(configs); + enrichedConfig.put(SerdeConfig.CHECK_PERIOD_MS, SerdeConfig.CHECK_PERIOD_MS_DEFAULT); + avroDeserializer.configure(enrichedConfig, isKey); + Object connector = configs.get(PixelsSinkConstants.DEBEZIUM_CONNECTOR_CLASS); + if (connector != null && !connector.toString().isBlank()) + { + DebeziumSourceAdapter adapter = + DebeziumSourceAdapterRegistry.resolve(connector.toString()); + converter = new DebeziumAvroRowConverter( + TableMetadataRegistry.Instance(), adapter); + } + } + + @Override + public RowChangeEvent deserialize(String topic, byte[] data) + { + if (data == null || data.length == 0) + { + return null; + } + try + { + GenericRecord avroRecord = avroDeserializer.deserialize(topic, data); + return converter.convert(avroRecord); + } catch (Exception e) + { + return null; + } + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowChangeEventJsonDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowChangeEventJsonDeserializer.java new file mode 100644 index 0000000..65428de --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowChangeEventJsonDeserializer.java @@ -0,0 +1,78 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.source.kafka.serde; + +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumJsonRowConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapterRegistry; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import io.pixelsdb.pixels.sink.source.engine.adapter.DebeziumSourceAdapterSelector; +import org.apache.kafka.common.serialization.Deserializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +public class RowChangeEventJsonDeserializer implements Deserializer +{ + private static final Logger LOGGER = + LoggerFactory.getLogger(RowChangeEventJsonDeserializer.class); + private DebeziumJsonRowConverter converter; + + public RowChangeEventJsonDeserializer() + { + this.converter = new DebeziumJsonRowConverter( + TableMetadataRegistry.Instance(), + DebeziumSourceAdapterSelector.configuredIfPresent()); + } + + @Override + public void configure(Map configs, boolean isKey) + { + Object connector = configs.get(PixelsSinkConstants.DEBEZIUM_CONNECTOR_CLASS); + if (connector != null && !connector.toString().isBlank()) + { + DebeziumSourceAdapter adapter = + DebeziumSourceAdapterRegistry.resolve(connector.toString()); + converter = new DebeziumJsonRowConverter( + TableMetadataRegistry.Instance(), adapter); + } + } + + @Override + public RowChangeEvent deserialize(String topic, byte[] data) + { + if (data == null || data.length == 0) + { + return null; + } + try + { + return converter.convert(data); + } catch (Exception e) + { + LOGGER.warn("Failed to convert Debezium JSON row from topic {}", topic, e); + return null; + } + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowRecordDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowRecordDeserializer.java new file mode 100644 index 0000000..6dcb2ef --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowRecordDeserializer.java @@ -0,0 +1,53 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.source.kafka.serde; + +import com.google.protobuf.InvalidProtocolBufferException; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.conversion.sinkproto.RowRecordConverter; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.serialization.Deserializer; + +public final class RowRecordDeserializer implements Deserializer +{ + private final RowRecordConverter converter = + new RowRecordConverter(TableMetadataRegistry.Instance()); + + @Override + public RowChangeEvent deserialize(String topic, byte[] data) + { + if (data == null || data.length == 0) + { + return null; + } + try + { + return converter.convert(SinkProto.RowRecord.parseFrom(data)); + } catch (InvalidProtocolBufferException | SinkException e) + { + throw new SerializationException( + "Failed to deserialize canonical row record from " + topic, e); + } + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataAvroDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataAvroDeserializer.java new file mode 100644 index 0000000..0f116b1 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataAvroDeserializer.java @@ -0,0 +1,78 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.source.kafka.serde; + +import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumAvroTransactionConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapterRegistry; +import io.pixelsdb.pixels.sink.source.engine.adapter.DebeziumSourceAdapterSelector; +import org.apache.avro.generic.GenericRecord; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.serialization.Deserializer; + +import java.util.HashMap; +import java.util.Map; + +public class TransactionMetadataAvroDeserializer + implements Deserializer +{ + private final AvroKafkaDeserializer avroDeserializer = + new AvroKafkaDeserializer<>(); + private DebeziumAvroTransactionConverter converter = + new DebeziumAvroTransactionConverter( + DebeziumSourceAdapterSelector.configuredIfPresent()); + + @Override + public void configure(Map configs, boolean isKey) + { + Map enrichedConfig = new HashMap<>(configs); + enrichedConfig.put(SerdeConfig.CHECK_PERIOD_MS, SerdeConfig.CHECK_PERIOD_MS_DEFAULT); + avroDeserializer.configure(enrichedConfig, isKey); + Object connector = configs.get(PixelsSinkConstants.DEBEZIUM_CONNECTOR_CLASS); + if (connector != null && !connector.toString().isBlank()) + { + DebeziumSourceAdapter adapter = + DebeziumSourceAdapterRegistry.resolve(connector.toString()); + converter = new DebeziumAvroTransactionConverter(adapter); + } + } + + @Override + public SinkProto.TransactionMetadata deserialize(String topic, byte[] data) + { + if (data == null || data.length == 0) + { + return null; + } + try + { + return converter.convert(avroDeserializer.deserialize(topic, data)); + } catch (Exception e) + { + throw new SerializationException( + "Failed to deserialize Avro transaction message", e); + } + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataDeserializer.java new file mode 100644 index 0000000..f0237ca --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataDeserializer.java @@ -0,0 +1,47 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.source.kafka.serde; + +import com.google.protobuf.InvalidProtocolBufferException; +import io.pixelsdb.pixels.sink.SinkProto; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.serialization.Deserializer; + +public final class TransactionMetadataDeserializer + implements Deserializer +{ + @Override + public SinkProto.TransactionMetadata deserialize(String topic, byte[] data) + { + if (data == null || data.length == 0) + { + return null; + } + try + { + return SinkProto.TransactionMetadata.parseFrom(data); + } catch (InvalidProtocolBufferException e) + { + throw new SerializationException( + "Failed to deserialize canonical transaction metadata from " + topic, e); + } + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataJsonDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataJsonDeserializer.java new file mode 100644 index 0000000..6ed769c --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataJsonDeserializer.java @@ -0,0 +1,78 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.source.kafka.serde; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumEnvelopeNormalizer; +import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapterRegistry; +import io.pixelsdb.pixels.sink.source.engine.adapter.DebeziumSourceAdapterSelector; +import org.apache.kafka.common.serialization.Deserializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +public class TransactionMetadataJsonDeserializer + implements Deserializer +{ + private static final Logger LOGGER = + LoggerFactory.getLogger(TransactionMetadataJsonDeserializer.class); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private DebeziumSourceAdapter adapter = + DebeziumSourceAdapterSelector.configuredIfPresent(); + + @Override + public void configure(Map configs, boolean isKey) + { + Object connector = configs.get(PixelsSinkConstants.DEBEZIUM_CONNECTOR_CLASS); + if (connector != null && !connector.toString().isBlank()) + { + adapter = DebeziumSourceAdapterRegistry.resolve(connector.toString()); + } + } + + @Override + public SinkProto.TransactionMetadata deserialize(String topic, byte[] data) + { + if (data == null || data.length == 0) + { + return null; + } + try + { + JsonNode payload = OBJECT_MAPPER.readTree(data).path("payload"); + if (adapter == null) + { + throw new IllegalStateException( + "No Debezium source adapter configured for " + topic); + } + return DebeziumEnvelopeNormalizer.normalizeTransactionMetadata(payload, adapter); + } catch (Exception e) + { + LOGGER.error("Failed to deserialize transaction message from {}", topic, e); + throw new RuntimeException("Deserialization error", e); + } + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractMemorySinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractMemorySinkStorageSource.java index d3e0f7c..78f7e63 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractMemorySinkStorageSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractMemorySinkStorageSource.java @@ -50,8 +50,7 @@ public abstract class AbstractMemorySinkStorageSource extends AbstractSinkStorag public void start() { this.running.set(true); - this.transactionProcessorThread.start(); - this.transactionProviderThread.start(); + this.transactionPipeline.start(); try { /* ===================================================== diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractReaderSinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractReaderSinkStorageSource.java deleted file mode 100644 index 8be58ea..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractReaderSinkStorageSource.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.source.storage; - -import io.pixelsdb.pixels.common.physical.PhysicalReader; -import io.pixelsdb.pixels.common.physical.PhysicalReaderUtil; -import io.pixelsdb.pixels.common.physical.Storage; -import io.pixelsdb.pixels.core.utils.Pair; -import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; -import io.pixelsdb.pixels.sink.provider.ProtoType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.LinkedBlockingQueue; - -public abstract class AbstractReaderSinkStorageSource extends AbstractSinkStorageSource -{ - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractReaderSinkStorageSource.class); - - @Override - public void start() - { - this.running.set(true); - this.transactionProcessorThread.start(); - this.transactionProviderThread.start(); - for (String file : files) - { - Storage.Scheme scheme = Storage.Scheme.fromPath(file); - LOGGER.info("Start read from file {}", file); - PhysicalReader reader; - try - { - reader = PhysicalReaderUtil.newPhysicalReader(scheme, file); - } catch (IOException e) - { - throw new RuntimeException(e); - } - readers.add(reader); - } - do - { - for (PhysicalReader reader : readers) - { - LOGGER.info("Start Read {}", reader.getPath()); - long offset = 0; - while (true) - { - try - { - int key, valueLen; - reader.seek(offset); - try - { - key = reader.readInt(ByteOrder.BIG_ENDIAN); - valueLen = reader.readInt(ByteOrder.BIG_ENDIAN); - } catch (IOException e) - { - // EOF - break; - } - - ProtoType protoType = getProtoType(key); - offset += Integer.BYTES * 2; - CompletableFuture valueFuture = reader.readAsync(offset, valueLen) - .thenApply(this::copyToHeap) - .thenApply(buf -> buf.order(ByteOrder.BIG_ENDIAN)); - // move offset for next record - offset += valueLen; - - - // Get or create queue - BlockingQueue, Integer>> queue = - queueMap.computeIfAbsent(key, - k -> new LinkedBlockingQueue<>(PixelsSinkConstants.MAX_QUEUE_SIZE)); - - // Put future in queue - if (protoType.equals(ProtoType.ROW)) - { - sourceRateLimiter.acquire(1); - } - queue.put(new Pair<>(valueFuture, loopId)); - // Start consumer thread if not exists - consumerThreads.computeIfAbsent(key, k -> - { - Thread t = new Thread(() -> consumeQueue(k, queue, protoType)); - t.setName("consumer-" + key); - t.start(); - return t; - }); - } catch (IOException | InterruptedException e) - { - break; - } - } - } - ++loopId; - } while (storageLoopEnabled && isRunning()); - - clean(); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java index f65ae60..7c72288 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java @@ -24,13 +24,17 @@ import io.pixelsdb.pixels.core.utils.Pair; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.conversion.sinkproto.RowRecordConverter; +import io.pixelsdb.pixels.sink.conversion.sinkproto.TransactionMetadataConverter; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; -import io.pixelsdb.pixels.sink.processor.TransactionProcessor; +import io.pixelsdb.pixels.sink.pipeline.TablePipelineManager; +import io.pixelsdb.pixels.sink.pipeline.TransactionPipeline; import io.pixelsdb.pixels.sink.provider.ProtoType; -import io.pixelsdb.pixels.sink.provider.TableProviderAndProcessorPipelineManager; -import io.pixelsdb.pixels.sink.provider.TransactionEventStorageLoopProvider; import io.pixelsdb.pixels.sink.source.SinkSource; import io.pixelsdb.pixels.sink.util.EtcdFileRegistry; +import io.pixelsdb.pixels.sink.util.DataTransform; import io.pixelsdb.pixels.sink.util.MetricsFacade; import io.pixelsdb.pixels.sink.util.rateLimiter.FlushRateLimiter; import io.pixelsdb.pixels.sink.util.rateLimiter.FlushRateLimiterFactory; @@ -62,13 +66,13 @@ public abstract class AbstractSinkStorageSource implements SinkSource protected final Map, Integer>>> queueMap = new ConcurrentHashMap<>(); protected final boolean storageLoopEnabled; protected final FlushRateLimiter sourceRateLimiter; - private final TableMetadataRegistry tableMetadataRegistry = TableMetadataRegistry.Instance(); + protected final TablePipelineManager tablePipelineManager = new TablePipelineManager(); + protected final TransactionPipeline transactionPipeline = new TransactionPipeline(); + protected final RowRecordConverter rowRecordConverter; + protected final TransactionMetadataConverter transactionMetadataConverter = + new TransactionMetadataConverter(); + protected final boolean freshnessTimestamp; private final MetricsFacade metricsFacade = MetricsFacade.getInstance(); - private final TableProviderAndProcessorPipelineManager> tablePipelineManager = new TableProviderAndProcessorPipelineManager<>(); - protected TransactionEventStorageLoopProvider> transactionEventProvider; - protected TransactionProcessor transactionProcessor; - protected Thread transactionProviderThread; - protected Thread transactionProcessorThread; protected int loopId = 0; protected List readers = new ArrayList<>(); @@ -80,12 +84,8 @@ protected AbstractSinkStorageSource() this.etcdFileRegistry = new EtcdFileRegistry(topic, baseDir); this.files = this.etcdFileRegistry.listAllFiles(); this.storageLoopEnabled = pixelsSinkConfig.isSinkStorageLoop(); - - this.transactionEventProvider = new TransactionEventStorageLoopProvider<>(); - this.transactionProviderThread = new Thread(transactionEventProvider); - - this.transactionProcessor = new TransactionProcessor(transactionEventProvider); - this.transactionProcessorThread = new Thread(transactionProcessor, "debezium-processor"); + this.rowRecordConverter = new RowRecordConverter(TableMetadataRegistry.Instance()); + this.freshnessTimestamp = pixelsSinkConfig.isSinkMonitorFreshnessTimestamp(); this.sourceRateLimiter = FlushRateLimiterFactory.getNewInstance(); } @@ -125,11 +125,22 @@ protected void clean() LOGGER.warn("Failed to close reader", e); } } + tablePipelineManager.close(); + transactionPipeline.close(); } protected void handleTransactionSourceRecord(ByteBuffer record, Integer loopId) { - transactionEventProvider.putTransRawEvent(new Pair<>(record, loopId)); + try + { + SinkProto.TransactionMetadata metadata = + transactionMetadataConverter.convert(record, loopId); + metricsFacade.recordSerdTxChange(); + transactionPipeline.publish(metadata); + } catch (Exception e) + { + LOGGER.warn("Failed to convert storage transaction metadata", e); + } } protected void consumeQueue(int key, BlockingQueue, Integer>> queue, ProtoType protoType) @@ -173,7 +184,28 @@ protected ByteBuffer copyToHeap(ByteBuffer directBuffer) protected void handleRowChangeSourceRecord(int key, ByteBuffer dataBuffer, int loopId) { - tablePipelineManager.routeRecord(key, new Pair<>(dataBuffer, loopId)); + try + { + SinkProto.RowRecord.Builder builder = + rowRecordConverter.parse(dataBuffer).toBuilder(); + if (freshnessTimestamp) + { + DataTransform.updateRecordTimestamp(builder, System.currentTimeMillis() * 1000); + } + if (builder.hasTransaction()) + { + SinkProto.TransactionInfo transaction = builder.getTransaction(); + builder.setTransaction(transaction.toBuilder() + .setId(transaction.getId() + "_" + loopId) + .build()); + } + RowChangeEvent event = rowRecordConverter.convert(builder.build()); + metricsFacade.recordSerdRowChange(); + tablePipelineManager.route(event); + } catch (Exception e) + { + LOGGER.warn("Failed to convert storage row record", e); + } } @Override @@ -186,8 +218,7 @@ public boolean isRunning() public void stopProcessor() { running.set(false); - transactionProviderThread.interrupt(); - transactionProcessorThread.interrupt(); - transactionProcessor.stopProcessor(); + tablePipelineManager.close(); + transactionPipeline.close(); } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/FasterSinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/FasterSinkStorageSource.java index fc49922..6e76cba 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/storage/FasterSinkStorageSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/storage/FasterSinkStorageSource.java @@ -31,7 +31,7 @@ /** * @package: io.pixelsdb.pixels.sink.source - * @className: LegacySinkStorageSource + * @className: FasterSinkStorageSource * @author: AntiO2 * @date: 2025/10/5 11:43 */ diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/LegacySinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/LegacySinkStorageSource.java deleted file mode 100644 index a8d636b..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/source/storage/LegacySinkStorageSource.java +++ /dev/null @@ -1,247 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.source.storage; - - -import io.pixelsdb.pixels.common.metadata.SchemaTableName; -import io.pixelsdb.pixels.common.physical.PhysicalReader; -import io.pixelsdb.pixels.common.physical.PhysicalReaderUtil; -import io.pixelsdb.pixels.common.physical.Storage; -import io.pixelsdb.pixels.core.utils.Pair; -import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; -import io.pixelsdb.pixels.sink.processor.TransactionProcessor; -import io.pixelsdb.pixels.sink.provider.ProtoType; -import io.pixelsdb.pixels.sink.provider.TableProviderAndProcessorPipelineManager; -import io.pixelsdb.pixels.sink.provider.TransactionEventEngineProvider; -import io.pixelsdb.pixels.sink.source.SinkSource; -import io.pixelsdb.pixels.sink.util.MetricsFacade; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.Map; -import java.util.concurrent.*; - -/** - * @package: io.pixelsdb.pixels.sink.source - * @className: LegacySinkStorageSource - * @author: AntiO2 - * @date: 2025/10/5 11:43 - */ -@Deprecated -public class LegacySinkStorageSource extends AbstractReaderSinkStorageSource implements SinkSource -{ - private static final Logger LOGGER = LoggerFactory.getLogger(LegacySinkStorageSource.class); - static SchemaTableName transactionSchemaTableName = new SchemaTableName("freak", "transaction"); - private final TransactionEventEngineProvider transactionEventProvider = TransactionEventEngineProvider.INSTANCE; - - private final TableProviderAndProcessorPipelineManager tableProvidersManagerImpl = new TableProviderAndProcessorPipelineManager<>(); - private final TransactionProcessor transactionProcessor = new TransactionProcessor(transactionEventProvider); - private final MetricsFacade metricsFacade = MetricsFacade.getInstance(); - private final Map>> queueMap = new ConcurrentHashMap<>(); - private final Map consumerThreads = new ConcurrentHashMap<>(); - private final int maxQueueCapacity = 10000; - private final TableMetadataRegistry tableMetadataRegistry = TableMetadataRegistry.Instance(); - private final CompletableFuture POISON_PILL = new CompletableFuture<>(); - - - private static String readString(ByteBuffer buffer, int len) - { - byte[] bytes = new byte[len]; - buffer.get(bytes); - return new String(bytes); - } - - @Override - ProtoType getProtoType(int i) - { - return ProtoType.fromInt(i); - } - - @Override - public void start() - { - - for (String file : files) - { - Storage.Scheme scheme = Storage.Scheme.fromPath(file); - LOGGER.info("Start read from file {}", file); - try (PhysicalReader reader = PhysicalReaderUtil.newPhysicalReader(scheme, file)) - { - long offset = 0; - BlockingQueue>> rowQueue = new LinkedBlockingQueue<>(); - BlockingQueue> transQueue = new LinkedBlockingQueue<>(); - while (true) - { - try - { - int keyLen, valueLen; - reader.seek(offset); - try - { - keyLen = reader.readInt(ByteOrder.BIG_ENDIAN); - valueLen = reader.readInt(ByteOrder.BIG_ENDIAN); - } catch (IOException e) - { - // EOF - break; - } - - ByteBuffer keyBuffer = copyToHeap(reader.readFully(keyLen)).order(ByteOrder.BIG_ENDIAN); - ProtoType protoType = getProtoType(keyBuffer.getInt()); - offset += Integer.BYTES * 2 + keyLen; - CompletableFuture valueFuture = reader.readAsync(offset, valueLen) - .thenApply(this::copyToHeap) - .thenApply(buf -> buf.order(ByteOrder.BIG_ENDIAN)); - // move offset for next record - offset += valueLen; - - // Compute queue key (for example: schemaName + tableName or protoType) - SchemaTableName queueKey = computeQueueKey(keyBuffer, protoType); - - // Get or create queue - BlockingQueue> queue = - queueMap.computeIfAbsent(queueKey, - k -> new LinkedBlockingQueue<>(maxQueueCapacity)); - - // Put future in queue - queue.put(valueFuture); - - // Start consumer thread if not exists - consumerThreads.computeIfAbsent(queueKey, k -> - { - Thread t = new Thread(() -> consumeQueue(k, queue, protoType)); - t.setName("consumer-" + queueKey); - t.start(); - return t; - }); - } catch (IOException | InterruptedException e) - { - break; - } - } - } catch (IOException e) - { - throw new RuntimeException(e); - } - - } - - // signal all queues to stop - queueMap.values().forEach(q -> - { - try - { - q.put(POISON_PILL); - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - } - }); - - // wait all consumers to finish - consumerThreads.values().forEach(t -> - { - try - { - t.join(); - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - } - }); - } - - private void consumeQueue(SchemaTableName key, BlockingQueue> queue, ProtoType protoType) - { - try - { - while (true) - { - CompletableFuture value = queue.take(); - if (value == POISON_PILL) - { - break; - } - ByteBuffer valueBuffer = value.get(); - metricsFacade.recordDebeziumEvent(); - switch (protoType) - { - case ROW -> handleRowChangeSourceRecord(0, valueBuffer, 0); - case TRANS -> handleTransactionSourceRecord(valueBuffer, 0); - } - } - } catch (InterruptedException e) - { - Thread.currentThread().interrupt(); - } catch (ExecutionException e) - { - LOGGER.error("Error in async processing", e); - } - } - - private SchemaTableName computeQueueKey(ByteBuffer keyBuffer, ProtoType protoType) - { - switch (protoType) - { - case ROW -> - { - int schemaLen = keyBuffer.getInt(); - int tableLen = keyBuffer.getInt(); - String schemaName = readString(keyBuffer, schemaLen); - String tableName = readString(keyBuffer, tableLen); - return new SchemaTableName(schemaName, tableName); - } - case TRANS -> - { - return transactionSchemaTableName; - } - default -> - { - throw new IllegalArgumentException("Proto type " + protoType.toString()); - } - } - } - - private void handleRowChangeSourceRecord(SchemaTableName schemaTableName, ByteBuffer dataBuffer) - { - tableProvidersManagerImpl.routeRecord(schemaTableName, dataBuffer); - } - - private void handleRowChangeSourceRecord(ByteBuffer keyBuffer, ByteBuffer dataBuffer) - { - { - // CODE BLOCK VERSION 2 -// long tableId = keyBuffer.getLong(); -// try -// { -// schemaTableName = tableMetadataRegistry.getSchemaTableName(tableId); -// } catch (SinkException e) -// { -// throw new RuntimeException(e); -// } - } - -// tableProvidersManagerImpl.routeRecord(schemaTableName, dataBuffer); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/DataTransform.java b/src/main/java/io/pixelsdb/pixels/sink/util/DataTransform.java index 92b656e..d0c1f24 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/DataTransform.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/DataTransform.java @@ -21,7 +21,6 @@ package io.pixelsdb.pixels.sink.util; import com.google.protobuf.ByteString; -import io.pixelsdb.pixels.retina.RetinaProto; import io.pixelsdb.pixels.sink.SinkProto; import java.nio.ByteBuffer; @@ -36,44 +35,6 @@ private static ByteString longToByteString(long value) return ByteString.copyFrom(bytes); } - private static int byteStringToInt(ByteString bytes) - { - return ByteBuffer.wrap(bytes.toByteArray()).getInt(); - } - - @Deprecated - public static void updateTimeStamp(List updateData, long txStartTime) - { - ByteString timestampBytes = longToByteString(txStartTime); - - for (RetinaProto.TableUpdateData.Builder tableUpdateDataBuilder : updateData) - { - int insertDataCount = tableUpdateDataBuilder.getInsertDataCount(); - for (int i = 0; i < insertDataCount; i++) - { - RetinaProto.InsertData.Builder insertBuilder = tableUpdateDataBuilder.getInsertDataBuilder(i); - int colValueCount = insertBuilder.getColValuesCount(); - if (colValueCount > 0) - { - insertBuilder.setColValues(colValueCount - 1, timestampBytes); - } - } - - int updateDataCount = tableUpdateDataBuilder.getUpdateDataCount(); - for (int i = 0; i < updateDataCount; i++) - { - RetinaProto.UpdateData.Builder updateBuilder = tableUpdateDataBuilder.getUpdateDataBuilder(i); - - int colValueCount = updateBuilder.getColValuesCount(); - if (colValueCount > 0) - { - updateBuilder.setColValues(colValueCount - 1, timestampBytes); - } - } - } - } - - public static List updateRecordTimestamp(List records, long timestamp) { if (records == null || records.isEmpty()) @@ -129,22 +90,6 @@ public static void updateRecordTimestamp(SinkProto.RowRecord.Builder recordBuild } } - public static void transIdToBigint(SinkProto.RowRecord.Builder recordBuilder) - { - - if (recordBuilder.hasAfter()) - { - SinkProto.RowValue.Builder afterBuilder = recordBuilder.getAfterBuilder(); - afterBuilder.setValues(0, getTimestampColumn(byteStringToInt(afterBuilder.getValues(0).getValue()))); - } - if (recordBuilder.hasBefore()) - { - SinkProto.RowValue.Builder beforeBuilder = recordBuilder.getBeforeBuilder(); - beforeBuilder.setValues(0, getTimestampColumn(byteStringToInt(beforeBuilder.getValues(0).getValue()))); - } - } - - private static SinkProto.RowRecord updateRecordTimestamp(SinkProto.RowRecord.Builder recordBuilder, SinkProto.ColumnValue timestampColumn) { switch (recordBuilder.getOp()) diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/DateUtil.java b/src/main/java/io/pixelsdb/pixels/sink/util/DateUtil.java index 5b6735c..e630270 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/DateUtil.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/DateUtil.java @@ -21,15 +21,8 @@ package io.pixelsdb.pixels.sink.util; -import io.pixelsdb.pixels.core.utils.DatetimeUtils; - import java.text.DateFormat; import java.text.SimpleDateFormat; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.Calendar; import java.util.Date; /** @@ -41,29 +34,6 @@ public class DateUtil { - public static Date fromDebeziumDate(int epochDay) - { - Calendar cal = Calendar.getInstance(); - cal.clear(); - cal.set(1970, Calendar.JANUARY, 1); // epoch 起点 - cal.add(Calendar.DAY_OF_MONTH, epochDay); // 加上天数 - return cal.getTime(); - } - - // TIMESTAMP(1), TIMESTAMP(2), TIMESTAMP(3) - public static Date fromDebeziumTimestamp(long epochTs) - { - return new Date(epochTs / 1000); - } - - public static String convertDateToDayString(Date date) - { - // "yyyy-MM-dd HH:mm:ss.SSS" - DateFormat df = new SimpleDateFormat("yyyy-MM-dd"); - String dateToString = df.format(date); - return (dateToString); - } - public static String convertDateToString(Date date) { // "yyyy-MM-dd HH:mm:ss.SSS" @@ -72,20 +42,4 @@ public static String convertDateToString(Date date) return (dateToString); } - public static String convertTimestampToString(Date date) - { - if (date == null) - { - return null; - } - SimpleDateFormat sdf = new SimpleDateFormat("HH:mm:ss.SSS"); - return sdf.format(date); - } - - public static String convertDebeziumTimestampToString(long epochTs) - { - LocalDateTime dateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(epochTs), ZoneId.systemDefault()); - DateTimeFormatter formatter = DatetimeUtils.SQL_LOCAL_DATE_TIME; - return dateTime.format(formatter); - } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistry.java b/src/main/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistry.java index a5330cf..b06b8b8 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistry.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistry.java @@ -191,8 +191,4 @@ public void markFileCompleted(String fileName) LOGGER.info("Marked file [{}] as completed", fileName); } - public void cleanData() - { - etcd.deleteByPrefix(topicPrefix()); - } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/LatencySimulator.java b/src/main/java/io/pixelsdb/pixels/sink/util/LatencySimulator.java deleted file mode 100644 index ad3f86c..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/util/LatencySimulator.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.util; - -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; - -import java.util.Random; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; - -public class LatencySimulator -{ - private static final Random RANDOM = ThreadLocalRandom.current(); - private static final double longTailProb = 0.05; - private static final double longTailScale = 30; - private static final double tailVariance = 0.1; - private static final double normalVariance = 0.4; - - private static long generateLongTailDelay(long baseDelayMs) - { - if (RANDOM.nextDouble() < longTailProb) - { - double variance = 1 + (RANDOM.nextDouble() * 2 - 1) * tailVariance; - return (long) (baseDelayMs * longTailScale * variance); - } - - return (long) (baseDelayMs * (1 + (RANDOM.nextDouble() - 0.5) * normalVariance)); - } - - public static void smartDelay() - { - try - { - TimeUnit.MILLISECONDS.sleep(generateLongTailDelay(PixelsSinkConfigFactory.getInstance().getMockRpcDelay())); - } catch (InterruptedException ignored) - { - - } - } -} \ No newline at end of file diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriter.java index eaf4f3b..32cbee3 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriter.java @@ -90,7 +90,6 @@ public boolean writeRow(RowChangeEvent event) } metricsFacade.recordRowChange(event.getTable(), event.getOp()); - event.startLatencyTimer(); if (event.getTransaction() == null || event.getTransaction().getId().isEmpty()) { handleNonTxEvent(event); diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableCrossTxWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableCrossTxWriter.java index c73864b..d720542 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableCrossTxWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableCrossTxWriter.java @@ -106,13 +106,6 @@ public void flush(List batch) // flushRateLimiter.acquire(batch.size()); long txStartTime = System.currentTimeMillis(); -// if(freshnessLevel.equals("embed")) -// { -// long freshness_ts = txStartTime * 1000; -// FreshnessClient.getInstance().addMonitoredTable(tableName); -// DataTransform.updateTimeStamp(tableUpdateDataBuilderList, freshness_ts); -// } - List tableUpdateData = new ArrayList<>(tableUpdateDataBuilderList.size()); for (RetinaProto.TableUpdateData.Builder tableUpdateDataItem : tableUpdateDataBuilderList) { diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TransactionProxy.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TransactionProxy.java index 8128a7e..8514dbd 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TransactionProxy.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TransactionProxy.java @@ -125,12 +125,6 @@ private void requestTransactions() } } - @Deprecated - public TransContext getNewTransContext() - { - return getNewTransContext("None"); - } - public TransContext getNewTransContext(String txId) { beginCount.incrementAndGet(); diff --git a/src/main/resources/META-INF/services/io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter b/src/main/resources/META-INF/services/io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter new file mode 100644 index 0000000..2ea4ccd --- /dev/null +++ b/src/main/resources/META-INF/services/io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter @@ -0,0 +1,2 @@ +io.pixelsdb.pixels.sink.conversion.debezium.source.MySqlSourceAdapter +io.pixelsdb.pixels.sink.conversion.debezium.source.PostgresSourceAdapter diff --git a/src/main/resources/pixels-sink.aws.properties b/src/main/resources/pixels-sink.aws.properties index 5cabc00..b99cbae 100644 --- a/src/main/resources/pixels-sink.aws.properties +++ b/src/main/resources/pixels-sink.aws.properties @@ -7,8 +7,8 @@ bootstrap.servers=realtime-kafka-2:29092 group.id=3078 auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -#value.deserializer=io.pixelsdb.pixels.writer.deserializer.RowChangeEventAvroDeserializer -value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventJsonDeserializer +#value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventAvroDeserializer +value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer # Topic & Database Config topic.prefix=postgresql.oltp_server consumer.capture_database=pixels_bench_sf1x @@ -38,8 +38,8 @@ sink.proto.maxRecords=1000000 sink.registry.url=http://localhost:8080/apis/registry/v2 # Transaction Config transaction.topic.suffix=transaction -#transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.deserializer.TransactionAvroMessageDeserializer -transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.TransactionJsonMessageDeserializer +#transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataAvroDeserializer +transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer sink.trans.batch.size=200 ## Batch or trans or record sink.trans.mode=batch @@ -50,7 +50,6 @@ sink.monitor.report.interval=5000 sink.monitor.report.file=/home/ubuntu/pixels-sink/tmp/readonly3.csv # Interact with other rpc sink.rpc.enable=true -sink.rpc.mock.delay=20 # debezium debezium.name=testEngine debezium.connector.class=io.debezium.connector.postgresql.PostgresConnector diff --git a/src/main/resources/pixels-sink.local.properties b/src/main/resources/pixels-sink.local.properties index 360cfb1..0b24c54 100644 --- a/src/main/resources/pixels-sink.local.properties +++ b/src/main/resources/pixels-sink.local.properties @@ -7,8 +7,8 @@ bootstrap.servers=localhost:29092 group.id=3107 auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -#value.deserializer=io.pixelsdb.pixels.sink.deserializer.RowChangeEventAvroDeserializer -value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventJsonDeserializer +#value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventAvroDeserializer +value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer # Topic & Database Config topic.prefix=postgresql.oltp_server consumer.capture_database=pixels_bench_sf1x @@ -39,8 +39,8 @@ sink.proto.maxRecords=1000000 sink.registry.url=http://localhost:8080/apis/registry/v2 # Transaction Config transaction.topic.suffix=transaction -#transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.deserializer.TransactionAvroMessageDeserializer -transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.TransactionJsonMessageDeserializer +#transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataAvroDeserializer +transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer sink.trans.batch.size=100 ## Batch or trans or record sink.trans.mode=batch @@ -49,7 +49,6 @@ sink.monitor.enable=true sink.monitor.port=9464 # Interact with other rpc sink.rpc.enable=true -sink.rpc.mock.delay=20 # debezium debezium.name=testEngine debezium.connector.class=io.debezium.connector.postgresql.PostgresConnector diff --git a/src/main/resources/pixels-sink.properties b/src/main/resources/pixels-sink.properties index a903fea..83247f3 100644 --- a/src/main/resources/pixels-sink.properties +++ b/src/main/resources/pixels-sink.properties @@ -7,7 +7,7 @@ bootstrap.servers=pixels_kafka:9092 group.id=docker-pixels-table auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventJsonDeserializer +value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer # Topic & Database Config topic.prefix=oltp_server @@ -40,7 +40,7 @@ sink.registry.url=http://apicurio:8080/apis/registry/v2 # Transaction Config transaction.topic.suffix=transaction transaction.topic.group_id=transaction_consumer -transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.event.deserializer.TransactionJsonMessageDeserializer +transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer sink.trans.batch.size=100 sink.trans.mode=batch transaction.timeout=300 @@ -51,4 +51,3 @@ sink.monitor.port=9464 # Interact with other rpc sink.rpc.enable=false -sink.rpc.mock.delay=100 diff --git a/src/test/java/io/pixelsdb/pixels/sink/DebeziumEngineTest.java b/src/test/java/io/pixelsdb/pixels/sink/DebeziumEngineTest.java index 58659ff..b238ed1 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/DebeziumEngineTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/DebeziumEngineTest.java @@ -24,6 +24,7 @@ import io.debezium.engine.RecordChangeEvent; import io.debezium.engine.format.ChangeEventFormat; import org.apache.kafka.connect.source.SourceRecord; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.util.List; @@ -40,6 +41,7 @@ public class DebeziumEngineTest { @Test + @Disabled("Manual PostgreSQL integration test; requires an external database") public void testPostgresCDC() { final Properties props = new Properties(); diff --git a/src/test/java/io/pixelsdb/pixels/sink/consumer/AvroConsumerTest.java b/src/test/java/io/pixelsdb/pixels/sink/consumer/AvroConsumerTest.java index 3ee867d..b4b636c 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/consumer/AvroConsumerTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/consumer/AvroConsumerTest.java @@ -23,8 +23,8 @@ import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; import io.pixelsdb.pixels.sink.SinkProto; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventAvroDeserializer; import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.event.deserializer.RowChangeEventAvroDeserializer; import io.pixelsdb.pixels.sink.exception.SinkException; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizerTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizerTest.java new file mode 100644 index 0000000..e2acabd --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizerTest.java @@ -0,0 +1,251 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + */ +package io.pixelsdb.pixels.sink.conversion.debezium; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.pixelsdb.pixels.common.metadata.SchemaTableName; +import io.pixelsdb.pixels.common.metadata.domain.Table; +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapterRegistry; +import io.pixelsdb.pixels.sink.conversion.debezium.source.MySqlSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.source.PostgresSourceAdapter; +import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.apache.kafka.connect.data.Struct; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class DebeziumEnvelopeNormalizerTest +{ + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final SchemaTableName MYSQL_TABLE = + new SchemaTableName("cdc_verify", "binlog_test"); + private static final SchemaTableName POSTGRES_TABLE = + new SchemaTableName("pixels_realtime_crud", "region"); + private static TableMetadata previousMySqlMetadata; + private static TableMetadata previousPostgresMetadata; + private static boolean hadMySqlMetadata; + private static boolean hadPostgresMetadata; + + @BeforeAll + static void setUpConfig() throws Exception + { + PixelsSinkConfigFactory.reset(); + PixelsSinkConfigFactory.initialize(Objects.requireNonNull( + DebeziumEnvelopeNormalizerTest.class.getClassLoader() + .getResource("pixels-sink-test.properties")).getPath()); + Map registry = metadataRegistry(); + hadMySqlMetadata = registry.containsKey(MYSQL_TABLE); + previousMySqlMetadata = registry.get(MYSQL_TABLE); + hadPostgresMetadata = registry.containsKey(POSTGRES_TABLE); + previousPostgresMetadata = registry.get(POSTGRES_TABLE); + registry.put(MYSQL_TABLE, tableMetadata(1, "binlog_test")); + registry.put(POSTGRES_TABLE, tableMetadata(2, "region")); + } + + @AfterAll + static void resetConfig() throws ReflectiveOperationException + { + Map registry = metadataRegistry(); + if (hadMySqlMetadata) + { + registry.put(MYSQL_TABLE, previousMySqlMetadata); + } + else + { + registry.remove(MYSQL_TABLE); + } + if (hadPostgresMetadata) + { + registry.put(POSTGRES_TABLE, previousPostgresMetadata); + } + else + { + registry.remove(POSTGRES_TABLE); + } + PixelsSinkConfigFactory.reset(); + } + + @Test + void shouldNormalizeMySqlSourceWithoutSchema() throws SinkException + { + Struct source = mysqlSource(); + + SinkProto.SourceInfo normalized = DebeziumEnvelopeNormalizer.normalizeSource( + source, MySqlSourceAdapter.INSTANCE); + + assertEquals("cdc_verify", normalized.getDb()); + assertEquals("", normalized.getSchema()); + assertEquals("binlog_test", normalized.getTable()); + assertEquals("cdc_verify.binlog_test", rowChangeEventFor(normalized).getFullTableName()); + } + + @Test + void shouldNormalizePostgresSourceAndTransactionId() throws SinkException + { + Schema sourceSchema = SchemaBuilder.struct() + .field("connector", Schema.STRING_SCHEMA) + .field("db", Schema.STRING_SCHEMA) + .field("schema", Schema.STRING_SCHEMA) + .field("table", Schema.STRING_SCHEMA) + .build(); + Struct source = new Struct(sourceSchema) + .put("connector", "postgresql") + .put("db", "pixels_realtime_crud") + .put("schema", "public") + .put("table", "region"); + + SinkProto.SourceInfo normalized = DebeziumEnvelopeNormalizer.normalizeSource( + source, PostgresSourceAdapter.INSTANCE); + + assertEquals("pixels_realtime_crud", normalized.getDb()); + assertEquals("public", normalized.getSchema()); + assertEquals("region", normalized.getTable()); + assertEquals("public.region", rowChangeEventFor(normalized).getFullTableName()); + assertEquals("779", + PostgresSourceAdapter.INSTANCE.normalizeTransactionId("779:39110592")); + assertEquals("779", + PostgresSourceAdapter.INSTANCE.normalizeTransactionId("779")); + } + + @Test + void shouldRejectConfiguredConnectorMismatch() + { + IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, + () -> DebeziumEnvelopeNormalizer.normalizeSource( + mysqlSource(), PostgresSourceAdapter.INSTANCE)); + + assertTrue(error.getMessage().contains("source.connector mysql")); + } + + @Test + void shouldResolveConnectorAdaptersThroughSpi() + { + assertEquals("mysql", + DebeziumSourceAdapterRegistry.resolve( + "io.debezium.connector.mysql.MySqlConnector").connector()); + assertEquals("postgresql", + DebeziumSourceAdapterRegistry.resolve("postgresql").connector()); + } + + @Test + void shouldPreserveMySqlGtidAsOpaqueTransactionId() throws Exception + { + String gtid = "8d02ae5e-aeff-ca52-daa1-ab14474cee00:3"; + JsonNode end = OBJECT_MAPPER.readTree(""" + { + "status": "END", + "id": "%s", + "event_count": 4, + "ts_ms": 1750000000000, + "data_collections": [ + {"data_collection": "cdc_verify.binlog_test", "event_count": 4} + ] + } + """.formatted(gtid)); + + SinkProto.TransactionMetadata transaction = + DebeziumEnvelopeNormalizer.normalizeTransactionMetadata( + end, MySqlSourceAdapter.INSTANCE); + + assertEquals(SinkProto.TransactionStatus.END, transaction.getStatus()); + assertEquals(gtid, transaction.getId()); + assertEquals(4, transaction.getEventCount()); + assertEquals("cdc_verify.binlog_test", + transaction.getDataCollections(0).getDataCollection()); + } + + @Test + void shouldRoundTripCanonicalProto() throws Exception + { + SinkProto.RowRecord record = SinkProto.RowRecord.newBuilder() + .setOp(SinkProto.OperationType.INSERT) + .setSource(DebeziumEnvelopeNormalizer.normalizeSource( + mysqlSource(), MySqlSourceAdapter.INSTANCE)) + .setTransaction(SinkProto.TransactionInfo.newBuilder() + .setId("gtid:8") + .setTotalOrder(1) + .setDataCollectionOrder(1)) + .setAfter(SinkProto.RowValue.newBuilder() + .addValues(SinkProto.ColumnValue.newBuilder() + .setValue(com.google.protobuf.ByteString.copyFromUtf8("")) + .setIsNull(false)) + .addValues(SinkProto.ColumnValue.newBuilder() + .setIsNull(true))) + .build(); + + SinkProto.RowRecord restored = + SinkProto.RowRecord.parseFrom(record.toByteArray()); + + assertEquals(record, restored); + assertTrue(restored.getAfter().getValues(1).getIsNull()); + } + + private Struct mysqlSource() + { + Schema sourceSchema = SchemaBuilder.struct() + .field("connector", Schema.STRING_SCHEMA) + .field("db", Schema.STRING_SCHEMA) + .field("table", Schema.STRING_SCHEMA) + .build(); + return new Struct(sourceSchema) + .put("connector", "mysql") + .put("db", "cdc_verify") + .put("table", "binlog_test"); + } + + private static RowChangeEvent rowChangeEventFor(SinkProto.SourceInfo source) throws SinkException + { + return new RowChangeEvent( + SinkProto.RowRecord.newBuilder() + .setSource(source) + .setOp(SinkProto.OperationType.SNAPSHOT) + .setAfter(SinkProto.RowValue.newBuilder().build()) + .build(), + TypeDescription.createSchemaFromStrings(new ArrayList<>(), new ArrayList<>())); + } + + private static TableMetadata tableMetadata(long id, String name) throws Exception + { + Table table = new Table(); + table.setId(id); + table.setName(name); + return new TableMetadata(table, null, List.of()); + } + + @SuppressWarnings("unchecked") + private static Map metadataRegistry() + throws ReflectiveOperationException + { + Field registry = TableMetadataRegistry.class.getDeclaredField("registry"); + registry.setAccessible(true); + return (Map) registry.get(TableMetadataRegistry.Instance()); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverterTest.java new file mode 100644 index 0000000..a8eae45 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverterTest.java @@ -0,0 +1,169 @@ +/* + * Copyright 2025 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.pixelsdb.pixels.sink.conversion.debezium; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.util.TestDateUtil; +import org.apache.avro.generic.GenericData; +import org.apache.kafka.connect.data.Decimal; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.apache.kafka.connect.data.Struct; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.Base64; +import java.util.Date; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class DebeziumRowValueConverterTest +{ + // BqQ= 17 + // JbR7 24710.35 +// @ParameterizedTest +// @CsvSource({ +// // encodedValue, expectedValue, precision, scale +// "BqQ=, 17.00, 15, 2", +// "JbR7, 24710.35, 15, 2", +// }) +// void testParseDecimalValid(String encodedValue, String expectedValue, int precision, int scale) { +// JsonNode node = new TextNode(encodedValue); +// TypeDescription type = TypeDescription.createDecimal(precision, scale); +// DebeziumRowValueConverter rowDataParser = new DebeziumRowValueConverter(type); +// BigDecimal result = rowDataParser.parseDecimal(node, type); +// assertEquals(new BigDecimal(expectedValue), result); +// } + + @Test + void shouldEncodeCanonicalStructValues() throws Exception + { + TypeDescription typeDescription = TypeDescription.createSchemaFromStrings( + List.of("id", "name", "amount", "note", "empty_value"), + List.of("bigint", "varchar(64)", "decimal(10,2)", "varchar(64)", "varchar(64)")); + Schema rowSchema = SchemaBuilder.struct() + .field("id", Schema.INT64_SCHEMA) + .field("name", Schema.STRING_SCHEMA) + .field("amount", Decimal.builder(2) + .parameter("connect.decimal.precision", "10").build()) + .field("note", Schema.OPTIONAL_STRING_SCHEMA) + .field("empty_value", Schema.STRING_SCHEMA) + .build(); + Struct row = new Struct(rowSchema) + .put("id", 9223372036854775806L) + .put("name", "TDSQL value ") + .put("amount", new BigDecimal("24710.35")) + .put("note", null) + .put("empty_value", ""); + + SinkProto.RowValue.Builder builder = SinkProto.RowValue.newBuilder(); + new DebeziumRowValueConverter(typeDescription).parse(row, builder); + SinkProto.RowValue value = builder.build(); + + assertEquals(9223372036854775806L, + ByteBuffer.wrap(value.getValues(0).getValue().toByteArray()).getLong()); + assertEquals("TDSQL value ", value.getValues(1).getValue().toStringUtf8()); + assertEquals("24710.35", value.getValues(2).getValue().toStringUtf8()); + assertTrue(value.getValues(3).getIsNull()); + assertEquals(0, value.getValues(3).getValue().size()); + assertFalse(value.getValues(4).getIsNull()); + assertEquals("", value.getValues(4).getValue().toStringUtf8()); + + ObjectNode jsonRow = new ObjectMapper().createObjectNode(); + jsonRow.put("id", 9223372036854775806L); + jsonRow.put("name", "TDSQL value "); + jsonRow.put("amount", Base64.getEncoder().encodeToString( + new BigDecimal("24710.35").unscaledValue().toByteArray())); + jsonRow.putNull("note"); + jsonRow.put("empty_value", ""); + SinkProto.RowValue.Builder jsonBuilder = SinkProto.RowValue.newBuilder(); + new DebeziumRowValueConverter(typeDescription).parse(jsonRow, jsonBuilder); + + org.apache.avro.Schema avroSchema = new org.apache.avro.Schema.Parser().parse(""" + { + "type": "record", + "name": "CanonicalRow", + "fields": [ + {"name": "id", "type": "long"}, + {"name": "name", "type": "string"}, + {"name": "amount", "type": "bytes"}, + {"name": "note", "type": ["null", "string"], "default": null}, + {"name": "empty_value", "type": "string"} + ] + } + """); + GenericData.Record avroRow = new GenericData.Record(avroSchema); + avroRow.put("id", 9223372036854775806L); + avroRow.put("name", "TDSQL value "); + avroRow.put("amount", new BigDecimal("24710.35")); + avroRow.put("note", null); + avroRow.put("empty_value", ""); + SinkProto.RowValue.Builder avroBuilder = SinkProto.RowValue.newBuilder(); + new DebeziumRowValueConverter(typeDescription).parse(avroRow, avroBuilder); + + assertEquals(value, jsonBuilder.build()); + assertEquals(value, avroBuilder.build()); + } + + @Test + void shouldPreserveExistingIntegerWidths() throws Exception + { + TypeDescription typeDescription = TypeDescription.createSchemaFromStrings( + List.of("tiny", "small", "integer"), + List.of("tinyint", "smallint", "integer")); + Schema rowSchema = SchemaBuilder.struct() + .field("tiny", Schema.INT8_SCHEMA) + .field("small", Schema.INT16_SCHEMA) + .field("integer", Schema.INT32_SCHEMA) + .build(); + Struct row = new Struct(rowSchema) + .put("tiny", (byte) 7) + .put("small", (short) 1024) + .put("integer", 65536); + + SinkProto.RowValue.Builder builder = SinkProto.RowValue.newBuilder(); + new DebeziumRowValueConverter(typeDescription).parse(row, builder); + + assertEquals(1, builder.getValues(0).getValue().size()); + assertEquals(2, builder.getValues(1).getValue().size()); + assertEquals(4, builder.getValues(2).getValue().size()); + } + + @Test + void testParseDate() + { + int day = 17059; + Date debeziumDate = TestDateUtil.fromDebeziumDate(day); + String dayString = TestDateUtil.convertDateToDayString(debeziumDate); + long ts = 1473927308302000L; + LocalDateTime localDateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts / 1000), ZoneOffset.UTC); + ZonedDateTime zonedDateTime = Instant.ofEpochMilli(ts).atZone(ZoneOffset.UTC); + boolean pause = true; + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventJsonDeserializerTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventJsonDeserializerTest.java new file mode 100644 index 0000000..d970187 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventJsonDeserializerTest.java @@ -0,0 +1,182 @@ +/* + * Copyright 2025 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.pixelsdb.pixels.sink.conversion.debezium; + +import io.pixelsdb.pixels.common.metadata.SchemaTableName; +import io.pixelsdb.pixels.common.metadata.domain.Column; +import io.pixelsdb.pixels.common.metadata.domain.Table; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import org.apache.kafka.common.serialization.Deserializer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class RowChangeEventJsonDeserializerTest +{ + private static final SchemaTableName REGION_TABLE = + new SchemaTableName("pixels_realtime_crud", "region"); + private static TableMetadata previousRegionMetadata; + private static boolean hadRegionMetadata; + + private final Deserializer deserializer = new RowChangeEventJsonDeserializer(); + private final Deserializer transactionDeserializer = + new TransactionMetadataJsonDeserializer(); + + @BeforeAll + static void setUpConfig() throws Exception + { + PixelsSinkConfigFactory.reset(); + PixelsSinkConfigFactory.initialize(Objects.requireNonNull( + RowChangeEventJsonDeserializerTest.class.getClassLoader() + .getResource("pixels-sink-test.properties")).getPath()); + Map registry = metadataRegistry(); + hadRegionMetadata = registry.containsKey(REGION_TABLE); + previousRegionMetadata = registry.get(REGION_TABLE); + registry.put(REGION_TABLE, regionMetadata()); + } + + @AfterAll + static void resetConfig() throws ReflectiveOperationException + { + Map registry = metadataRegistry(); + if (hadRegionMetadata) + { + registry.put(REGION_TABLE, previousRegionMetadata); + } + else + { + registry.remove(REGION_TABLE); + } + PixelsSinkConfigFactory.reset(); + } + + private String loadSchemaFromFile(String filename) throws IOException, URISyntaxException + { + ClassLoader classLoader = getClass().getClassLoader(); + return new String(Files.readAllBytes(Paths.get( + Objects.requireNonNull(classLoader.getResource(filename)).toURI() + ))); + } + + // @ParameterizedTest +// @EnumSource(value = OperationType.class, names = {"INSERT", "UPDATE"}) +// //, , "SNAPSHOT" +// void shouldParseValidOperations(OperationType opType) throws Exception { +// String jsonData = loadSchemaFromFile("records/" + opType.name().toLowerCase() + ".json"); +// RowChangeEvent event = deserializer.deserialize("test_topic", jsonData.getBytes()); +// +// assertNotNull(event); +// assertEquals(opType, event.getOp()); +// assertEquals("region", event.getTable()); +// +// Map data = opType == OperationType.DELETE ? +// event.getBeforeData() : event.getAfterData(); +// assertNotNull(data); +// } + + @Test + void shouldHandleDeleteOperation() throws Exception + { + String jsonData = loadSchemaFromFile("records/delete.json"); + RowChangeEvent event = deserializer.deserialize("test_topic", jsonData.getBytes()); + + assertTrue(event.isDelete()); +// assertNotNull(event.getBeforeData()); +// assertNull(event.getAfterData()); + } + + + @Test + void shouldHandleEmptyData() + { + RowChangeEvent event = deserializer.deserialize("empty_topic", new byte[0]); + assertNull(event); + } + + @Test + void shouldDeserializeTransactionMetadataWithConfiguredConnector() + { + String json = """ + { + "payload": { + "status": "END", + "id": "mysql-tx-1", + "event_count": 1, + "ts_ms": 1750000000000, + "data_collections": [ + {"data_collection": "pixels_realtime_crud.region", "event_count": 1} + ] + } + } + """; + + SinkProto.TransactionMetadata transaction = transactionDeserializer.deserialize( + "transaction", json.getBytes()); + + assertEquals(SinkProto.TransactionStatus.END, transaction.getStatus()); + assertEquals("mysql-tx-1", transaction.getId()); + assertEquals("pixels_realtime_crud.region", + transaction.getDataCollections(0).getDataCollection()); + } + + private static TableMetadata regionMetadata() throws Exception + { + Table table = new Table(); + table.setId(2); + table.setName("region"); + return new TableMetadata(table, null, List.of( + column("R_REGIONKEY", "int"), + column("R_NAME", "string"), + column("R_COMMENT", "string"))); + } + + private static Column column(String name, String type) + { + Column column = new Column(); + column.setName(name); + column.setType(type); + return column; + } + + @SuppressWarnings("unchecked") + private static Map metadataRegistry() + throws ReflectiveOperationException + { + Field registry = TableMetadataRegistry.class.getDeclaredField("registry"); + registry.setAccessible(true); + return (Map) registry.get(TableMetadataRegistry.Instance()); + } +} + diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/kafka/KafkaRecordConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/kafka/KafkaRecordConverterTest.java new file mode 100644 index 0000000..55559cf --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/kafka/KafkaRecordConverterTest.java @@ -0,0 +1,56 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.conversion.kafka; + +import org.apache.kafka.common.serialization.Deserializer; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class KafkaRecordConverterTest +{ + @Test + void shouldCreateConfiguredDeserializerAndConvertRawBytes() + { + String converterKey = "test.converter"; + Properties properties = new Properties(); + properties.put(converterKey, Utf8Deserializer.class.getName()); + + try (KafkaRecordConverter converter = + KafkaRecordConverter.create(properties, converterKey)) + { + assertEquals("record", converter.convert( + "topic", "record".getBytes(StandardCharsets.UTF_8))); + } + } + + public static class Utf8Deserializer implements Deserializer + { + @Override + public String deserialize(String topic, byte[] data) + { + return new String(data, StandardCharsets.UTF_8); + } + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverterTest.java new file mode 100644 index 0000000..d58f2b0 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverterTest.java @@ -0,0 +1,102 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.conversion.sinkproto; + +import io.pixelsdb.pixels.common.metadata.SchemaTableName; +import io.pixelsdb.pixels.common.metadata.domain.Column; +import io.pixelsdb.pixels.common.metadata.domain.Table; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class RowRecordConverterTest +{ + private static final SchemaTableName TABLE = + new SchemaTableName("storage_test", "records"); + private Map registry; + private TableMetadata previousMetadata; + + @BeforeEach + @SuppressWarnings("unchecked") + void setUp() throws Exception + { + Field field = TableMetadataRegistry.class.getDeclaredField("registry"); + field.setAccessible(true); + registry = (Map) + field.get(TableMetadataRegistry.Instance()); + previousMetadata = registry.put(TABLE, metadata()); + } + + @AfterEach + void tearDown() + { + if (previousMetadata == null) + { + registry.remove(TABLE); + } else + { + registry.put(TABLE, previousMetadata); + } + } + + @Test + void shouldConvertCanonicalRowRecordIntoRuntimeEvent() + throws Exception + { + SinkProto.RowRecord record = SinkProto.RowRecord.newBuilder() + .setOp(SinkProto.OperationType.INSERT) + .setSource(SinkProto.SourceInfo.newBuilder() + .setDb(TABLE.getSchemaName()) + .setTable(TABLE.getTableName())) + .setAfter(SinkProto.RowValue.newBuilder() + .addValues(SinkProto.ColumnValue.newBuilder().setIsNull(true)) + .build()) + .build(); + + RowChangeEvent event = + new RowRecordConverter(TableMetadataRegistry.Instance()).convert(record); + + assertEquals(record, event.getRowRecord()); + assertEquals(TABLE.getSchemaName(), event.getDb()); + assertEquals(TABLE.getTableName(), event.getTable()); + } + + private TableMetadata metadata() throws Exception + { + Table table = new Table(); + table.setId(1); + table.setName(TABLE.getTableName()); + Column column = new Column(); + column.setName("id"); + column.setType("int"); + return new TableMetadata(table, null, List.of(column)); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/RowBatchTest.java b/src/test/java/io/pixelsdb/pixels/sink/event/RowBatchTest.java similarity index 97% rename from src/test/java/io/pixelsdb/pixels/sink/event/deserializer/RowBatchTest.java rename to src/test/java/io/pixelsdb/pixels/sink/event/RowBatchTest.java index 0c36b72..a467267 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/RowBatchTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/event/RowBatchTest.java @@ -18,7 +18,7 @@ * . */ -package io.pixelsdb.pixels.sink.event.deserializer; +package io.pixelsdb.pixels.sink.event; import io.pixelsdb.pixels.core.TypeDescription; import io.pixelsdb.pixels.core.vector.BinaryColumnVector; diff --git a/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactoryTest.java b/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactoryTest.java new file mode 100644 index 0000000..8cfbfac --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactoryTest.java @@ -0,0 +1,116 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.event; + +import com.google.protobuf.ByteString; +import io.pixelsdb.pixels.common.metadata.SchemaTableName; +import io.pixelsdb.pixels.common.metadata.domain.Column; +import io.pixelsdb.pixels.common.metadata.domain.Table; +import io.pixelsdb.pixels.core.TypeDescription; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class RowChangeEventFactoryTest +{ + private static final SchemaTableName TABLE = + new SchemaTableName("factory_test", "records"); + private Map registry; + private TableMetadata previousMetadata; + + @BeforeEach + @SuppressWarnings("unchecked") + void setUp() throws Exception + { + Field field = TableMetadataRegistry.class.getDeclaredField("registry"); + field.setAccessible(true); + registry = (Map) + field.get(TableMetadataRegistry.Instance()); + previousMetadata = registry.put(TABLE, metadata()); + } + + @AfterEach + void tearDown() + { + if (previousMetadata == null) + { + registry.remove(TABLE); + } else + { + registry.put(TABLE, previousMetadata); + } + } + + @Test + void shouldBuildCanonicalRowChangeEvent() + throws Exception + { + TypeDescription schema = TypeDescription.createSchemaFromStrings( + List.of("id"), List.of("int")); + SinkProto.SourceInfo source = SinkProto.SourceInfo.newBuilder() + .setDb(TABLE.getSchemaName()) + .setTable(TABLE.getTableName()) + .build(); + SinkProto.RowValue after = SinkProto.RowValue.newBuilder() + .addValues(SinkProto.ColumnValue.newBuilder() + .setValue(ByteString.copyFromUtf8("1"))) + .build(); + + RowChangeEvent event = RowChangeEventFactory.create( + SinkProto.OperationType.INSERT, + source, + SinkProto.TransactionInfo.newBuilder().setId("tx-1").build(), + null, + after, + schema); + + assertEquals(SinkProto.OperationType.INSERT, event.getOp()); + assertEquals(TABLE.getSchemaName(), event.getDb()); + assertEquals(TABLE.getTableName(), event.getTable()); + assertEquals("tx-1", event.getTransaction().getId()); + assertEquals(after, event.getAfter()); + } + + private TableMetadata metadata() throws Exception + { + Table table = new Table(); + table.setId(1); + table.setName(TABLE.getTableName()); + return new TableMetadata(table, null, List.of(column("id", "int"))); + } + + private Column column(String name, String type) + { + Column column = new Column(); + column.setName(name); + column.setType(type); + return column; + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventDeserializerTest.java b/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventDeserializerTest.java deleted file mode 100644 index 783bbcb..0000000 --- a/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/RowChangeEventDeserializerTest.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import org.apache.kafka.common.serialization.Deserializer; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Objects; - -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class RowChangeEventDeserializerTest -{ - - private final Deserializer deserializer = new RowChangeEventJsonDeserializer(); - - private String loadSchemaFromFile(String filename) throws IOException, URISyntaxException - { - ClassLoader classLoader = getClass().getClassLoader(); - return new String(Files.readAllBytes(Paths.get( - Objects.requireNonNull(classLoader.getResource(filename)).toURI() - ))); - } - - // @ParameterizedTest -// @EnumSource(value = OperationType.class, names = {"INSERT", "UPDATE"}) -// //, , "SNAPSHOT" -// void shouldParseValidOperations(OperationType opType) throws Exception { -// String jsonData = loadSchemaFromFile("records/" + opType.name().toLowerCase() + ".json"); -// RowChangeEvent event = deserializer.deserialize("test_topic", jsonData.getBytes()); -// -// assertNotNull(event); -// assertEquals(opType, event.getOp()); -// assertEquals("region", event.getTable()); -// -// Map data = opType == OperationType.DELETE ? -// event.getBeforeData() : event.getAfterData(); -// assertNotNull(data); -// } - - @Test - void shouldHandleDeleteOperation() throws Exception - { - String jsonData = loadSchemaFromFile("records/delete.json"); - RowChangeEvent event = deserializer.deserialize("test_topic", jsonData.getBytes()); - - assertTrue(event.isDelete()); -// assertNotNull(event.getBeforeData()); -// assertNull(event.getAfterData()); - } - - - @Test - void shouldHandleEmptyData() - { - RowChangeEvent event = deserializer.deserialize("empty_topic", new byte[0]); - assertNull(event); - } -} - diff --git a/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/RowDataParserTest.java b/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/RowDataParserTest.java deleted file mode 100644 index 0eee54b..0000000 --- a/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/RowDataParserTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - -import io.pixelsdb.pixels.sink.util.DateUtil; -import org.junit.jupiter.api.Test; - -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.util.Date; - -public class RowDataParserTest -{ - // BqQ= 17 - // JbR7 24710.35 -// @ParameterizedTest -// @CsvSource({ -// // encodedValue, expectedValue, precision, scale -// "BqQ=, 17.00, 15, 2", -// "JbR7, 24710.35, 15, 2", -// }) -// void testParseDecimalValid(String encodedValue, String expectedValue, int precision, int scale) { -// JsonNode node = new TextNode(encodedValue); -// TypeDescription type = TypeDescription.createDecimal(precision, scale); -// RowDataParser rowDataParser = new RowDataParser(type); -// BigDecimal result = rowDataParser.parseDecimal(node, type); -// assertEquals(new BigDecimal(expectedValue), result); -// } - - @Test - void testParseDate() - { - int day = 17059; - Date debeziumDate = DateUtil.fromDebeziumDate(day); - String dayString = DateUtil.convertDateToDayString(debeziumDate); - long ts = 1473927308302000L; - LocalDateTime localDateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts / 1000), ZoneOffset.UTC); - ZonedDateTime zonedDateTime = Instant.ofEpochMilli(ts).atZone(ZoneOffset.UTC); - boolean pause = true; - } -} diff --git a/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/SchemaDeserializerTest.java b/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/SchemaDeserializerTest.java deleted file mode 100644 index bdd14e4..0000000 --- a/src/test/java/io/pixelsdb/pixels/sink/event/deserializer/SchemaDeserializerTest.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package io.pixelsdb.pixels.sink.event.deserializer; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.pixelsdb.pixels.core.TypeDescription; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Objects; - -public class SchemaDeserializerTest -{ - private static final ObjectMapper objectMapper = new ObjectMapper(); - - @BeforeEach - public void setUp() throws IOException, URISyntaxException - { - // String schemaStr = schemaNode.toString(); - - } - - private String loadSchemaFromFile(String filename) throws IOException, URISyntaxException - { - ClassLoader classLoader = getClass().getClassLoader(); - return new String(Files.readAllBytes(Paths.get(Objects.requireNonNull(classLoader.getResource(filename)).toURI()))); - } - - @Test - public void testParseNationStruct() throws IOException, URISyntaxException - { - String jsonSchema = loadSchemaFromFile("records/nation.json"); - JsonNode rootNode = objectMapper.readTree(jsonSchema); - JsonNode schemaNode = rootNode.get("schema"); - - TypeDescription typeDesc = SchemaDeserializer.parseFromBeforeOrAfter(schemaNode, "after"); - - Assertions.assertEquals(TypeDescription.Category.STRUCT, typeDesc.getCategory()); - Assertions.assertEquals(4, typeDesc.getChildren().size()); - Assertions.assertEquals("n_nationkey", typeDesc.getFieldNames().get(0)); - Assertions.assertEquals(TypeDescription.Category.INT, typeDesc.getChildren().get(0).getCategory()); - Assertions.assertEquals("n_name", typeDesc.getFieldNames().get(1)); - Assertions.assertEquals(TypeDescription.Category.STRING, typeDesc.getChildren().get(1).getCategory()); - Assertions.assertEquals("n_regionkey", typeDesc.getFieldNames().get(2)); - Assertions.assertEquals(TypeDescription.Category.INT, typeDesc.getChildren().get(2).getCategory()); - Assertions.assertEquals("n_comment", typeDesc.getFieldNames().get(3)); - Assertions.assertEquals(TypeDescription.Category.STRING, typeDesc.getChildren().get(3).getCategory()); - } - - @Test - public void testParseDecimalType() throws IOException - { - String jsonSchema = "[{" - + "\"type\": \"bytes\"," - + "\"name\": \"org.apache.kafka.connect.data.Decimal\"," - + "\"parameters\": {" - + " \"scale\": \"2\"," - + " \"connect.decimal.precision\": \"15\"" - + "}," - + "\"field\": \"price\"" - + "}]"; - - JsonNode rootNode = objectMapper.readTree(jsonSchema); - - TypeDescription typeDesc = SchemaDeserializer.parseStruct(rootNode); - TypeDescription filedType = typeDesc.getChildren().get(0); - Assertions.assertEquals(TypeDescription.Category.DECIMAL, filedType.getCategory()); - Assertions.assertEquals(15, filedType.getPrecision()); - Assertions.assertEquals(2, filedType.getScale()); - } - - @Test - public void testParseDateType() throws IOException - { - - String jsonSchema = "[{" - + "\"type\": \"int32\"," - + "\"name\": \"io.debezium.time.Date\"," - + "\"field\": \"created_at\"" - + "}]"; - JsonNode rootNode = objectMapper.readTree(jsonSchema); - TypeDescription typeDesc = SchemaDeserializer.parseStruct(rootNode); - TypeDescription filedType = typeDesc.getChildren().get(0); - Assertions.assertEquals(TypeDescription.Category.DATE, filedType.getCategory()); - } - - // 测试未知类型异常 - @Test - public void testParseInvalidType() throws IOException - { - String jsonSchema = "[{" - + "\"type\": \"unknown_type\"," - + "\"field\": \"invalid_field\"" - + "}]"; - JsonNode rootNode = objectMapper.readTree(jsonSchema); - Assertions.assertThrows( - IllegalArgumentException.class, () -> - { - SchemaDeserializer.parseStruct(rootNode); - } - ); - } - - @Test - public void testMissingRequiredField() throws IOException - { - String jsonSchema = "{" - + "\"type\": \"struct\"," - + "\"fields\": [" - + " {\"field\": \"id\"}" // lack type - + "]" - + "}"; - JsonNode rootNode = objectMapper.readTree(jsonSchema); - Assertions.assertThrows( - IllegalArgumentException.class, () -> - { - SchemaDeserializer.parseFieldType(rootNode); - } - ); - } -} \ No newline at end of file diff --git a/src/test/java/io/pixelsdb/pixels/sink/provider/BlockingEventProviderTest.java b/src/test/java/io/pixelsdb/pixels/sink/provider/BlockingEventProviderTest.java new file mode 100644 index 0000000..c00b721 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/provider/BlockingEventProviderTest.java @@ -0,0 +1,61 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.provider; + +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +class BlockingEventProviderTest +{ + @Test + void shouldPublishAndTakeCanonicalEvent() + { + BlockingEventProvider provider = new BlockingEventProvider<>(); + provider.publish(7); + + assertEquals(7, provider.take()); + provider.close(); + } + + @Test + void shouldWakeConsumerWhenClosed() + throws Exception + { + BlockingEventProvider provider = new BlockingEventProvider<>(); + CountDownLatch finished = new CountDownLatch(1); + Thread consumer = new Thread(() -> + { + assertNull(provider.take()); + finished.countDown(); + }); + consumer.start(); + + provider.close(); + + assertEquals(true, finished.await(1, TimeUnit.SECONDS)); + consumer.join(); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumerTest.java b/src/test/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumerTest.java new file mode 100644 index 0000000..c422ec4 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumerTest.java @@ -0,0 +1,204 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + */ +package io.pixelsdb.pixels.sink.source.engine; + +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.source.SourceRecord; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static io.pixelsdb.pixels.sink.source.engine.PixelsDebeziumConsumer.RecordType.ROW; +import static io.pixelsdb.pixels.sink.source.engine.PixelsDebeziumConsumer.RecordType.TOMBSTONE; +import static io.pixelsdb.pixels.sink.source.engine.PixelsDebeziumConsumer.RecordType.TRANSACTION; +import static io.pixelsdb.pixels.sink.source.engine.PixelsDebeziumConsumer.RecordType.UNKNOWN_CONTROL; +import static org.junit.jupiter.api.Assertions.assertEquals; + +class PixelsDebeziumConsumerTest +{ + private static final String PREFIX = "mysql-cdc"; + private static final String ROW_TOPIC = PREFIX + ".cdc_verify.binlog_test"; + private static final String TRANSACTION_TOPIC = PREFIX + ".transaction"; + private static final Schema ROW_SCHEMA = SchemaBuilder.struct().optional() + .field("id", Schema.INT64_SCHEMA) + .field("name", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + private static final Schema SOURCE_SCHEMA = SchemaBuilder.struct() + .field("connector", Schema.STRING_SCHEMA) + .field("db", Schema.STRING_SCHEMA) + .field("table", Schema.OPTIONAL_STRING_SCHEMA) + .field("gtid", Schema.OPTIONAL_STRING_SCHEMA) + .field("file", Schema.STRING_SCHEMA) + .field("pos", Schema.INT64_SCHEMA) + .field("row", Schema.INT32_SCHEMA) + .build(); + private static final Schema TRANSACTION_SCHEMA = SchemaBuilder.struct().optional() + .field("id", Schema.STRING_SCHEMA) + .field("total_order", Schema.INT64_SCHEMA) + .field("data_collection_order", Schema.INT64_SCHEMA) + .build(); + + @Test + void shouldClassifyMySqlRowOperations() + { + assertEquals(ROW, classify(rowRecord("c", null, row(1), "gtid:3", 1))); + assertEquals(ROW, classify(rowRecord("u", row(1), row(1), "gtid:3", 2))); + assertEquals(ROW, classify(rowRecord("d", row(1), null, "gtid:3", 3))); + assertEquals(ROW, classify(rowRecord("r", null, row(1), "", 0))); + } + + @Test + void shouldTreatTwoRowsFromOneWriteRowsAsIndependentEvents() + { + SourceRecord first = rowRecord("c", null, row(1), "gtid:4", 1); + SourceRecord second = rowRecord("c", null, row(2), "gtid:4", 2); + + assertEquals(ROW, classify(first)); + assertEquals(ROW, classify(second)); + assertEquals("gtid:4", + ((Struct) ((Struct) first.value()).get("transaction")).getString("id")); + assertEquals("gtid:4", + ((Struct) ((Struct) second.value()).get("transaction")).getString("id")); + } + + @Test + void shouldKeepReplaceDeleteAndInsertInOneTransaction() + { + SourceRecord delete = rowRecord("d", row(2), null, "gtid:7", 1); + SourceRecord insert = rowRecord("c", null, row(2), "gtid:7", 2); + + assertEquals(ROW, classify(delete)); + assertEquals(ROW, classify(insert)); + } + + @Test + void shouldClassifyTransactionBoundaries() + { + assertEquals(TRANSACTION, classify(transactionRecord("BEGIN", "gtid:5"))); + assertEquals(TRANSACTION, classify(transactionRecord("END", "gtid:5"))); + assertEquals(UNKNOWN_CONTROL, classify(transactionRecord("COMMIT", "gtid:5"))); + } + + @Test + void shouldSkipControlEvents() + { + assertEquals(TOMBSTONE, classify(sourceRecord(ROW_TOPIC, null, null))); + assertEquals(UNKNOWN_CONTROL, classify(sourceRecord( + PREFIX + ".signal", SchemaBuilder.struct() + .field("type", Schema.STRING_SCHEMA).build(), + new Struct(SchemaBuilder.struct() + .field("type", Schema.STRING_SCHEMA).build()).put("type", "x")))); + } + + @Test + void shouldRejectMalformedRows() + { + assertEquals(UNKNOWN_CONTROL, + classify(rowRecord("u", null, row(1), "gtid:6", 1))); + assertEquals(UNKNOWN_CONTROL, + classify(rowRecord("d", null, null, "gtid:6", 1))); + } + + private PixelsDebeziumConsumer.RecordType classify(SourceRecord record) + { + return PixelsDebeziumConsumer.classify(record, TRANSACTION_TOPIC); + } + + private SourceRecord rowRecord( + String op, Struct before, Struct after, String transactionId, long order) + { + Schema transactionSchema = transactionSchema(); + Struct transaction = transactionId.isEmpty() ? null : new Struct(transactionSchema) + .put("id", transactionId) + .put("total_order", order) + .put("data_collection_order", order); + Schema sourceSchema = sourceSchema(); + Struct source = new Struct(sourceSchema) + .put("connector", "mysql") + .put("db", "cdc_verify") + .put("table", "binlog_test") + .put("gtid", transactionId.isEmpty() ? null : transactionId) + .put("file", "binlog.000004") + .put("pos", 152L) + .put("row", (int) order); + Schema envelopeSchema = SchemaBuilder.struct() + .name("mysql-cdc.cdc_verify.binlog_test.Envelope") + .field("before", rowSchema()) + .field("after", rowSchema()) + .field("source", sourceSchema) + .field("transaction", transactionSchema) + .field("op", Schema.STRING_SCHEMA) + .build(); + Struct envelope = new Struct(envelopeSchema) + .put("before", before) + .put("after", after) + .put("source", source) + .put("transaction", transaction) + .put("op", op); + return sourceRecord(ROW_TOPIC, envelopeSchema, envelope); + } + + private SourceRecord transactionRecord(String status, String id) + { + Schema collectionSchema = SchemaBuilder.struct() + .field("data_collection", Schema.STRING_SCHEMA) + .field("event_count", Schema.INT64_SCHEMA) + .build(); + Schema schema = SchemaBuilder.struct() + .field("status", Schema.STRING_SCHEMA) + .field("id", Schema.STRING_SCHEMA) + .field("event_count", Schema.OPTIONAL_INT64_SCHEMA) + .field("data_collections", SchemaBuilder.array(collectionSchema).optional().build()) + .field("ts_ms", Schema.INT64_SCHEMA) + .build(); + Struct value = new Struct(schema) + .put("status", status) + .put("id", id) + .put("event_count", status.equals("END") ? 4L : null) + .put("data_collections", null) + .put("ts_ms", 1750000000000L); + return sourceRecord(TRANSACTION_TOPIC, schema, value); + } + + private SourceRecord sourceRecord(String topic, Schema schema, Object value) + { + return new SourceRecord( + Map.of("server", PREFIX), + Map.of("file", "binlog.000004", "pos", 152L), + topic, + null, + null, + schema, + value); + } + + private Struct row(long id) + { + return new Struct(rowSchema()).put("id", id).put("name", "row-" + id); + } + + private Schema rowSchema() + { + return ROW_SCHEMA; + } + + private Schema sourceSchema() + { + return SOURCE_SCHEMA; + } + + private Schema transactionSchema() + { + return TRANSACTION_SCHEMA; + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistryTest.java b/src/test/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistryTest.java index aee4a0e..e38c507 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistryTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistryTest.java @@ -19,6 +19,7 @@ package io.pixelsdb.pixels.sink.util; +import io.pixelsdb.pixels.common.utils.EtcdUtil; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,6 +50,6 @@ public void testCreateFile() { LOGGER.info(file); } - etcdFileRegistry.cleanData(); + EtcdUtil.Instance().deleteByPrefix("/sink/proto/registry/test"); } } diff --git a/src/test/java/io/pixelsdb/pixels/sink/util/TestDateUtil.java b/src/test/java/io/pixelsdb/pixels/sink/util/TestDateUtil.java new file mode 100644 index 0000000..aea207d --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/util/TestDateUtil.java @@ -0,0 +1,57 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.util; + +import io.pixelsdb.pixels.core.utils.DatetimeUtils; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Calendar; +import java.util.Date; + +public final class TestDateUtil +{ + private TestDateUtil() + { + } + + public static Date fromDebeziumDate(int epochDay) + { + Calendar calendar = Calendar.getInstance(); + calendar.clear(); + calendar.set(1970, Calendar.JANUARY, 1); + calendar.add(Calendar.DAY_OF_MONTH, epochDay); + return calendar.getTime(); + } + + public static String convertDateToDayString(Date date) + { + return new java.text.SimpleDateFormat("yyyy-MM-dd").format(date); + } + + public static String convertDebeziumTimestampToString(long epochTs) + { + LocalDateTime dateTime = LocalDateTime.ofInstant( + Instant.ofEpochMilli(epochTs), ZoneId.systemDefault()); + return dateTime.format(DatetimeUtils.SQL_LOCAL_DATE_TIME); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/TestRetinaWriter.java b/src/test/java/io/pixelsdb/pixels/sink/writer/TestRetinaWriter.java index 15e43ea..d21f0a3 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/TestRetinaWriter.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/TestRetinaWriter.java @@ -33,7 +33,7 @@ import io.pixelsdb.pixels.sink.event.RowChangeEvent; import io.pixelsdb.pixels.sink.exception.SinkException; import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; -import io.pixelsdb.pixels.sink.util.DateUtil; +import io.pixelsdb.pixels.sink.util.TestDateUtil; import io.pixelsdb.pixels.sink.writer.retina.RetinaServiceProxy; import io.pixelsdb.pixels.sink.writer.retina.TransactionProxy; import org.junit.jupiter.api.Assertions; @@ -228,7 +228,7 @@ public void testCheckingAccountInsertPerformance() throws for (int sb = 0; sb < samllBatchCount; sb++) { ++b; - TransContext ctx = manager.getNewTransContext(); + TransContext ctx = manager.getNewTransContext("batch-" + b); long timeStamp = ctx.getTimestamp(); RetinaProto.TableUpdateData.Builder tableUpdateDataBuilder = @@ -361,7 +361,7 @@ public void testCheckingAccountUpdatePerformance() throws // 轮询选择客户端 RetinaServiceProxy writer = writers.get(batchIndex % clientCount); - TransContext ctx = manager.getNewTransContext(); + TransContext ctx = manager.getNewTransContext("batch-" + batchIndex); long timeStamp = ctx.getTimestamp(); List tableUpdateData = new ArrayList<>(); @@ -386,14 +386,14 @@ public void testCheckingAccountUpdatePerformance() throws .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Integer.toString(userID))).build()) .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Float.toString(oldBalance))).build()) .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Integer.toString(isBlocked))).build()) - .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(DateUtil.convertDebeziumTimestampToString(oldTs))).build()); + .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(TestDateUtil.convertDebeziumTimestampToString(oldTs))).build()); SinkProto.RowValue.Builder afterValueBuilder = SinkProto.RowValue.newBuilder() .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Integer.toString(accountID))).build()) .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Integer.toString(userID))).build()) .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Float.toString(newBalance))).build()) .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Integer.toString(isBlocked))).build()) - .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(DateUtil.convertDebeziumTimestampToString(newTs))).build()); + .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(TestDateUtil.convertDebeziumTimestampToString(newTs))).build()); SinkProto.RowRecord.Builder rowBuilder = SinkProto.RowRecord.newBuilder() .setOp(SinkProto.OperationType.UPDATE) @@ -415,7 +415,7 @@ public void testCheckingAccountUpdatePerformance() throws .addColValues(ByteString.copyFromUtf8(Integer.toString(userID))) .addColValues(ByteString.copyFromUtf8(Float.toString(newBalance))) .addColValues(ByteString.copyFromUtf8(Integer.toString(isBlocked))) - .addColValues(ByteString.copyFromUtf8(DateUtil.convertDebeziumTimestampToString(newTs))) + .addColValues(ByteString.copyFromUtf8(TestDateUtil.convertDebeziumTimestampToString(newTs))) .addIndexKeys(rowChangeEvent.getAfterKey()); tableUpdateDataBuilder.addInsertData(insertDataBuilder.build()); } @@ -481,7 +481,7 @@ public void testInsertTwoTablePerformance() throws for (int b = 0; b < batchCount; b++) { - TransContext ctx = manager.getNewTransContext(); + TransContext ctx = manager.getNewTransContext("batch-" + b); long timeStamp = ctx.getTimestamp(); List tableUpdateData = new ArrayList<>(); @@ -506,7 +506,7 @@ public void testInsertTwoTablePerformance() throws cols[1] = Integer.toString(userID).getBytes(StandardCharsets.UTF_8); cols[2] = Float.toString(balance).getBytes(StandardCharsets.UTF_8); cols[3] = Integer.toString(isBlocked).getBytes(StandardCharsets.UTF_8); - cols[4] = DateUtil.convertDebeziumTimestampToString(ts).getBytes(StandardCharsets.UTF_8); + cols[4] = TestDateUtil.convertDebeziumTimestampToString(ts).getBytes(StandardCharsets.UTF_8); // cols[4] = Long.toString(ts).getBytes(StandardCharsets.UTF_8); // after row SinkProto.RowValue.Builder afterValueBuilder = SinkProto.RowValue.newBuilder() diff --git a/src/test/resources/pixels-sink-test.properties b/src/test/resources/pixels-sink-test.properties new file mode 100644 index 0000000..b610781 --- /dev/null +++ b/src/test/resources/pixels-sink-test.properties @@ -0,0 +1,3 @@ +sink.monitor.enable=false +sink.monitor.report.enable=false +debezium.connector.class=io.debezium.connector.mysql.MySqlConnector From aebc87c640800090b2d07698a87528e46328da43 Mon Sep 17 00:00:00 2001 From: dannygeng Date: Wed, 29 Jul 2026 15:03:18 +0800 Subject: [PATCH 02/13] refactor: Replace event providers with pipeline-owned BlockingBoundedQueue --- docs/overview.md | 27 ++-- docs/usage.md | 3 +- pom.xml | 45 ++++++ .../source/DebeziumSourceAdapterRegistry.java | 6 +- .../pixels/sink/pipeline/TablePipeline.java | 13 +- .../sink/pipeline/TransactionPipeline.java | 13 +- .../pixels/sink/processor/TableProcessor.java | 16 ++- .../sink/processor/TransactionProcessor.java | 15 +- .../sink/provider/RowEventProvider.java | 36 ----- .../provider/TransactionEventProvider.java | 37 ----- .../BlockingBoundedQueue.java} | 31 +++-- .../pixels/sink/DebeziumEngineTest.java | 48 +++++-- .../io/pixelsdb/pixels/sink/TestConfig.java | 64 +++++++++ .../pixelsdb/pixels/sink/TestSplitString.java | 25 +++- .../sink/concurrent/RetinaWriterTest.java | 13 +- .../concurrent/TransactionServiceTest.java | 12 +- .../sink/consumer/AvroConsumerTest.java | 80 ++++++++--- .../DebeziumEnvelopeNormalizerTest.java | 24 +++- .../DebeziumRowValueConverterTest.java | 12 +- .../RowChangeEventJsonDeserializerTest.java | 31 +++-- .../pixels/sink/event/RowBatchTest.java | 44 +++--- .../pixels/sink/event/RowChangeEventTest.java | 30 ++-- .../sink/freshness/TestFreshnessClient.java | 55 +++++--- .../sink/metadata/TestIndexService.java | 13 +- .../pixels/sink/pipeline/PipelineTest.java | 62 +++++++++ .../sink/processor/TableProcessorTest.java | 108 +++++++++++++++ .../processor/TransactionProcessorTest.java | 95 +++++++++++++ .../provider/BlockingEventProviderTest.java | 61 --------- .../sink/util/BlockingBoundedQueueTest.java | 83 +++++++++++ .../sink/util/EtcdFileRegistryTest.java | 39 ++++-- .../pixels/sink/writer/RpcEndToEndTest.java | 31 +++-- .../pixels/sink/writer/TestProtoWriter.java | 129 ++++++++++-------- .../pixels/sink/writer/TestRetinaWriter.java | 23 ++-- .../pixelsdb/pixels/sink/writer/TpcHTest.java | 4 +- .../writer/retina/TableWriterProxyTest.java | 27 ++-- .../resources/pixels-sink-test.properties | 1 + 36 files changed, 919 insertions(+), 437 deletions(-) delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/provider/RowEventProvider.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventProvider.java rename src/main/java/io/pixelsdb/pixels/sink/{provider/BlockingEventProvider.java => util/BlockingBoundedQueue.java} (71%) create mode 100644 src/test/java/io/pixelsdb/pixels/sink/TestConfig.java create mode 100644 src/test/java/io/pixelsdb/pixels/sink/pipeline/PipelineTest.java create mode 100644 src/test/java/io/pixelsdb/pixels/sink/processor/TableProcessorTest.java create mode 100644 src/test/java/io/pixelsdb/pixels/sink/processor/TransactionProcessorTest.java delete mode 100644 src/test/java/io/pixelsdb/pixels/sink/provider/BlockingEventProviderTest.java create mode 100644 src/test/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueueTest.java diff --git a/docs/overview.md b/docs/overview.md index e60b0b7..e139405 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -27,28 +27,23 @@ The source stage owns input lifecycle and invokes the configured converter. **Source Outputs** - Kafka, Engine, and Storage sources publish only canonical `RowChangeEvent` or `SinkProto.TransactionMetadata` objects. -- Protocol conversion is implemented in `conversion`; providers never receive +- Protocol conversion is implemented in `conversion`; pipeline queues never receive Kafka bytes, Engine `SourceRecord`, or Storage `ByteBuffer`. -**Provider** -Providers are bounded canonical-event channels. They provide backpressure, -ordered delivery, and lifecycle management without knowing the source -protocol. +**Pipeline Queue** +Pipelines own bounded canonical-event queues. The queue provides backpressure, +ordered delivery, and lifecycle management without knowing the source protocol. ```mermaid classDiagram direction TB - class BlockingEventProvider~T~ { - +publish(T event) + class BlockingBoundedQueue~T~ { + +put(T value) +take() +close() } - class RowEventProvider { - } - class TransactionEventProvider { - } class TablePipeline { +publish(RowChangeEvent) } @@ -59,20 +54,18 @@ classDiagram +route(RowChangeEvent) } - BlockingEventProvider <|-- RowEventProvider - BlockingEventProvider <|-- TransactionEventProvider TablePipelineManager --> TablePipeline - TablePipeline --> RowEventProvider - TransactionPipeline --> TransactionEventProvider + TablePipeline --> BlockingBoundedQueue + TransactionPipeline --> BlockingBoundedQueue ``` **Processor** -Processors pull events from providers and write to the sink writers. +Processors pull events from pipeline queues and write to the sink writers. - `TableProcessor` instances are created by `TablePipelineManager`. - There is typically one `TableProcessor` per table to maintain per-table ordering. -- `TransactionPipeline` owns the transaction provider and `TransactionProcessor`. +- `TransactionPipeline` owns the transaction queue and `TransactionProcessor`. **Writer** Writers implement `PixelsSinkWriter`: diff --git a/docs/usage.md b/docs/usage.md index 5beaa54..d27ead6 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -25,7 +25,8 @@ mvn -q -DskipTests exec:java \ **Pipeline Summary** - A source reads change events from Debezium Engine, Kafka, or storage files. -- A provider converts source records into Pixels events. +- Conversion normalizes source records into canonical Pixels events. +- Pipelines route events through bounded queues to their processors. - Processors enforce ordering by table and write to the configured sink. - The sink writer persists the events to Retina, CSV, Proto, Flink, or a no-op sink. diff --git a/pom.xml b/pom.xml index e3038ae..4d8574f 100644 --- a/pom.xml +++ b/pom.xml @@ -38,10 +38,25 @@ 5.8 1.18.42 3.2.3.Final + 2.16.2 1.4.13 440 + + + + + com.fasterxml.jackson + jackson-bom + ${dep.jackson.version} + pom + import + + + + io.pixelsdb @@ -276,6 +291,16 @@ + + org.apache.maven.plugins + maven-surefire-plugin + + false + integration + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-exports=java.base/sun.nio.ch=ALL-UNNAMED + + org.apache.maven.plugins maven-shade-plugin @@ -346,4 +371,24 @@ + + + + integration-tests + + + + org.apache.maven.plugins + maven-surefire-plugin + + false + + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-exports=java.base/sun.nio.ch=ALL-UNNAMED + + + + + + \ No newline at end of file diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceAdapterRegistry.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceAdapterRegistry.java index 1f841f5..d6938ad 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceAdapterRegistry.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceAdapterRegistry.java @@ -27,11 +27,7 @@ private DebeziumSourceAdapterRegistry() public static DebeziumSourceAdapter forSource(String connector) { - if (connector != null && !connector.isBlank()) - { - return resolve(connector); - } - return configured(); + return resolve(connector); } public static DebeziumSourceAdapter resolve(String connector) diff --git a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipeline.java b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipeline.java index 6704391..2b33f72 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipeline.java +++ b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipeline.java @@ -20,23 +20,24 @@ package io.pixelsdb.pixels.sink.pipeline; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; import io.pixelsdb.pixels.sink.event.RowChangeEvent; import io.pixelsdb.pixels.sink.processor.TableProcessor; -import io.pixelsdb.pixels.sink.provider.RowEventProvider; +import io.pixelsdb.pixels.sink.util.BlockingBoundedQueue; import io.pixelsdb.pixels.sink.writer.PixelsSinkWriter; import io.pixelsdb.pixels.sink.writer.PixelsSinkWriterFactory; public final class TablePipeline implements AutoCloseable { - private final RowEventProvider provider; + private final BlockingBoundedQueue eventQueue; private final PixelsSinkWriter writer; private final TableProcessor processor; public TablePipeline() { - this.provider = new RowEventProvider(); + this.eventQueue = new BlockingBoundedQueue<>(PixelsSinkConstants.MAX_QUEUE_SIZE); this.writer = PixelsSinkWriterFactory.getWriter(); - this.processor = new TableProcessor(provider, writer); + this.processor = new TableProcessor(eventQueue, writer); } public void start() @@ -46,13 +47,13 @@ public void start() public void publish(RowChangeEvent event) { - provider.publishRowChangeEvent(event); + eventQueue.put(event); } @Override public void close() { processor.stopProcessor(); - provider.close(); + eventQueue.close(); } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TransactionPipeline.java b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TransactionPipeline.java index 3ecb604..1ccdb72 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TransactionPipeline.java +++ b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TransactionPipeline.java @@ -21,23 +21,24 @@ package io.pixelsdb.pixels.sink.pipeline; import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; import io.pixelsdb.pixels.sink.processor.TransactionProcessor; -import io.pixelsdb.pixels.sink.provider.TransactionEventProvider; +import io.pixelsdb.pixels.sink.util.BlockingBoundedQueue; import io.pixelsdb.pixels.sink.writer.PixelsSinkWriter; import io.pixelsdb.pixels.sink.writer.PixelsSinkWriterFactory; public final class TransactionPipeline implements AutoCloseable { - private final TransactionEventProvider provider; + private final BlockingBoundedQueue eventQueue; private final PixelsSinkWriter writer; private final TransactionProcessor processor; private final Thread processorThread; public TransactionPipeline() { - this.provider = new TransactionEventProvider(); + this.eventQueue = new BlockingBoundedQueue<>(PixelsSinkConstants.MAX_QUEUE_SIZE); this.writer = PixelsSinkWriterFactory.getWriter(); - this.processor = new TransactionProcessor(provider, writer); + this.processor = new TransactionProcessor(eventQueue, writer); this.processorThread = new Thread(processor, "transaction-processor"); } @@ -51,14 +52,14 @@ public void start() public void publish(SinkProto.TransactionMetadata transaction) { - provider.publishTransaction(transaction); + eventQueue.put(transaction); } @Override public void close() { processor.stopProcessor(); - provider.close(); + eventQueue.close(); processorThread.interrupt(); } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/processor/TableProcessor.java b/src/main/java/io/pixelsdb/pixels/sink/processor/TableProcessor.java index c13c6b3..71b9fb9 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/processor/TableProcessor.java +++ b/src/main/java/io/pixelsdb/pixels/sink/processor/TableProcessor.java @@ -22,7 +22,7 @@ import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.provider.RowEventProvider; +import io.pixelsdb.pixels.sink.util.BlockingBoundedQueue; import io.pixelsdb.pixels.sink.util.MetricsFacade; import io.pixelsdb.pixels.sink.writer.PixelsSinkWriter; import io.pixelsdb.pixels.sink.writer.PixelsSinkWriterFactory; @@ -42,20 +42,22 @@ public class TableProcessor implements StoppableProcessor, Runnable private static final Logger LOGGER = LoggerFactory.getLogger(TableProcessor.class); private final AtomicBoolean running = new AtomicBoolean(true); private final PixelsSinkWriter pixelsSinkWriter; - private final RowEventProvider rowEventProvider; + private final BlockingBoundedQueue eventQueue; private final MetricsFacade metricsFacade = MetricsFacade.getInstance(); private Thread processorThread; private boolean tableAdded = false; - public TableProcessor(RowEventProvider rowEventProvider) + public TableProcessor(BlockingBoundedQueue eventQueue) { - this(rowEventProvider, PixelsSinkWriterFactory.getWriter()); + this(eventQueue, PixelsSinkWriterFactory.getWriter()); } - public TableProcessor(RowEventProvider rowEventProvider, PixelsSinkWriter pixelsSinkWriter) + public TableProcessor( + BlockingBoundedQueue eventQueue, + PixelsSinkWriter pixelsSinkWriter) { this.pixelsSinkWriter = pixelsSinkWriter; - this.rowEventProvider = rowEventProvider; + this.eventQueue = eventQueue; } @Override @@ -69,7 +71,7 @@ private void processLoop() { while (running.get()) { - RowChangeEvent event = rowEventProvider.takeRowChangeEvent(); + RowChangeEvent event = eventQueue.take(); if (event == null) { continue; diff --git a/src/main/java/io/pixelsdb/pixels/sink/processor/TransactionProcessor.java b/src/main/java/io/pixelsdb/pixels/sink/processor/TransactionProcessor.java index a18da8a..a4153ed 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/processor/TransactionProcessor.java +++ b/src/main/java/io/pixelsdb/pixels/sink/processor/TransactionProcessor.java @@ -21,7 +21,7 @@ package io.pixelsdb.pixels.sink.processor; import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.provider.TransactionEventProvider; +import io.pixelsdb.pixels.sink.util.BlockingBoundedQueue; import io.pixelsdb.pixels.sink.writer.PixelsSinkWriter; import io.pixelsdb.pixels.sink.writer.PixelsSinkWriterFactory; import org.slf4j.Logger; @@ -34,18 +34,19 @@ public class TransactionProcessor implements Runnable, StoppableProcessor private static final Logger LOGGER = LoggerFactory.getLogger(TransactionProcessor.class); private final PixelsSinkWriter sinkWriter; private final AtomicBoolean running = new AtomicBoolean(true); - private final TransactionEventProvider transactionEventProvider; + private final BlockingBoundedQueue eventQueue; - public TransactionProcessor(TransactionEventProvider transactionEventProvider) + public TransactionProcessor( + BlockingBoundedQueue eventQueue) { - this(transactionEventProvider, PixelsSinkWriterFactory.getWriter()); + this(eventQueue, PixelsSinkWriterFactory.getWriter()); } public TransactionProcessor( - TransactionEventProvider transactionEventProvider, + BlockingBoundedQueue eventQueue, PixelsSinkWriter sinkWriter) { - this.transactionEventProvider = transactionEventProvider; + this.eventQueue = eventQueue; this.sinkWriter = sinkWriter; } @@ -54,7 +55,7 @@ public void run() { while (running.get()) { - SinkProto.TransactionMetadata transaction = transactionEventProvider.takeTransaction(); + SinkProto.TransactionMetadata transaction = eventQueue.take(); if (transaction == null) { LOGGER.warn("Received null transaction"); diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/RowEventProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/RowEventProvider.java deleted file mode 100644 index 595a63b..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/RowEventProvider.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - -import io.pixelsdb.pixels.sink.event.RowChangeEvent; - -public final class RowEventProvider extends BlockingEventProvider -{ - public void publishRowChangeEvent(RowChangeEvent event) - { - publish(event); - } - - public RowChangeEvent takeRowChangeEvent() - { - return take(); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventProvider.java b/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventProvider.java deleted file mode 100644 index 9c47856..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/TransactionEventProvider.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - -import io.pixelsdb.pixels.sink.SinkProto; - -public final class TransactionEventProvider - extends BlockingEventProvider -{ - public void publishTransaction(SinkProto.TransactionMetadata transaction) - { - publish(transaction); - } - - public SinkProto.TransactionMetadata takeTransaction() - { - return take(); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/BlockingEventProvider.java b/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueue.java similarity index 71% rename from src/main/java/io/pixelsdb/pixels/sink/provider/BlockingEventProvider.java rename to src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueue.java index d41db4e..eb47f2b 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/BlockingEventProvider.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueue.java @@ -18,38 +18,43 @@ * . */ -package io.pixelsdb.pixels.sink.provider; - -import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; +package io.pixelsdb.pixels.sink.util; import java.io.Closeable; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; /** - * Bounded channel for already converted sink events. + * A bounded blocking queue with an explicit shutdown signal. * - *

The provider deliberately does not know how an event was read or - * converted. Source adapters publish canonical objects and processors consume - * them from this channel.

+ *

Closing the queue discards pending values and wakes a blocked consumer. + * This is intended for pipeline shutdown, not graceful draining.

*/ -public class BlockingEventProvider implements Closeable +public final class BlockingBoundedQueue implements Closeable { private static final Object POISON_PILL = new Object(); - private final BlockingQueue queue = - new LinkedBlockingQueue<>(PixelsSinkConstants.MAX_QUEUE_SIZE); + private final BlockingQueue queue; private volatile boolean closed; - public void publish(T event) + public BlockingBoundedQueue(int capacity) + { + if (capacity <= 0) + { + throw new IllegalArgumentException("capacity must be positive"); + } + this.queue = new LinkedBlockingQueue<>(capacity); + } + + public void put(T value) { - if (event == null || closed) + if (value == null || closed) { return; } try { - queue.put(event); + queue.put(value); } catch (InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/src/test/java/io/pixelsdb/pixels/sink/DebeziumEngineTest.java b/src/test/java/io/pixelsdb/pixels/sink/DebeziumEngineTest.java index b238ed1..d4f9c80 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/DebeziumEngineTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/DebeziumEngineTest.java @@ -25,9 +25,12 @@ import io.debezium.engine.format.ChangeEventFormat; import org.apache.kafka.connect.source.SourceRecord; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import java.util.List; +import java.nio.file.Path; import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -38,11 +41,12 @@ * @author: AntiO2 * @date: 2025/9/25 12:16 */ -public class DebeziumEngineTest +@Tag("integration") +class DebeziumEngineTest { @Test @Disabled("Manual PostgreSQL integration test; requires an external database") - public void testPostgresCDC() + void testPostgresCDC(@TempDir Path tempDir) throws Exception { final Properties props = new Properties(); @@ -51,17 +55,24 @@ public void testPostgresCDC() props.setProperty("provide.transaction.metadata", "true"); props.setProperty("offset.storage", "org.apache.kafka.connect.storage.FileOffsetBackingStore"); - props.setProperty("offset.storage.file.filename", "/tmp/offsets.dat"); + props.setProperty("offset.storage.file.filename", + tempDir.resolve("offsets.dat").toString()); props.setProperty("offset.flush.interval.ms", "60000"); props.setProperty("schema.history.internal", "io.debezium.storage.file.history.FileSchemaHistory"); - props.setProperty("schema.history.internal.file.filename", "/tmp/schemahistory.dat"); - - props.setProperty("database.hostname", "localhost"); - props.setProperty("database.port", "5432"); - props.setProperty("database.user", "pixels"); - props.setProperty("database.password", "pixels_realtime_crud"); - props.setProperty("database.dbname", "pixels_bench_sf1x"); + props.setProperty("schema.history.internal.file.filename", + tempDir.resolve("schemahistory.dat").toString()); + + props.setProperty("database.hostname", + requiredProperty("pixels.sink.debezium.database.hostname")); + props.setProperty("database.port", + requiredProperty("pixels.sink.debezium.database.port")); + props.setProperty("database.user", + requiredProperty("pixels.sink.debezium.database.user")); + props.setProperty("database.password", + requiredProperty("pixels.sink.debezium.database.password")); + props.setProperty("database.dbname", + requiredProperty("pixels.sink.debezium.database.name")); props.setProperty("plugin.name", "pgoutput"); props.setProperty("database.server.id", "1"); props.setProperty("schema.include.list", "public"); @@ -83,11 +94,24 @@ public void testPostgresCDC() ExecutorService executor = Executors.newSingleThreadExecutor(); executor.execute(engine); - - while (true) + try { + Thread.sleep(Long.getLong("pixels.sink.debezium.run.millis", 1000L)); + } finally + { + engine.close(); + executor.shutdownNow(); + } + } + private static String requiredProperty(String key) + { + String value = System.getProperty(key); + if (value == null || value.isBlank()) + { + throw new IllegalStateException("Set -D" + key + " to run this integration test"); } + return value; } class MyChangeConsumer implements DebeziumEngine.ChangeConsumer> diff --git a/src/test/java/io/pixelsdb/pixels/sink/TestConfig.java b/src/test/java/io/pixelsdb/pixels/sink/TestConfig.java new file mode 100644 index 0000000..ab32852 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/TestConfig.java @@ -0,0 +1,64 @@ +/* + * Copyright 2025 PixelsDB. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.pixelsdb.pixels.sink; + +import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import org.junit.jupiter.api.Assumptions; + +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Objects; + +public final class TestConfig +{ + public static final String INTEGRATION_CONFIG_PROPERTY = + "pixels.sink.integration.config"; + + private TestConfig() + { + } + + public static void initializeUnitConfig() throws Exception + { + PixelsSinkConfigFactory.reset(); + PixelsSinkConfigFactory.initialize(resourcePath("pixels-sink-test.properties")); + } + + public static void initializeIntegrationConfig() throws Exception + { + String configPath = System.getProperty(INTEGRATION_CONFIG_PROPERTY); + Assumptions.assumeTrue(configPath != null && !configPath.isBlank(), + "Set -D" + INTEGRATION_CONFIG_PROPERTY + " to run integration tests"); + PixelsSinkConfigFactory.reset(); + PixelsSinkConfigFactory.initialize(configPath); + } + + public static String resourcePath(String resourceName) + { + try + { + Path path = Paths.get(Objects.requireNonNull( + TestConfig.class.getClassLoader().getResource(resourceName), + "Missing test resource: " + resourceName).toURI()); + return path.toString(); + } catch (URISyntaxException e) + { + throw new IllegalStateException("Invalid test resource: " + resourceName, e); + } + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/TestSplitString.java b/src/test/java/io/pixelsdb/pixels/sink/TestSplitString.java index 5d32ba9..5abe465 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/TestSplitString.java +++ b/src/test/java/io/pixelsdb/pixels/sink/TestSplitString.java @@ -16,20 +16,33 @@ */ package io.pixelsdb.pixels.sink; -import org.junit.Test; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; /** * Created at: 29/04/2021 * Author: hank */ -public class TestSplitString +class TestSplitString { @Test - public void test() + void shouldSplitPipeDelimitedRecordAndKeepTrailingEmptyField() { String s = "1|3689999|O|224560.83|1996-01-02|5-LOW|Clerk#000095055|0|nstructions sleep furiously among |"; - String reg = "\\\\|"; - String[] splits = s.split(reg); - System.out.println(splits.length); + String[] splits = s.split("\\|", -1); + + assertArrayEquals(new String[] { + "1", + "3689999", + "O", + "224560.83", + "1996-01-02", + "5-LOW", + "Clerk#000095055", + "0", + "nstructions sleep furiously among ", + "" + }, splits); } } diff --git a/src/test/java/io/pixelsdb/pixels/sink/concurrent/RetinaWriterTest.java b/src/test/java/io/pixelsdb/pixels/sink/concurrent/RetinaWriterTest.java index 74b9e4f..9325032 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/concurrent/RetinaWriterTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/concurrent/RetinaWriterTest.java @@ -18,12 +18,13 @@ package io.pixelsdb.pixels.sink.concurrent; import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.TestConfig; import io.pixelsdb.pixels.sink.TestUtils; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; import io.pixelsdb.pixels.sink.event.RowChangeEvent; import io.pixelsdb.pixels.sink.exception.SinkException; import io.pixelsdb.pixels.sink.writer.retina.RetinaWriter; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; @@ -43,6 +44,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +@Tag("integration") class RetinaWriterTest { private static final Logger LOGGER = LoggerFactory.getLogger(RetinaWriterTest.class); @@ -52,9 +54,9 @@ class RetinaWriterTest private CountDownLatch latch; @BeforeEach - void setUp() throws IOException + void setUp() throws Exception { - PixelsSinkConfigFactory.initialize(""); + TestConfig.initializeIntegrationConfig(); testExecutor = TestUtils.synchronousExecutor(); dispatchedEvents = Collections.synchronizedList(new ArrayList<>()); @@ -155,16 +157,12 @@ void shouldProcessNonTransactionalEvents(SinkProto.OperationType opType) throws coordinator.writeRow(event); TimeUnit.MILLISECONDS.sleep(10); assertEquals(1, dispatchedEvents.size()); - PixelsSinkConfigFactory.reset(); } @ParameterizedTest @ValueSource(ints = {1, 3, 9, 16}) void shouldHandleConcurrentEvents(int threadCount) throws SinkException, IOException, InterruptedException { - PixelsSinkConfigFactory.reset(); - PixelsSinkConfigFactory.initialize(""); - latch = new CountDownLatch(threadCount); coordinator.writeTrans(buildBeginTx("tx5")); // concurrently send event @@ -192,6 +190,5 @@ void shouldHandleConcurrentEvents(int threadCount) throws SinkException, IOExcep LOGGER.debug("Thread Count: {} DispatchedEvents size: {}", threadCount, dispatchedEvents.size()); LOGGER.debug("Thread Count: {} DispatchedEvents size: {}", threadCount, dispatchedEvents.size()); assertEquals(threadCount, dispatchedEvents.size()); - PixelsSinkConfigFactory.reset(); } } \ No newline at end of file diff --git a/src/test/java/io/pixelsdb/pixels/sink/concurrent/TransactionServiceTest.java b/src/test/java/io/pixelsdb/pixels/sink/concurrent/TransactionServiceTest.java index 0f590f1..3ea3b18 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/concurrent/TransactionServiceTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/concurrent/TransactionServiceTest.java @@ -25,6 +25,7 @@ import io.pixelsdb.pixels.common.transaction.TransContext; import io.pixelsdb.pixels.common.transaction.TransService; import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,16 +36,17 @@ import static org.junit.jupiter.api.Assertions.assertTrue; @Slf4j -public class TransactionServiceTest +@Tag("integration") +class TransactionServiceTest { private static final Logger logger = LoggerFactory.getLogger(TransactionServiceTest.class); @Test public void testTransactionService() { - int numTransactions = 10; + int numTransactions = Integer.getInteger("pixels.sink.test.transactions", 10); - TransService transService = TransService.CreateInstance("localhost", 18889); + TransService transService = TransService.Instance(); try { List transContexts = transService.beginTransBatch(numTransactions, false); @@ -66,9 +68,9 @@ public void testTransactionService() @Test public void testBatchRequest() { - int numTransactions = 1000; + int numTransactions = Integer.getInteger("pixels.sink.test.batch.transactions", 100); - TransService transService = TransService.CreateInstance("localhost", 18889); + TransService transService = TransService.Instance(); try { List transContexts = transService.beginTransBatch(numTransactions, false); diff --git a/src/test/java/io/pixelsdb/pixels/sink/consumer/AvroConsumerTest.java b/src/test/java/io/pixelsdb/pixels/sink/consumer/AvroConsumerTest.java index b4b636c..d52322f 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/consumer/AvroConsumerTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/consumer/AvroConsumerTest.java @@ -22,7 +22,7 @@ import io.apicurio.registry.serde.SerdeConfig; import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.TestConfig; import io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventAvroDeserializer; import io.pixelsdb.pixels.sink.event.RowChangeEvent; import io.pixelsdb.pixels.sink.exception.SinkException; @@ -33,30 +33,31 @@ import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.common.serialization.StringDeserializer; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import java.io.IOException; import java.time.Duration; import java.util.Collections; import java.util.Properties; +import java.util.UUID; -public class AvroConsumerTest -{ +import static org.junit.jupiter.api.Assertions.assertTrue; - private static final String TOPIC = "oltp_server.pixels_realtime_crud.customer"; - private static final String REGISTRY_URL = "http://localhost:8080/apis/registry/v2"; - private static final String BOOTSTRAP_SERVERS = "localhost:29092"; - private static final String GROUP_ID = "avro-consumer-test-group-1"; +@Tag("integration") +class AvroConsumerTest +{ + private static final int MAX_POLL_CYCLES = 100; private static KafkaConsumer getRowChangeEventAvroKafkaConsumer() { Properties props = new Properties(); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); - props.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers()); + props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId()); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, RowChangeEventAvroDeserializer.class.getName()); props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - props.put(SerdeConfig.REGISTRY_URL, REGISTRY_URL); + props.put(SerdeConfig.REGISTRY_URL, registryUrl()); props.put(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true"); props.put(SerdeConfig.CHECK_PERIOD_MS, "30000"); @@ -76,12 +77,12 @@ private static void processRecord(RowChangeEvent event) private static KafkaConsumer getStringGenericRecordKafkaConsumer() { Properties props = new Properties(); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVERS); - props.put(ConsumerConfig.GROUP_ID_CONFIG, GROUP_ID); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers()); + props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId()); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, AvroKafkaDeserializer.class.getName()); props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - props.put(SerdeConfig.REGISTRY_URL, REGISTRY_URL); + props.put(SerdeConfig.REGISTRY_URL, registryUrl()); props.put(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true"); props.put(SerdeConfig.CHECK_PERIOD_MS, "30000"); @@ -132,23 +133,26 @@ private static String getSchemaIdFromRegistry(RegistryClient client, Schema sche } @Test - public void avroConsumerTest() + void shouldConsumeGenericAvroRecords() { KafkaConsumer consumer = getStringGenericRecordKafkaConsumer(); - consumer.subscribe(Collections.singletonList(TOPIC)); + consumer.subscribe(Collections.singletonList(topic())); - RegistryClient registryClient = RegistryClientFactory.create(REGISTRY_URL); + RegistryClient registryClient = RegistryClientFactory.create(registryUrl()); try { - while (true) + int recordCount = 0; + for (int i = 0; i < MAX_POLL_CYCLES; ++i) { ConsumerRecords records = consumer.poll(Duration.ofMillis(100)); for (ConsumerRecord record : records) { processRecord(record, registryClient); + recordCount++; } } + assertTrue(recordCount > 0, "No Avro records were consumed"); } finally { consumer.close(); @@ -156,25 +160,57 @@ public void avroConsumerTest() } @Test - public void sinkConsumerTest() throws IOException + void shouldConsumeRowChangeEvents() throws Exception { - PixelsSinkConfigFactory.initialize("/home/anti/work/pixels-sink/src/main/resources/pixels-sink.local.properties"); + TestConfig.initializeIntegrationConfig(); KafkaConsumer consumer = getRowChangeEventAvroKafkaConsumer(); - consumer.subscribe(Collections.singletonList(TOPIC)); + consumer.subscribe(Collections.singletonList(topic())); try { - while (true) + int recordCount = 0; + for (int i = 0; i < MAX_POLL_CYCLES; ++i) { ConsumerRecords records = consumer.poll(Duration.ofMillis(100)); for (ConsumerRecord record : records) { processRecord(record.value()); + recordCount++; } } + assertTrue(recordCount > 0, "No row-change records were consumed"); } finally { consumer.close(); } } + + private static String topic() + { + return requiredProperty("pixels.sink.test.topic"); + } + + private static String registryUrl() + { + return requiredProperty("pixels.sink.test.registry.url"); + } + + private static String bootstrapServers() + { + return requiredProperty("pixels.sink.test.bootstrap.servers"); + } + + private static String groupId() + { + return System.getProperty( + "pixels.sink.test.group.id", "pixels-sink-test-" + UUID.randomUUID()); + } + + private static String requiredProperty(String key) + { + String value = System.getProperty(key); + Assumptions.assumeTrue(value != null && !value.isBlank(), + "Set -D" + key + " to run this integration test"); + return value; + } } \ No newline at end of file diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizerTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizerTest.java index e2acabd..2a4b5c8 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizerTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizerTest.java @@ -16,6 +16,7 @@ import io.pixelsdb.pixels.common.metadata.domain.Table; import io.pixelsdb.pixels.core.TypeDescription; import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.TestConfig; import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapterRegistry; import io.pixelsdb.pixels.sink.conversion.debezium.source.MySqlSourceAdapter; import io.pixelsdb.pixels.sink.conversion.debezium.source.PostgresSourceAdapter; @@ -35,7 +36,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Objects; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -56,10 +56,7 @@ class DebeziumEnvelopeNormalizerTest @BeforeAll static void setUpConfig() throws Exception { - PixelsSinkConfigFactory.reset(); - PixelsSinkConfigFactory.initialize(Objects.requireNonNull( - DebeziumEnvelopeNormalizerTest.class.getClassLoader() - .getResource("pixels-sink-test.properties")).getPath()); + TestConfig.initializeUnitConfig(); Map registry = metadataRegistry(); hadMySqlMetadata = registry.containsKey(MYSQL_TABLE); previousMySqlMetadata = registry.get(MYSQL_TABLE); @@ -155,6 +152,23 @@ void shouldResolveConnectorAdaptersThroughSpi() DebeziumSourceAdapterRegistry.resolve("postgresql").connector()); } + @Test + void shouldRejectBlankConnector() + { + assertThrows(IllegalArgumentException.class, + () -> DebeziumSourceAdapterRegistry.resolve(" ")); + } + + @Test + void shouldRejectUnsupportedConnector() + { + IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, + () -> DebeziumSourceAdapterRegistry.resolve("oracle")); + + assertTrue(error.getMessage().contains("Unsupported Debezium connector")); + } + @Test void shouldPreserveMySqlGtidAsOpaqueTransactionId() throws Exception { diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverterTest.java index a8eae45..1a31623 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverterTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverterTest.java @@ -31,10 +31,6 @@ import java.math.BigDecimal; import java.nio.ByteBuffer; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; import java.util.Base64; import java.util.Date; import java.util.List; @@ -156,14 +152,12 @@ void shouldPreserveExistingIntegerWidths() throws Exception } @Test - void testParseDate() + void shouldConvertDebeziumEpochDay() { int day = 17059; Date debeziumDate = TestDateUtil.fromDebeziumDate(day); String dayString = TestDateUtil.convertDateToDayString(debeziumDate); - long ts = 1473927308302000L; - LocalDateTime localDateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts / 1000), ZoneOffset.UTC); - ZonedDateTime zonedDateTime = Instant.ofEpochMilli(ts).atZone(ZoneOffset.UTC); - boolean pause = true; + + assertEquals("2016-09-15", dayString); } } diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventJsonDeserializerTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventJsonDeserializerTest.java index d970187..d7e369f 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventJsonDeserializerTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventJsonDeserializerTest.java @@ -21,6 +21,7 @@ import io.pixelsdb.pixels.common.metadata.domain.Column; import io.pixelsdb.pixels.common.metadata.domain.Table; import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.TestConfig; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; import io.pixelsdb.pixels.sink.event.RowChangeEvent; import io.pixelsdb.pixels.sink.metadata.TableMetadata; @@ -35,6 +36,7 @@ import java.net.URISyntaxException; import java.nio.file.Files; import java.nio.file.Paths; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import java.util.Objects; @@ -57,10 +59,7 @@ class RowChangeEventJsonDeserializerTest @BeforeAll static void setUpConfig() throws Exception { - PixelsSinkConfigFactory.reset(); - PixelsSinkConfigFactory.initialize(Objects.requireNonNull( - RowChangeEventJsonDeserializerTest.class.getClassLoader() - .getResource("pixels-sink-test.properties")).getPath()); + TestConfig.initializeUnitConfig(); Map registry = metadataRegistry(); hadRegionMetadata = registry.containsKey(REGION_TABLE); previousRegionMetadata = registry.get(REGION_TABLE); @@ -85,9 +84,9 @@ static void resetConfig() throws ReflectiveOperationException private String loadSchemaFromFile(String filename) throws IOException, URISyntaxException { ClassLoader classLoader = getClass().getClassLoader(); - return new String(Files.readAllBytes(Paths.get( - Objects.requireNonNull(classLoader.getResource(filename)).toURI() - ))); + return Files.readString(Paths.get( + Objects.requireNonNull(classLoader.getResource(filename)).toURI()), + StandardCharsets.UTF_8); } // @ParameterizedTest @@ -110,13 +109,27 @@ private String loadSchemaFromFile(String filename) throws IOException, URISyntax void shouldHandleDeleteOperation() throws Exception { String jsonData = loadSchemaFromFile("records/delete.json"); - RowChangeEvent event = deserializer.deserialize("test_topic", jsonData.getBytes()); + RowChangeEvent event = deserializer.deserialize( + "test_topic", jsonData.getBytes(StandardCharsets.UTF_8)); assertTrue(event.isDelete()); // assertNotNull(event.getBeforeData()); // assertNull(event.getAfterData()); } + @Test + void shouldHandleUpdateOperation() throws Exception + { + String jsonData = loadSchemaFromFile("records/update.json"); + RowChangeEvent event = deserializer.deserialize( + "test_topic", jsonData.getBytes(StandardCharsets.UTF_8)); + + assertTrue(event.isUpdate()); + assertEquals("region", event.getTable()); + assertTrue(event.hasBeforeData()); + assertTrue(event.hasAfterData()); + } + @Test void shouldHandleEmptyData() @@ -143,7 +156,7 @@ void shouldDeserializeTransactionMetadataWithConfiguredConnector() """; SinkProto.TransactionMetadata transaction = transactionDeserializer.deserialize( - "transaction", json.getBytes()); + "transaction", json.getBytes(StandardCharsets.UTF_8)); assertEquals(SinkProto.TransactionStatus.END, transaction.getStatus()); assertEquals("mysql-tx-1", transaction.getId()); diff --git a/src/test/java/io/pixelsdb/pixels/sink/event/RowBatchTest.java b/src/test/java/io/pixelsdb/pixels/sink/event/RowBatchTest.java index a467267..932b1f3 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/event/RowBatchTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/event/RowBatchTest.java @@ -22,42 +22,54 @@ import io.pixelsdb.pixels.core.TypeDescription; import io.pixelsdb.pixels.core.vector.BinaryColumnVector; +import io.pixelsdb.pixels.core.vector.IntColumnVector; import io.pixelsdb.pixels.core.vector.VectorizedRowBatch; import org.junit.jupiter.api.Test; -import java.util.ArrayList; -import java.util.List; +import static org.junit.jupiter.api.Assertions.assertEquals; -public class RowBatchTest +class RowBatchTest { - private final List columnNames = new ArrayList<>(); - private final List columnTypes = new ArrayList<>(); - @Test - public void integerRowBatchTest() + void integerRowBatchTest() { - columnNames.add("id"); - columnTypes.add("int"); + TypeDescription schema = TypeDescription.createSchemaFromStrings( + java.util.List.of("id"), java.util.List.of("int")); + VectorizedRowBatch rowBatch = schema.createRowBatch( + 3, TypeDescription.Mode.CREATE_INT_VECTOR_FOR_INT); + IntColumnVector vector = (IntColumnVector) rowBatch.cols[0]; + vector.add(10); + vector.add(20); + vector.add(30); + rowBatch.size = 3; - TypeDescription schema = TypeDescription.createSchemaFromStrings(columnNames, columnTypes); - VectorizedRowBatch rowBatch = schema.createRowBatch(3, TypeDescription.Mode.CREATE_INT_VECTOR_FOR_INT); VectorizedRowBatch newRowBatch = VectorizedRowBatch.deserialize(rowBatch.serialize()); + assertEquals(3, newRowBatch.size); + IntColumnVector newVector = (IntColumnVector) newRowBatch.cols[0]; + assertEquals(10, newVector.vector[0]); + assertEquals(20, newVector.vector[1]); + assertEquals(30, newVector.vector[2]); } @Test - public void varcharRowBatchTest() + void varcharRowBatchTest() { - columnNames.add("name"); - columnTypes.add("varchar(100)"); - - TypeDescription schema = TypeDescription.createSchemaFromStrings(columnNames, columnTypes); + TypeDescription schema = TypeDescription.createSchemaFromStrings( + java.util.List.of("name"), java.util.List.of("varchar(100)")); VectorizedRowBatch rowBatch = schema.createRowBatch(3, TypeDescription.Mode.CREATE_INT_VECTOR_FOR_INT); BinaryColumnVector v = (BinaryColumnVector) rowBatch.cols[0]; v.add("rr"); v.add("zz"); v.add("rr"); + rowBatch.size = 3; + VectorizedRowBatch newRowBatch = VectorizedRowBatch.deserialize(rowBatch.serialize()); + assertEquals(3, newRowBatch.size); + BinaryColumnVector newVector = (BinaryColumnVector) newRowBatch.cols[0]; + assertEquals("rr", new String(newVector.vector[0], newVector.start[0], newVector.lens[0])); + assertEquals("zz", new String(newVector.vector[1], newVector.start[1], newVector.lens[1])); + assertEquals("rr", new String(newVector.vector[2], newVector.start[2], newVector.lens[2])); } } diff --git a/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventTest.java b/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventTest.java index 139be39..afe807c 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventTest.java @@ -21,38 +21,30 @@ package io.pixelsdb.pixels.sink.event; import com.google.protobuf.ByteString; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.io.IOException; import java.nio.ByteBuffer; -public class RowChangeEventTest -{ - private static Logger LOGGER = LoggerFactory.getLogger(RowChangeEventTest.class); - - @BeforeAll - public static void init() throws IOException - { - PixelsSinkConfigFactory.initialize("/home/ubuntu/pixels-sink/conf/pixels-sink.aws.properties"); - } - +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +class RowChangeEventTest +{ @Test - public void testSameHash() + void shouldReturnStableNonNegativeBucket() { + ByteString indexKey = getIndexKey(0); + int expectedBucket = RowChangeEvent.getBucketIdFromByteBuffer(indexKey); + for (int i = 0; i < 10; ++i) { - ByteString indexKey = getIndexKey(0); int bucket = RowChangeEvent.getBucketIdFromByteBuffer(indexKey); - LOGGER.info("Bucket: {}", bucket); + assertEquals(expectedBucket, bucket); } + assertTrue(expectedBucket >= 0); } - private ByteString getIndexKey(int key) + private static ByteString getIndexKey(int key) { int keySize = Integer.BYTES; ByteBuffer byteBuffer = ByteBuffer.allocate(keySize); diff --git a/src/test/java/io/pixelsdb/pixels/sink/freshness/TestFreshnessClient.java b/src/test/java/io/pixelsdb/pixels/sink/freshness/TestFreshnessClient.java index a524b0a..f6ce0c8 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/freshness/TestFreshnessClient.java +++ b/src/test/java/io/pixelsdb/pixels/sink/freshness/TestFreshnessClient.java @@ -21,21 +21,27 @@ package io.pixelsdb.pixels.sink.freshness; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.TestConfig; import io.pixelsdb.pixels.sink.util.MetricsFacade; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import static org.junit.jupiter.api.Assertions.assertTrue; + // We extend FreshnessClient to access the protected queryAndCalculateFreshness method -public class TestFreshnessClient +@Tag("integration") +class TestFreshnessClient { // Mocks for JDBC dependencies @@ -49,44 +55,53 @@ public class TestFreshnessClient private FreshnessClient client; // The instance of the client to test @BeforeAll - public static void setUp() throws IOException + static void setUp() throws Exception { - // Initialization as per the user's template - PixelsSinkConfigFactory.initialize("/home/ubuntu/disk1/opt/pixels-sink/conf/pixels-sink.hudi.properties"); + TestConfig.initializeIntegrationConfig(); } @Test - public void testFreshnessCalculationSuccess() throws Exception + void testFreshnessCalculationSuccess() throws Exception { FreshnessClient freshnessClient = FreshnessClient.getInstance(); freshnessClient.addMonitoredTable("customer"); - freshnessClient.start(); - while (true) + try + { + freshnessClient.start(); + Thread.sleep(1000); + } finally { + freshnessClient.stop(); } } @Test - public void testSnapshotTs() throws SQLException + void testSnapshotTs() throws SQLException { FreshnessClient freshnessClient = FreshnessClient.getInstance(); - Connection connection = freshnessClient.createNewConnection(123456L); - String query = String.format("SELECT max(freshness_ts) FROM company"); - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(query); - resultSet.next(); + try (Connection connection = freshnessClient.createNewConnection(123456L); + Statement statement = connection.createStatement()) + { + String query = "SELECT max(freshness_ts) FROM company"; + try (ResultSet resultSet = statement.executeQuery(query)) + { + assertTrue(resultSet.next()); + } + } } @Test - public void testLoanTransQueryPerformance() throws SQLException + void testLoanTransQueryPerformance(@org.junit.jupiter.api.io.TempDir Path tempDir) + throws SQLException { FreshnessClient freshnessClient = FreshnessClient.getInstance(); Connection connection = freshnessClient.createNewConnection(12345689100L); String query = "SELECT max(freshness_ts) FROM nation"; - String csvFileName = "loantrans_query_results.csv"; - int iterations = 1000; - try (PrintWriter writer = new PrintWriter(new FileWriter(csvFileName))) + Path csvFile = tempDir.resolve("loantrans_query_results.csv"); + int iterations = Integer.getInteger("pixels.sink.test.iterations", 10); + try (PrintWriter writer = new PrintWriter( + Files.newBufferedWriter(csvFile, StandardCharsets.UTF_8))) { for (int i = 0; i < iterations; i++) @@ -113,7 +128,7 @@ public void testLoanTransQueryPerformance() throws SQLException writer.printf("%d,%d,%d%n", startTime, maxFreshnessTs, durationMs); writer.flush(); } - System.out.println("Test completed. Results saved to: " + csvFileName); + System.out.println("Test completed. Results saved to: " + csvFile); } catch (IOException e) { e.printStackTrace(); diff --git a/src/test/java/io/pixelsdb/pixels/sink/metadata/TestIndexService.java b/src/test/java/io/pixelsdb/pixels/sink/metadata/TestIndexService.java index 0804b7f..efccfc8 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/metadata/TestIndexService.java +++ b/src/test/java/io/pixelsdb/pixels/sink/metadata/TestIndexService.java @@ -23,6 +23,7 @@ import io.pixelsdb.pixels.common.exception.MetadataException; import io.pixelsdb.pixels.common.index.IndexOption; import io.pixelsdb.pixels.common.index.service.IndexService; +import io.pixelsdb.pixels.common.index.service.IndexServiceProvider; import io.pixelsdb.pixels.common.metadata.MetadataService; import io.pixelsdb.pixels.common.metadata.domain.Layout; import io.pixelsdb.pixels.common.metadata.domain.SinglePointIndex; @@ -30,6 +31,7 @@ import io.pixelsdb.pixels.daemon.MetadataProto; import io.pixelsdb.pixels.index.IndexProto; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.nio.ByteBuffer; @@ -40,11 +42,13 @@ * @author: AntiO2 * @date: 2025/8/5 04:34 */ -public class TestIndexService +@Tag("integration") +class TestIndexService { private final MetadataService metadataService = MetadataService.Instance(); - private final IndexService indexService = null; // TODO + private final IndexService indexService = + IndexServiceProvider.getService(IndexServiceProvider.ServiceMode.rpc); @Test public void testCreateFreshnessIndex() throws MetadataException @@ -67,7 +71,6 @@ public void testCreateFreshnessIndex() throws MetadataException SinglePointIndex index = new SinglePointIndex(singlePointIndexbuilder.build()); boolean result = metadataService.createSinglePointIndex(index); Assertions.assertTrue(result); - boolean pause = true; } @Test @@ -93,7 +96,6 @@ public void testCreateIndex() throws MetadataException SinglePointIndex index = new SinglePointIndex(singlePointIndexbuilder.build()); boolean result = metadataService.createSinglePointIndex(index); Assertions.assertTrue(result); - boolean pause = true; } @Test @@ -106,7 +108,6 @@ public void testGetIndex() throws MetadataException SinglePointIndex index = metadataService.getPrimaryIndex(id); Assertions.assertNotNull(index); - boolean pause = true; } @Test @@ -115,7 +116,6 @@ public void testGetRowID() throws MetadataException, IndexException int numRowIds = 10000; IndexProto.RowIdBatch rowIdBatch = indexService.allocateRowIdBatch(4, numRowIds); Assertions.assertEquals(rowIdBatch.getLength(), numRowIds); - boolean pause = true; } @Test @@ -164,6 +164,5 @@ public void testPutAndDelete() throws MetadataException, IndexException IndexProto.RowLocation rowLocation = indexService.deletePrimaryIndexEntry(builder.getIndexKey(), indexOption); - boolean pause = false; } } diff --git a/src/test/java/io/pixelsdb/pixels/sink/pipeline/PipelineTest.java b/src/test/java/io/pixelsdb/pixels/sink/pipeline/PipelineTest.java new file mode 100644 index 0000000..415b690 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/pipeline/PipelineTest.java @@ -0,0 +1,62 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.pipeline; + +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.TestConfig; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + +class PipelineTest +{ + @Test + void shouldStartAndCloseTablePipeline() throws Exception + { + TestConfig.initializeUnitConfig(); + + assertDoesNotThrow(() -> { + try (TablePipeline pipeline = new TablePipeline()) + { + pipeline.start(); + } + }); + } + + @Test + void shouldStartTransactionPipelineOnlyOnce() throws Exception + { + TestConfig.initializeUnitConfig(); + + assertDoesNotThrow(() -> { + try (TransactionPipeline pipeline = new TransactionPipeline()) + { + pipeline.start(); + pipeline.start(); + pipeline.publish(SinkProto.TransactionMetadata.newBuilder() + .setId("transaction-1") + .setStatus(SinkProto.TransactionStatus.END) + .build()); + } + }); + } + +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/processor/TableProcessorTest.java b/src/test/java/io/pixelsdb/pixels/sink/processor/TableProcessorTest.java new file mode 100644 index 0000000..ec2b778 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/processor/TableProcessorTest.java @@ -0,0 +1,108 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.processor; + +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.TestConfig; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.util.BlockingBoundedQueue; +import io.pixelsdb.pixels.sink.writer.PixelsSinkWriter; +import io.pixelsdb.pixels.core.TypeDescription; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class TableProcessorTest +{ + @Test + void shouldForwardRowsToWriter() throws Exception + { + TestConfig.initializeUnitConfig(); + BlockingBoundedQueue queue = new BlockingBoundedQueue<>(1); + RecordingWriter writer = new RecordingWriter(); + TableProcessor processor = new TableProcessor(queue, writer); + RowChangeEvent event = rowEvent(); + + try + { + processor.run(); + queue.put(event); + + assertTrue(writer.rowWritten.await(1, TimeUnit.SECONDS)); + assertSame(event, writer.row); + } finally + { + processor.stopProcessor(); + queue.close(); + } + } + + private static RowChangeEvent rowEvent() throws Exception + { + SinkProto.RowRecord record = SinkProto.RowRecord.newBuilder() + .setOp(SinkProto.OperationType.INSERT) + .setSource(SinkProto.SourceInfo.newBuilder() + .setDb("test_db") + .setTable("test_table")) + .setAfter(SinkProto.RowValue.newBuilder() + .addValues(SinkProto.ColumnValue.newBuilder().setIsNull(true))) + .build(); + TypeDescription schema = TypeDescription.createSchemaFromStrings( + List.of("id"), List.of("int")); + return new RowChangeEvent(record, schema, null); + } + + private static final class RecordingWriter implements PixelsSinkWriter + { + private final CountDownLatch rowWritten = new CountDownLatch(1); + private RowChangeEvent row; + + @Override + public void flush() + { + } + + @Override + public boolean writeRow(RowChangeEvent rowChangeEvent) + { + row = rowChangeEvent; + rowWritten.countDown(); + return true; + } + + @Override + public boolean writeTrans(SinkProto.TransactionMetadata transactionMetadata) + { + return false; + } + + @Override + public void close() throws IOException + { + } + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/processor/TransactionProcessorTest.java b/src/test/java/io/pixelsdb/pixels/sink/processor/TransactionProcessorTest.java new file mode 100644 index 0000000..a18f9f3 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/processor/TransactionProcessorTest.java @@ -0,0 +1,95 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.processor; + +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.util.BlockingBoundedQueue; +import io.pixelsdb.pixels.sink.writer.PixelsSinkWriter; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class TransactionProcessorTest +{ + @Test + void shouldForwardTransactionsAndStop() throws Exception + { + BlockingBoundedQueue queue = + new BlockingBoundedQueue<>(1); + RecordingWriter writer = new RecordingWriter(); + TransactionProcessor processor = new TransactionProcessor(queue, writer); + Thread processorThread = new Thread(processor, "transaction-processor-test"); + SinkProto.TransactionMetadata transaction = SinkProto.TransactionMetadata.newBuilder() + .setId("transaction-1") + .build(); + + try + { + processorThread.start(); + queue.put(transaction); + + assertTrue(writer.transactionWritten.await(1, TimeUnit.SECONDS)); + assertSame(transaction, writer.transaction); + } finally + { + processor.stopProcessor(); + queue.close(); + processorThread.interrupt(); + processorThread.join(1000); + } + assertTrue(!processorThread.isAlive()); + } + + private static final class RecordingWriter implements PixelsSinkWriter + { + private final CountDownLatch transactionWritten = new CountDownLatch(1); + private SinkProto.TransactionMetadata transaction; + + @Override + public void flush() + { + } + + @Override + public boolean writeRow(io.pixelsdb.pixels.sink.event.RowChangeEvent rowChangeEvent) + { + return false; + } + + @Override + public boolean writeTrans(SinkProto.TransactionMetadata transactionMetadata) + { + transaction = transactionMetadata; + transactionWritten.countDown(); + return true; + } + + @Override + public void close() throws IOException + { + } + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/provider/BlockingEventProviderTest.java b/src/test/java/io/pixelsdb/pixels/sink/provider/BlockingEventProviderTest.java deleted file mode 100644 index c00b721..0000000 --- a/src/test/java/io/pixelsdb/pixels/sink/provider/BlockingEventProviderTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.provider; - -import org.junit.jupiter.api.Test; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; - -class BlockingEventProviderTest -{ - @Test - void shouldPublishAndTakeCanonicalEvent() - { - BlockingEventProvider provider = new BlockingEventProvider<>(); - provider.publish(7); - - assertEquals(7, provider.take()); - provider.close(); - } - - @Test - void shouldWakeConsumerWhenClosed() - throws Exception - { - BlockingEventProvider provider = new BlockingEventProvider<>(); - CountDownLatch finished = new CountDownLatch(1); - Thread consumer = new Thread(() -> - { - assertNull(provider.take()); - finished.countDown(); - }); - consumer.start(); - - provider.close(); - - assertEquals(true, finished.await(1, TimeUnit.SECONDS)); - consumer.join(); - } -} diff --git a/src/test/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueueTest.java b/src/test/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueueTest.java new file mode 100644 index 0000000..162d1b5 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueueTest.java @@ -0,0 +1,83 @@ +/* + * Copyright 2025 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the license, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.util; + +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class BlockingBoundedQueueTest +{ + @Test + void shouldRejectNonPositiveCapacity() + { + assertThrows(IllegalArgumentException.class, () -> new BlockingBoundedQueue<>(0)); + assertThrows(IllegalArgumentException.class, () -> new BlockingBoundedQueue<>(-1)); + } + + @Test + void shouldPutAndTakeValue() + { + BlockingBoundedQueue queue = new BlockingBoundedQueue<>(1); + queue.put(7); + + assertEquals(7, queue.take()); + queue.close(); + } + + @Test + void shouldWakeConsumerWhenClosed() + throws Exception + { + BlockingBoundedQueue queue = new BlockingBoundedQueue<>(1); + CountDownLatch finished = new CountDownLatch(1); + Thread consumer = new Thread(() -> + { + assertNull(queue.take()); + finished.countDown(); + }); + consumer.start(); + + queue.close(); + + assertTrue(finished.await(1, TimeUnit.SECONDS)); + consumer.join(); + } + + @Test + void shouldDiscardPendingValuesWhenClosed() + { + BlockingBoundedQueue queue = new BlockingBoundedQueue<>(1); + queue.put(7); + queue.put(null); + + queue.close(); + + assertNull(queue.take()); + queue.close(); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistryTest.java b/src/test/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistryTest.java index e38c507..1d65203 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistryTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistryTest.java @@ -20,11 +20,16 @@ import io.pixelsdb.pixels.common.utils.EtcdUtil; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.nio.file.Path; import java.util.List; +import java.util.UUID; + +import org.junit.jupiter.api.io.TempDir; + +import static org.junit.jupiter.api.Assertions.assertEquals; /** * @package: io.pixelsdb.pixels.sink.util @@ -32,24 +37,28 @@ * @author: AntiO2 * @date: 2025/10/5 08:54 */ -public class EtcdFileRegistryTest +@Tag("integration") +class EtcdFileRegistryTest { - private static final Logger LOGGER = LoggerFactory.getLogger(EtcdFileRegistryTest.class); - @Test - public void testCreateFile() + void shouldCreateAndListCompletedFiles(@TempDir Path tempDir) { - EtcdFileRegistry etcdFileRegistry = new EtcdFileRegistry("test", "file:///tmp/test/ray"); - for (int i = 0; i < 10; i++) + String topic = "test-" + UUID.randomUUID(); + try { - String newFile = etcdFileRegistry.createNewFile(); - etcdFileRegistry.markFileCompleted(newFile); - } - List files = etcdFileRegistry.listAllFiles(); - for (String file : files) + EtcdFileRegistry etcdFileRegistry = new EtcdFileRegistry( + topic, tempDir.resolve("ray").toUri().toString()); + for (int i = 0; i < 10; i++) + { + String newFile = etcdFileRegistry.createNewFile(); + etcdFileRegistry.markFileCompleted(newFile); + } + List files = etcdFileRegistry.listAllFiles(); + + assertEquals(10, files.size()); + } finally { - LOGGER.info(file); + EtcdUtil.Instance().deleteByPrefix("/sink/proto/registry/" + topic); } - EtcdUtil.Instance().deleteByPrefix("/sink/proto/registry/test"); } } diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/RpcEndToEndTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/RpcEndToEndTest.java index 5fd1764..33da52e 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/RpcEndToEndTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/RpcEndToEndTest.java @@ -40,12 +40,9 @@ public class RpcEndToEndTest { - private static final int TEST_PORT = 9091; - private static final int RECORD_COUNT = 5; private static final String TEST_SCHEMA = "pixels_bench_sf10x"; private static final String TEST_TABLE = "savingaccount"; private static final String FULL_TABLE_NAME = TEST_SCHEMA + "." + TEST_TABLE; - private static final String CONFIG_FILE_PATH = "conf/pixels-sink.aws.properties"; private static SinkSource sinkSource; // Keep a reference to stop it later private static HTTPServer prometheusHttpServer; @@ -56,14 +53,16 @@ private static void startServer() try { // === 1. Mimic init() method from PixelsSinkApp === - System.out.println("[SETUP] Initializing configuration from " + CONFIG_FILE_PATH + "..."); - PixelsSinkConfigFactory.initialize(CONFIG_FILE_PATH); + String configPath = requiredProperty("pixels.sink.integration.config"); + int testPort = configuredPort(); + System.out.println("[SETUP] Initializing configuration from " + configPath + "..."); + PixelsSinkConfigFactory.initialize(configPath); System.out.println("[SETUP] Initializing MetricsFacade..."); MetricsFacade.getInstance().setSinkContextManager(SinkContextManager.getInstance()); // For determinism in testing, override the port from the config file - System.setProperty("sink.flink.server.port", String.valueOf(TEST_PORT)); + System.setProperty("sink.flink.server.port", String.valueOf(testPort)); // === 2. Mimic main() method from PixelsSinkApp === PixelsSinkConfig config = PixelsSinkConfigFactory.getInstance(); System.out.println("[SETUP] Creating SinkSource application engine..."); @@ -99,7 +98,7 @@ public static void main(String[] args) throws InterruptedException, IOException // [REFACTORED] To ensure the FlinkPollingWriter uses our test port, // we set it as a system property before the writer is created. // The PixelsSinkConfigFactory should be configured to read this property. - System.setProperty("sink.flink.server.port", String.valueOf(TEST_PORT)); + System.setProperty("sink.flink.server.port", String.valueOf(configuredPort())); // [REFACTORED] The setup is now dramatically simpler. // Instantiating FlinkPollingWriter is the ONLY step needed. @@ -166,7 +165,8 @@ public static void main(String[] args) throws InterruptedException, IOException executor.submit(() -> { System.out.println("[CLIENT] Starting gRPC client..."); - ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", TEST_PORT) + ManagedChannel channel = ManagedChannelBuilder + .forAddress(requiredProperty("pixels.sink.test.host"), configuredPort()) .usePlaintext() .build(); @@ -241,4 +241,19 @@ public static void main(String[] args) throws InterruptedException, IOException } System.out.println("[CLEANUP] Test finished."); } + + private static int configuredPort() + { + return Integer.parseInt(requiredProperty("pixels.sink.test.port")); + } + + private static String requiredProperty(String key) + { + String value = System.getProperty(key); + if (value == null || value.isBlank()) + { + throw new IllegalStateException("Set -D" + key + " to run this integration test"); + } + return value; + } } diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/TestProtoWriter.java b/src/test/java/io/pixelsdb/pixels/sink/writer/TestProtoWriter.java index 0c932a3..dec41bc 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/TestProtoWriter.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/TestProtoWriter.java @@ -22,17 +22,22 @@ import com.google.protobuf.ByteString; import io.pixelsdb.pixels.common.physical.*; import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.TestConfig; import io.pixelsdb.pixels.sink.writer.proto.ProtoWriter; import io.pixelsdb.pixels.storage.localfs.PhysicalLocalReader; -import lombok.SneakyThrows; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * @package: io.pixelsdb.pixels.sink.writer @@ -40,17 +45,10 @@ * @author: AntiO2 * @date: 2025/10/5 09:24 */ -public class TestProtoWriter +class TestProtoWriter { - public static String schemaName = "test"; - public static String tableName = "ray"; - - @BeforeAll - public static void setUp() throws IOException - { - PixelsSinkConfigFactory.initialize("/home/pixels/projects/pixels-writer/src/main/resources/pixels-writer.local.properties"); -// PixelsSinkConfigFactory.initialize("/home/ubuntu/pixels-writer/src/main/resources/pixels-writer.aws.properties"); - } + private static final String SCHEMA_NAME = "test"; + private static final String TABLE_NAME = "ray"; private static SinkProto.RowRecord getRowRecord(int i) { @@ -74,8 +72,8 @@ private static SinkProto.RowRecord getRowRecord(int i) .setAfter(afterValueBuilder) .setSource( SinkProto.SourceInfo.newBuilder() - .setDb(schemaName) - .setTable(tableName) + .setDb(SCHEMA_NAME) + .setTable(TABLE_NAME) .build() ); return builder.build(); @@ -90,77 +88,96 @@ private static SinkProto.TransactionMetadata getTrans(int i, SinkProto.Transacti return builder.build(); } - @SneakyThrows + @Tag("integration") @Test - public void testWriteTransInfo() + void testWriteTransInfo() throws Exception { - ProtoWriter transWriter = new ProtoWriter(); - int maxTx = 1000; - - for (int i = 0; i < maxTx; i++) + TestConfig.initializeIntegrationConfig(); + int maxTx = Integer.getInteger("pixels.sink.test.transactions", 10); + try (ProtoWriter transWriter = new ProtoWriter()) { - transWriter.writeTrans(getTrans(i, SinkProto.TransactionStatus.BEGIN)); - transWriter.writeTrans(getTrans(i, SinkProto.TransactionStatus.END)); + for (int i = 0; i < maxTx; i++) + { + transWriter.writeTrans(getTrans(i, SinkProto.TransactionStatus.BEGIN)); + transWriter.writeTrans(getTrans(i, SinkProto.TransactionStatus.END)); + } } - transWriter.close(); } @Test - public void testWriteFile() throws IOException + void testWriteFile(@TempDir Path tempDir) throws IOException { - String path = "/home/pixels/projects/pixels-writer/tmp/write.dat"; - PhysicalWriter writer = PhysicalWriterUtil.newPhysicalWriter(Storage.Scheme.file, path); - - int writeNum = 3; - - ByteBuffer buf = ByteBuffer.allocate(writeNum * Integer.BYTES); - for (int i = 0; i < 3; i++) + Path path = tempDir.resolve("write.dat"); + try (PhysicalWriter writer = PhysicalWriterUtil.newPhysicalWriter( + Storage.Scheme.file, path.toString())) { - buf.putInt(i); + int writeNum = 3; + ByteBuffer buf = ByteBuffer.allocate(writeNum * Integer.BYTES); + for (int i = 0; i < writeNum; i++) + { + buf.putInt(i); + } + assertEquals(0, writer.append(buf)); } - writer.append(buf); - writer.close(); + + assertEquals(3L * Integer.BYTES, Files.size(path)); } @Test - public void testReadFile() throws IOException + void testReadFile(@TempDir Path tempDir) throws IOException { - String path = "/home/pixels/projects/pixels-writer/tmp/write.dat"; - PhysicalLocalReader reader = (PhysicalLocalReader) PhysicalReaderUtil.newPhysicalReader(Storage.Scheme.file, path); + Path path = tempDir.resolve("write.dat"); + try (PhysicalWriter writer = PhysicalWriterUtil.newPhysicalWriter( + Storage.Scheme.file, path.toString())) + { + ByteBuffer buffer = ByteBuffer.allocate(3 * Long.BYTES); + buffer.putLong(11L).putLong(22L).putLong(33L); + writer.append(buffer); + } - int writeNum = 12; - for (int i = 0; i < writeNum; i++) + try (PhysicalLocalReader reader = (PhysicalLocalReader) PhysicalReaderUtil + .newPhysicalReader(Storage.Scheme.file, path.toString())) { - reader.readLong(ByteOrder.BIG_ENDIAN); + assertEquals(3L * Long.BYTES, reader.getFileLength()); + assertEquals(11L, reader.readLong(ByteOrder.BIG_ENDIAN)); + assertEquals(22L, reader.readLong(ByteOrder.BIG_ENDIAN)); + assertEquals(33L, reader.readLong(ByteOrder.BIG_ENDIAN)); } } @Test - public void testReadEmptyFile() throws IOException + void testReadEmptyFile(@TempDir Path tempDir) throws IOException { - String path = "/home/pixels/projects/pixels-writer/tmp/empty.dat"; - PhysicalReader reader = PhysicalReaderUtil.newPhysicalReader(Storage.Scheme.file, path); - - int v = reader.readInt(ByteOrder.BIG_ENDIAN); + Path path = tempDir.resolve("empty.dat"); + Files.createFile(path); - return; + try (PhysicalReader reader = PhysicalReaderUtil.newPhysicalReader( + Storage.Scheme.file, path.toString())) + { + assertEquals(0, reader.getFileLength()); + assertThrows(IOException.class, () -> reader.readInt(ByteOrder.BIG_ENDIAN)); + } } + @Tag("integration") @Test - public void testWriteRowInfo() throws IOException + void testWriteRowInfo() throws Exception { - ProtoWriter transWriter = new ProtoWriter(); - int maxTx = 10000000; + TestConfig.initializeIntegrationConfig(); + int maxTx = Integer.getInteger("pixels.sink.test.transactions", 10); int rowCnt = 0; - for (int i = 0; i < maxTx; i++) + try (ProtoWriter transWriter = new ProtoWriter()) { - transWriter.writeTrans(getTrans(i, SinkProto.TransactionStatus.BEGIN)); - for (int j = i; j < 3; j++) + for (int i = 0; i < maxTx; i++) { - transWriter.write(getRowRecord(rowCnt++)); + transWriter.writeTrans(getTrans(i, SinkProto.TransactionStatus.BEGIN)); + for (int j = 0; j < 3; j++) + { + transWriter.write(getRowRecord(rowCnt++)); + } + transWriter.writeTrans(getTrans(i, SinkProto.TransactionStatus.END)); } - transWriter.writeTrans(getTrans(i, SinkProto.TransactionStatus.END)); } - transWriter.close(); + assertEquals(maxTx * 3, rowCnt); } } diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/TestRetinaWriter.java b/src/test/java/io/pixelsdb/pixels/sink/writer/TestRetinaWriter.java index d21f0a3..0087f08 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/TestRetinaWriter.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/TestRetinaWriter.java @@ -29,7 +29,7 @@ import io.pixelsdb.pixels.index.IndexProto; import io.pixelsdb.pixels.retina.RetinaProto; import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.TestConfig; import io.pixelsdb.pixels.sink.event.RowChangeEvent; import io.pixelsdb.pixels.sink.exception.SinkException; import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; @@ -38,6 +38,7 @@ import io.pixelsdb.pixels.sink.writer.retina.TransactionProxy; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,7 +54,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -public class TestRetinaWriter +@Tag("integration") +class TestRetinaWriter { static Logger logger = LoggerFactory.getLogger(TestRetinaWriter.class.getName()); @@ -65,15 +67,16 @@ public class TestRetinaWriter private final ExecutorService executor = Executors.newFixedThreadPool(16); @BeforeAll - public static void setUp() throws IOException + static void setUp() throws Exception { - PixelsSinkConfigFactory.initialize("/home/pixels/projects/pixels-sink/src/main/resources/pixels-sink.local.properties"); -// PixelsSinkConfigFactory.initialize("/home/ubuntu/pixels-sink/src/main/resources/pixels-sink.aws.properties"); + TestConfig.initializeIntegrationConfig(); retinaService = RetinaService.Instance(); metadataRegistry = TableMetadataRegistry.Instance(); transService = TransService.Instance(); - retinaPerformanceTestRowCount = 5_000_000; - retinaPerformanceTestMaxId = 2_000_000; + retinaPerformanceTestRowCount = Integer.getInteger( + "pixels.sink.test.retina.row.count", 1_000); + retinaPerformanceTestMaxId = Integer.getInteger( + "pixels.sink.test.retina.max.id", 2_000); } @Test @@ -305,8 +308,7 @@ public void testCheckingAccountInsertPerformance() throws Assertions.assertNotNull(writer); if (!writer.writeTrans(schemaName, tableUpdateData)) { - logger.error("Error Write Trans"); - System.exit(-1); + throw new AssertionError("Failed to write transaction"); } } @@ -425,8 +427,7 @@ public void testCheckingAccountUpdatePerformance() throws long startTime = System.currentTimeMillis(); if (!writer.writeTrans(schemaName, tableUpdateData)) { - logger.error("Error Write Trans"); - System.exit(-1); + throw new AssertionError("Failed to write transaction"); } long endTime = System.currentTimeMillis(); logger.debug("writeTrans batch " + batchIndex + " took " + (endTime - startTime) + " ms"); diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/TpcHTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/TpcHTest.java index a3edbca..7fd77c1 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/TpcHTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/TpcHTest.java @@ -29,12 +29,14 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.nio.charset.StandardCharsets; import java.util.logging.Logger; -public class TpcHTest +@Tag("integration") +class TpcHTest { static Logger logger = Logger.getLogger(TpcHTest.class.getName()); diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/retina/TableWriterProxyTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/TableWriterProxyTest.java index fa4e996..bdb78a0 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/retina/TableWriterProxyTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/TableWriterProxyTest.java @@ -20,35 +20,38 @@ package io.pixelsdb.pixels.sink.writer.retina; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.TestConfig; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; -public class TableWriterProxyTest -{ - private static final Logger LOGGER = LoggerFactory.getLogger(TableWriterProxyTest.class); - +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; - String tableName = "test"; +@Tag("integration") +class TableWriterProxyTest +{ + private static final String TABLE_NAME = "test"; @BeforeAll - public static void init() throws IOException + static void init() throws Exception { - PixelsSinkConfigFactory.initialize("/home/ubuntu/pixels-sink/conf/pixels-sink.aws.properties"); + TestConfig.initializeIntegrationConfig(); } @Test - public void testGetSameTableWriter() throws IOException + void shouldReuseTableWriter() throws IOException { TableWriterProxy tableWriterProxy = TableWriterProxy.getInstance(); + TableWriter first = tableWriterProxy.getTableWriter(TABLE_NAME, 0, 0); for (int i = 0; i < 10; i++) { - TableWriter tableWriter = tableWriterProxy.getTableWriter(tableName, 0, 0); + TableWriter tableWriter = tableWriterProxy.getTableWriter(TABLE_NAME, 0, 0); + assertNotNull(tableWriter); + assertSame(first, tableWriter); } } } diff --git a/src/test/resources/pixels-sink-test.properties b/src/test/resources/pixels-sink-test.properties index b610781..54cff79 100644 --- a/src/test/resources/pixels-sink-test.properties +++ b/src/test/resources/pixels-sink-test.properties @@ -1,3 +1,4 @@ sink.monitor.enable=false sink.monitor.report.enable=false +sink.mode=none debezium.connector.class=io.debezium.connector.mysql.MySqlConnector From d0bff509eac884453c6b909cadbbd31b9f0116de Mon Sep 17 00:00:00 2001 From: dannygeng Date: Wed, 29 Jul 2026 17:44:09 +0800 Subject: [PATCH 03/13] fix: Unify logging on Log4j2 and eliminate competing bindings --- conf/jvm.conf | 2 + pom.xml | 63 ++++++++++++++++++- .../RowChangeEventStructConverter.java | 10 +-- .../writer/retina/SinkContextManager.java | 1 - src/main/resources/log4j2.properties | 35 +++++------ src/main/resources/logging.properties | 3 - .../pixelsdb/pixels/sink/writer/TpcHTest.java | 5 +- src/test/resources/log4j2.properties | 27 -------- 8 files changed, 85 insertions(+), 61 deletions(-) delete mode 100644 src/main/resources/logging.properties delete mode 100644 src/test/resources/log4j2.properties diff --git a/conf/jvm.conf b/conf/jvm.conf index fcc3742..86d3aa6 100644 --- a/conf/jvm.conf +++ b/conf/jvm.conf @@ -3,6 +3,8 @@ -Dfile.encoding=UTF-8 -Duser.timezone=UTC +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager + -Xms8g -Xmx60g diff --git a/pom.xml b/pom.xml index 4d8574f..99696c5 100644 --- a/pom.xml +++ b/pom.xml @@ -83,6 +83,36 @@ jdk.tools jdk.tools + + + org.apache.logging.log4j + log4j-web + + + org.apache.logging.log4j + log4j-1.2-api + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-reload4j + + + ch.qos.reload4j + reload4j + + + + org.apache.hive + hive-llap-server + @@ -119,6 +149,18 @@ log4j-slf4j-impl ${dep.log4j.version} + + + org.slf4j + log4j-over-slf4j + + + + org.apache.logging.log4j + log4j-jul + ${dep.log4j.version} + @@ -195,6 +237,13 @@ io.debezium debezium-embedded ${dep.debezium.version} + + + + ch.qos.reload4j + reload4j + + io.debezium @@ -298,13 +347,14 @@ false integration --add-opens=java.base/java.nio=ALL-UNNAMED - --add-exports=java.base/sun.nio.ch=ALL-UNNAMED + --add-exports=java.base/sun.nio.ch=ALL-UNNAMED + -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager org.apache.maven.plugins maven-shade-plugin - 3.2.0 + ${maven.plugin.shade.version} package @@ -314,6 +364,7 @@ pixels-sink ${project.basedir}/target + false true full @@ -332,6 +383,11 @@ ${mainClass} + + + true + @@ -384,7 +440,8 @@ false --add-opens=java.base/java.nio=ALL-UNNAMED - --add-exports=java.base/sun.nio.ch=ALL-UNNAMED + --add-exports=java.base/sun.nio.ch=ALL-UNNAMED + -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventStructConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventStructConverter.java index 5ef8e1f..0cd5939 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventStructConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventStructConverter.java @@ -31,8 +31,8 @@ import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.errors.DataException; import org.apache.kafka.connect.source.SourceRecord; - -import java.util.logging.Logger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @package: io.pixelsdb.pixels.sink.conversion.debezium @@ -42,7 +42,7 @@ */ public class RowChangeEventStructConverter { - private static final Logger LOGGER = Logger.getLogger(RowChangeEventStructConverter.class.getName()); + private static final Logger LOGGER = LoggerFactory.getLogger(RowChangeEventStructConverter.class); private final TableMetadataRegistry tableMetadataRegistry; private final DebeziumSourceAdapter configuredAdapter; @@ -103,7 +103,7 @@ private RowChangeEvent buildRowRecord(Struct value, tableName = sourceInfo.getTable(); } catch (DataException | IllegalArgumentException e) { - LOGGER.warning("Missing source field in row record"); + LOGGER.warn("Missing source field in row record"); throw new SinkException(e); } @@ -121,7 +121,7 @@ private RowChangeEvent buildRowRecord(Struct value, } } catch (DataException e) { - LOGGER.warning("Missing transaction field in row record"); + LOGGER.warn("Missing transaction field in row record"); } SinkProto.RowValue beforeValue = null; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/SinkContextManager.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/SinkContextManager.java index e74829e..491afb2 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/SinkContextManager.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/SinkContextManager.java @@ -42,7 +42,6 @@ public class SinkContextManager { private static final Logger LOGGER = LoggerFactory.getLogger(SinkContextManager.class); - private static final Logger BUCKET_TRACE_LOGGER = LoggerFactory.getLogger("bucket_trace"); private static volatile SinkContextManager instance; private final BlockingBoundedMap activeTxContexts; // private final ConcurrentMap activeTxContexts = new ConcurrentHashMap<>(10000); diff --git a/src/main/resources/log4j2.properties b/src/main/resources/log4j2.properties index e1fe75b..13d59c3 100644 --- a/src/main/resources/log4j2.properties +++ b/src/main/resources/log4j2.properties @@ -1,4 +1,4 @@ -status=info +status=warn name=pixels-sink filter.threshold.type=ThresholdFilter filter.threshold.level=info @@ -6,36 +6,31 @@ appender.console.type=Console appender.console.name=STDOUT appender.console.layout.type=PatternLayout appender.console.layout.pattern=%-d{yyyy-MM-dd HH:mm:ss,SSS} [%c]-[%p] %m%n -appender.rolling.type=File +appender.rolling.type=RollingFile appender.rolling.name=log appender.rolling.append=true appender.rolling.fileName=${env:PIXELS_HOME}/logs/pixels-sink.log +appender.rolling.filePattern=${env:PIXELS_HOME}/logs/pixels-sink-%d{yyyy-MM-dd}-%i.log.gz appender.rolling.layout.type=PatternLayout appender.rolling.layout.pattern=%-d{yyyy-MM-dd HH:mm:ss,SSS} [%c]-[%p] %m%n +appender.rolling.policies.type=Policies +appender.rolling.policies.time.type=TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval=1 +appender.rolling.policies.time.modulate=true +appender.rolling.policies.size.type=SizeBasedTriggeringPolicy +appender.rolling.policies.size.size=100MB +appender.rolling.strategy.type=DefaultRolloverStrategy +appender.rolling.strategy.max=10 rootLogger.level=info rootLogger.appenderRef.stdout.ref=STDOUT rootLogger.appenderRef.log.ref=log -logger.transaction.name=io.pixelsdb.pixels.sink.sink.retina.RetinaWriter -logger.transaction.level=info -logger.transaction.appenderRef.log.ref=log -logger.transaction.appenderRef.stdout.ref=STDOUT -logger.transaction.additivity=false logger.grpc.name=io.grpc.netty.shaded.io.grpc.netty.NettyClientHandler -logger.grpc.level=info +logger.grpc.level=warn logger.grpc.additivity=false logger.grpc.appenderRef.log.ref=log logger.grpc.appenderRef.stdout.ref=STDOUT -log4j2.logger.io.grpc.netty.shaded.io.grpc.netty.NettyClientHandler=OFF logger.netty-shaded.name=io.grpc.netty.shaded.io.netty -logger.netty-shaded.level=info +logger.netty-shaded.level=warn logger.netty-shaded.additivity=false -appender.bucket_trace.type=File -appender.bucket_trace.name=BUCKET_TRACE -appender.bucket_trace.append=true -appender.bucket_trace.fileName=${env:PIXELS_HOME}/logs/bucket_trace.log -appender.bucket_trace.layout.type=PatternLayout -appender.bucket_trace.layout.pattern=%m%n -logger.bucket_trace.name=bucket_trace -logger.bucket_trace.level=warn -logger.bucket_trace.additivity=false -logger.bucket_trace.appenderRef.bucket_trace.ref=BUCKET_TRACE +logger.netty-shaded.appenderRef.log.ref=log +logger.netty-shaded.appenderRef.stdout.ref=STDOUT diff --git a/src/main/resources/logging.properties b/src/main/resources/logging.properties deleted file mode 100644 index 2c1fd24..0000000 --- a/src/main/resources/logging.properties +++ /dev/null @@ -1,3 +0,0 @@ -.level=INFO -io.grpc.level=INFO -io.grpc.netty.level=INFO diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/TpcHTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/TpcHTest.java index 7fd77c1..6b1b220 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/TpcHTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/TpcHTest.java @@ -31,15 +31,16 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.nio.charset.StandardCharsets; -import java.util.logging.Logger; @Tag("integration") class TpcHTest { - static Logger logger = Logger.getLogger(TpcHTest.class.getName()); + static Logger logger = LoggerFactory.getLogger(TpcHTest.class); static RetinaService retinaService; static MetadataService metadataService; static TransService transService; diff --git a/src/test/resources/log4j2.properties b/src/test/resources/log4j2.properties deleted file mode 100644 index 09ab7d9..0000000 --- a/src/test/resources/log4j2.properties +++ /dev/null @@ -1,27 +0,0 @@ -status=info -name=pixels-sink -filter.threshold.type=ThresholdFilter -filter.threshold.level=info -appender.console.type=Console -appender.console.name=STDOUT -appender.console.layout.type=PatternLayout -appender.console.layout.pattern=%-d{yyyy-MM-dd HH:mm:ss,SSS} [%c]-[%p] %m%n -appender.rolling.type=File -appender.rolling.name=log -appender.rolling.append=true -appender.rolling.fileName=${env:PIXELS_HOME}/logs/pixels-sink.log -appender.rolling.layout.type=PatternLayout -appender.rolling.layout.pattern=%-d{yyyy-MM-dd HH:mm:ss,SSS} [%c]-[%p] %m%n -rootLogger.level=info -rootLogger.appenderRef.stdout.ref=STDOUT -rootLogger.appenderRef.log.ref=log -logger.transaction.name=io.pixelsdb.pixels.sink.sink.retina.RetinaWriter -logger.transaction.level=info -logger.transaction.appenderRef.log.ref=log -logger.transaction.appenderRef.stdout.ref=STDOUT -logger.transaction.additivity=false -logger.grpc.name=io.grpc.netty.shaded.io.grpc.netty.NettyClientHandler -logger.grpc.level=info -logger.grpc.additivity=false -logger.grpc.appenderRef.log.ref=log -logger.grpc.appenderRef.stdout.ref=STDOUT From aa78cf9286fb4a2d8cfb36863453900424d46a8b Mon Sep 17 00:00:00 2001 From: dannygeng Date: Thu, 30 Jul 2026 14:49:14 +0800 Subject: [PATCH 04/13] feat: Add debug logs for Retina batch flush send and ack --- .../writer/retina/TableCrossTxWriter.java | 24 ++++++++++++++++--- src/main/resources/log4j2.properties | 15 +++++++++++- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableCrossTxWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableCrossTxWriter.java index d720542..040966c 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableCrossTxWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableCrossTxWriter.java @@ -112,16 +112,33 @@ public void flush(List batch) tableUpdateData.add(tableUpdateDataItem.build()); } + int rowCount = batch.size(); inFlightControlManager.acquire(1); + LOGGER.debug("Sending {} rows of table {} to retina, txIds={}", rowCount, tableName, txIds); CompletableFuture updateRecordResponseCompletableFuture = delegate.writeBatchAsync(batch.get(0).getSchemaName(), tableUpdateData); + if (updateRecordResponseCompletableFuture == null) + { + inFlightControlManager.release(1); + LOGGER.error("Failed to submit {} rows of table {} to retina, txIds={}", rowCount, tableName, txIds); + failCtxs(txIds); + return; + } - updateRecordResponseCompletableFuture.thenAccept( - resp -> + updateRecordResponseCompletableFuture.whenComplete( + (resp, err) -> { inFlightControlManager.release(1); - if (resp.getHeader().getErrorCode() != 0) + if (err != null) + { + LOGGER.error("Retina write failed for {} rows of table {}, txIds={}", + rowCount, tableName, txIds, err); + failCtxs(txIds); + } else if (resp.getHeader().getErrorCode() != 0) { + LOGGER.error("Retina rejected {} rows of table {}, txIds={}, errorCode={}, errorMsg={}", + rowCount, tableName, txIds, resp.getHeader().getErrorCode(), + resp.getHeader().getErrorMsg()); failCtxs(txIds); } else { @@ -131,6 +148,7 @@ public void flush(List batch) metricsFacade.recordFreshness(txEndTime - txStartTime); } updateCtxCounters(txIds, fullTableName, tableUpdateCount); + LOGGER.debug("Retina acked {} rows of table {}, txIds={}", rowCount, tableName, txIds); } } ); diff --git a/src/main/resources/log4j2.properties b/src/main/resources/log4j2.properties index 13d59c3..77fbf7b 100644 --- a/src/main/resources/log4j2.properties +++ b/src/main/resources/log4j2.properties @@ -1,7 +1,8 @@ status=warn name=pixels-sink filter.threshold.type=ThresholdFilter -filter.threshold.level=info +# allow DEBUG for selected loggers (e.g. DebeziumConsumer receive path) +filter.threshold.level=debug appender.console.type=Console appender.console.name=STDOUT appender.console.layout.type=PatternLayout @@ -24,6 +25,18 @@ appender.rolling.strategy.max=10 rootLogger.level=info rootLogger.appenderRef.stdout.ref=STDOUT rootLogger.appenderRef.log.ref=log +# CDC receive: topic/offset/tx/gtid/binlog pos +logger.debeziumConsumer.name=io.pixelsdb.pixels.sink.source.engine.PixelsDebeziumConsumer +logger.debeziumConsumer.level=debug +logger.debeziumConsumer.additivity=false +logger.debeziumConsumer.appenderRef.stdout.ref=STDOUT +logger.debeziumConsumer.appenderRef.log.ref=log +# Retina batch flush: send / ack (DEBUG), table/txIds/row count +logger.tableCrossFlush.name=io.pixelsdb.pixels.sink.writer.retina.TableCrossTxWriter +logger.tableCrossFlush.level=debug +logger.tableCrossFlush.additivity=false +logger.tableCrossFlush.appenderRef.stdout.ref=STDOUT +logger.tableCrossFlush.appenderRef.log.ref=log logger.grpc.name=io.grpc.netty.shaded.io.grpc.netty.NettyClientHandler logger.grpc.level=warn logger.grpc.additivity=false From 963f8363135ec548f8d90400021cf68c9330af03 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Thu, 30 Jul 2026 16:13:28 +0800 Subject: [PATCH 05/13] fix: Represent null Debezium columns as empty bytes instead of isNull --- .../conversion/debezium/DebeziumRowValueConverter.java | 6 ++---- .../debezium/DebeziumEnvelopeNormalizerTest.java | 7 +++---- .../conversion/debezium/DebeziumRowValueConverterTest.java | 4 ---- .../sink/conversion/sinkproto/RowRecordConverterTest.java | 3 ++- .../pixelsdb/pixels/sink/processor/TableProcessorTest.java | 3 ++- 5 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverter.java index cf8928b..26a4c78 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverter.java @@ -101,8 +101,7 @@ private SinkProto.ColumnValue.Builder parseValue(JsonNode valueNode, String fiel { return SinkProto.ColumnValue.newBuilder() // .setName(fieldName) - .setValue(ByteString.EMPTY) - .setIsNull(true); + .setValue(ByteString.EMPTY); } SinkProto.ColumnValue.Builder columnValueBuilder = SinkProto.ColumnValue.newBuilder(); @@ -218,8 +217,7 @@ private SinkProto.ColumnValue.Builder parseCanonicalValue(Object raw, TypeDescri if (raw == null) { return SinkProto.ColumnValue.newBuilder() - .setValue(ByteString.EMPTY) - .setIsNull(true); + .setValue(ByteString.EMPTY); } if (raw instanceof JsonNode jsonNode) { diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizerTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizerTest.java index 2a4b5c8..bb02149 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizerTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizerTest.java @@ -209,17 +209,16 @@ void shouldRoundTripCanonicalProto() throws Exception .setDataCollectionOrder(1)) .setAfter(SinkProto.RowValue.newBuilder() .addValues(SinkProto.ColumnValue.newBuilder() - .setValue(com.google.protobuf.ByteString.copyFromUtf8("")) - .setIsNull(false)) + .setValue(com.google.protobuf.ByteString.copyFromUtf8(""))) .addValues(SinkProto.ColumnValue.newBuilder() - .setIsNull(true))) + .setValue(com.google.protobuf.ByteString.EMPTY))) .build(); SinkProto.RowRecord restored = SinkProto.RowRecord.parseFrom(record.toByteArray()); assertEquals(record, restored); - assertTrue(restored.getAfter().getValues(1).getIsNull()); + assertEquals(0, restored.getAfter().getValues(1).getValue().size()); } private Struct mysqlSource() diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverterTest.java index 1a31623..e078f7f 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverterTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverterTest.java @@ -36,8 +36,6 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; public class DebeziumRowValueConverterTest { @@ -86,9 +84,7 @@ void shouldEncodeCanonicalStructValues() throws Exception ByteBuffer.wrap(value.getValues(0).getValue().toByteArray()).getLong()); assertEquals("TDSQL value ", value.getValues(1).getValue().toStringUtf8()); assertEquals("24710.35", value.getValues(2).getValue().toStringUtf8()); - assertTrue(value.getValues(3).getIsNull()); assertEquals(0, value.getValues(3).getValue().size()); - assertFalse(value.getValues(4).getIsNull()); assertEquals("", value.getValues(4).getValue().toStringUtf8()); ObjectNode jsonRow = new ObjectMapper().createObjectNode(); diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverterTest.java index d58f2b0..2dc280e 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverterTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverterTest.java @@ -77,7 +77,8 @@ void shouldConvertCanonicalRowRecordIntoRuntimeEvent() .setDb(TABLE.getSchemaName()) .setTable(TABLE.getTableName())) .setAfter(SinkProto.RowValue.newBuilder() - .addValues(SinkProto.ColumnValue.newBuilder().setIsNull(true)) + .addValues(SinkProto.ColumnValue.newBuilder() + .setValue(com.google.protobuf.ByteString.EMPTY)) .build()) .build(); diff --git a/src/test/java/io/pixelsdb/pixels/sink/processor/TableProcessorTest.java b/src/test/java/io/pixelsdb/pixels/sink/processor/TableProcessorTest.java index ec2b778..3ff6ab3 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/processor/TableProcessorTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/processor/TableProcessorTest.java @@ -69,7 +69,8 @@ private static RowChangeEvent rowEvent() throws Exception .setDb("test_db") .setTable("test_table")) .setAfter(SinkProto.RowValue.newBuilder() - .addValues(SinkProto.ColumnValue.newBuilder().setIsNull(true))) + .addValues(SinkProto.ColumnValue.newBuilder() + .setValue(com.google.protobuf.ByteString.EMPTY))) .build(); TypeDescription schema = TypeDescription.createSchemaFromStrings( List.of("id"), List.of("int")); From 846166b594fab5e39fc0e504f560975da69abb16 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Fri, 31 Jul 2026 10:20:13 +0800 Subject: [PATCH 06/13] refactor: rename freshness config to sink.query.* and make Hive JDBC optional --- conf/pixels-sink.aws.properties | 10 +- conf/pixels-sink.ch.properties | 8 +- conf/pixels-sink.flink.properties.chbench | 8 +- conf/pixels-sink.flink.properties.hybench | 8 +- conf/pixels-sink.hudi.properties | 10 +- conf/pixels-sink.mysql.properties | 8 +- conf/pixels-sink.pg.properties | 10 +- docs/configuration.md | 12 +- pom.xml | 246 ++++++++++-------- .../pixels/sink/config/PixelsSinkConfig.java | 16 +- .../sink/freshness/FreshnessClient.java | 103 +++++--- 11 files changed, 253 insertions(+), 186 deletions(-) diff --git a/conf/pixels-sink.aws.properties b/conf/pixels-sink.aws.properties index 012b60f..b7f34eb 100644 --- a/conf/pixels-sink.aws.properties +++ b/conf/pixels-sink.aws.properties @@ -20,11 +20,11 @@ sink.retina.trans.request.batch.size=100 sink.trans.mode=batch sink.monitor.report.enable=true # trino for freshness query -trino.url=jdbc:trino://realtime-pixels-coordinator:8080/pixels/pixels_bench -# trino.url=jdbc:trino://realtime-pixels-coordinator:8080/pixels/pixels_bench_sf10x -trino.user=pixels -trino.password=password -trino.parallel=1 +sink.query.url=jdbc:trino://realtime-pixels-coordinator:8080/pixels/pixels_bench +# sink.query.url=jdbc:trino://realtime-pixels-coordinator:8080/pixels/pixels_bench_sf10x +sink.query.user=pixels +sink.query.password=password +sink.query.parallel=1 # row or txn or embed sink.monitor.freshness.level=embed sink.monitor.freshness.verbose=true diff --git a/conf/pixels-sink.ch.properties b/conf/pixels-sink.ch.properties index c979389..7425a6f 100644 --- a/conf/pixels-sink.ch.properties +++ b/conf/pixels-sink.ch.properties @@ -23,10 +23,10 @@ sink.trans.mode=batch # sink.trans.mode=record sink.monitor.report.enable=true # trino for freshness query -trino.url=jdbc:trino://realtime-pixels-coordinator:8080/pixels/pixels_bench -trino.user=pixels -trino.password=password -trino.parallel=1 +sink.query.url=jdbc:trino://realtime-pixels-coordinator:8080/pixels/pixels_bench +sink.query.user=pixels +sink.query.password=password +sink.query.parallel=1 # row or txn or embed sink.monitor.freshness.level=embed sink.monitor.freshness.verbose=true diff --git a/conf/pixels-sink.flink.properties.chbench b/conf/pixels-sink.flink.properties.chbench index 6c87406..fac352f 100644 --- a/conf/pixels-sink.flink.properties.chbench +++ b/conf/pixels-sink.flink.properties.chbench @@ -11,10 +11,10 @@ sink.monitor.report.enable=true sink.monitor.report.file=/home/ubuntu/pixels-sink/result_delta/rate_test.csv sink.monitor.freshness.file=/home/ubuntu/pixels-sink/result_delta/fresh_batch10k.csv # trino for freshness query -trino.url=jdbc:trino://realtime-pixels-coordinator:8080/delta_lake/chbenchmark_w10000 -trino.user=pixels -trino.password=password -trino.parallel=1 +sink.query.url=jdbc:trino://realtime-pixels-coordinator:8080/delta_lake/chbenchmark_w10000 +sink.query.user=pixels +sink.query.password=password +sink.query.parallel=1 # row or txn or embed sink.monitor.freshness.level=embed sink.monitor.freshness.embed.warmup=10 diff --git a/conf/pixels-sink.flink.properties.hybench b/conf/pixels-sink.flink.properties.hybench index eb03619..ea6d956 100644 --- a/conf/pixels-sink.flink.properties.hybench +++ b/conf/pixels-sink.flink.properties.hybench @@ -12,10 +12,10 @@ sink.monitor.report.enable=true sink.monitor.report.file=/home/ubuntu/pixels-sink/result_delta/rate_hybench_rate500.csv sink.monitor.freshness.file=/home/ubuntu/pixels-sink/result_delta/fresh_hybench_rate500.csv # trino for freshness query -trino.url=jdbc:trino://realtime-pixels-coordinator:8080/delta_lake/hybench_sf1000 -trino.user=pixels -trino.password=password -trino.parallel=1 +sink.query.url=jdbc:trino://realtime-pixels-coordinator:8080/delta_lake/hybench_sf1000 +sink.query.user=pixels +sink.query.password=password +sink.query.parallel=1 # row or txn or embed sink.monitor.freshness.level=embed sink.monitor.freshness.embed.warmup=10 diff --git a/conf/pixels-sink.hudi.properties b/conf/pixels-sink.hudi.properties index 8ab4983..fb8f27f 100644 --- a/conf/pixels-sink.hudi.properties +++ b/conf/pixels-sink.hudi.properties @@ -10,11 +10,11 @@ sink.trans.mode=batch sink.monitor.report.enable=true sink.monitor.report.file=/home/ubuntu/pixels-sink/result_delta/rate_test.csv sink.monitor.freshness.file=/home/ubuntu/pixels-sink/result_delta/fresh_batch10k.csv -# trino for freshness query -trino.url=jdbc:hive2://172.31.16.214:10000/hudi_hybench_sf1333_2 -trino.user=pixels -trino.password= -trino.parallel=1 +# HiveServer2 for Hudi RO incremental freshness (requires: mvn -Phudi-hive package) +sink.query.url=jdbc:hive2://172.31.16.214:10000/hudi_hybench_sf1333_2 +sink.query.user=pixels +sink.query.password= +sink.query.parallel=1 # row or txn or embed sink.monitor.freshness.level=embed sink.monitor.freshness.embed.warmup=0 diff --git a/conf/pixels-sink.mysql.properties b/conf/pixels-sink.mysql.properties index 4b8bc97..b3520b1 100644 --- a/conf/pixels-sink.mysql.properties +++ b/conf/pixels-sink.mysql.properties @@ -12,10 +12,10 @@ sink.monitor.report.enable=true sink.monitor.report.file=/home/ubuntu/pixels-sink/resulti7i/100k_rate_mysql.csv sink.monitor.freshness.file=/home/ubuntu/pixels-sink/resulti7i/100k_freshness_mysql.csv # trino for freshness query -trino.url=jdbc:trino://realtime-kafka-2:8080/pixels/pixels_bench_sf10x -trino.user=pixels -trino.password=password -trino.parallel=8 +sink.query.url=jdbc:trino://realtime-kafka-2:8080/pixels/pixels_bench_sf10x +sink.query.user=pixels +sink.query.password=password +sink.query.parallel=8 # row or txn or embed sink.monitor.freshness.level=embed sink.monitor.freshness.embed.warmup=10 diff --git a/conf/pixels-sink.pg.properties b/conf/pixels-sink.pg.properties index 868c64d..079bfa1 100644 --- a/conf/pixels-sink.pg.properties +++ b/conf/pixels-sink.pg.properties @@ -12,11 +12,11 @@ sink.monitor.report.enable=true sink.monitor.report.file=/home/ubuntu/pixels-sink/resulti7i/100k_rate_2.csv sink.monitor.freshness.file=/home/ubuntu/pixels-sink/resulti7i/100k_freshness_2.csv # trino for freshness query -trino.url=jdbc:trino://realtime-kafka-2:8080/pixels/pixels_bench_sf10x -# trino.url=jdbc:trino://realtime-pixels-coordinator:8080/pixels/pixels_bench_sf10x -trino.user=pixels -trino.password=password -trino.parallel=8 +sink.query.url=jdbc:trino://realtime-kafka-2:8080/pixels/pixels_bench_sf10x +# sink.query.url=jdbc:trino://realtime-pixels-coordinator:8080/pixels/pixels_bench_sf10x +sink.query.user=pixels +sink.query.password=password +sink.query.parallel=8 # row or txn or embed sink.monitor.freshness.level=embed sink.monitor.freshness.embed.warmup=10 diff --git a/docs/configuration.md b/docs/configuration.md index 6355e23..3ad3326 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -151,13 +151,13 @@ Kafka source is deprecated. | `sink.monitor.freshness.verbose` | `false` | Verbose freshness logging. | | `sink.monitor.freshness.timestamp` | `false` | Include timestamps. | -Note: In the Retina paper experiments, `sink.monitor.freshness.level=embed` is used to query freshness from Trino. This requires the last column of each table to be `freshness_ts`. +Note: In the Retina paper experiments, `sink.monitor.freshness.level=embed` queries freshness through JDBC. This requires the last column of each table to be `freshness_ts`. Trino is available in the default build; HiveServer2 support for Hudi requires building with `-Phudi-hive`. -**Freshness Trino Settings** +**Freshness Query Settings** | Key | Default | Notes | | --- | --- | --- | -| `trino.url` | required for Trino-based freshness | JDBC URL. | -| `trino.user` | required for Trino-based freshness | Username. | -| `trino.password` | required for Trino-based freshness | Password. | -| `trino.parallel` | `1` | Parallel query count. | +| `sink.query.url` | required for embedded freshness | Trino or HiveServer2 JDBC URL. | +| `sink.query.user` | required for embedded freshness | Username. | +| `sink.query.password` | empty | Password. | +| `sink.query.parallel` | `1` | Parallel query count. | diff --git a/pom.xml b/pom.xml index 99696c5..f9203eb 100644 --- a/pom.xml +++ b/pom.xml @@ -34,19 +34,23 @@ true 0.9.0 - 3.8.0 - 5.8 + 3.9.1 1.18.42 3.2.3.Final 2.16.2 + 2.6.2.Final + 1.11.3 + 3.6.1 + 3.12.0 + + 440 + 2.3.9 1.4.13 - 440 - + com.fasterxml.jackson jackson-bom @@ -54,6 +58,13 @@ pom import + + io.netty + netty-bom + ${dep.netty.version} + pom + import + @@ -69,90 +80,59 @@ true - io.pixelsdb - pixels-retina + io.etcd + jetcd-core true - test + - org.apache.hive - hive-jdbc - 2.3.9 - - - jdk.tools - jdk.tools - - - - org.apache.logging.log4j - log4j-web - - - org.apache.logging.log4j - log4j-1.2-api - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-reload4j - - - ch.qos.reload4j - reload4j - - - - org.apache.hive - hive-llap-server - - + io.netty + netty-codec-http2 + runtime - io.etcd - jetcd-core - true + io.netty + netty-handler-proxy + runtime io.netty - netty-all + netty-resolver-dns + runtime - io.grpc - grpc-netty + io.netty + netty-transport-native-unix-common + runtime io.trino trino-jdbc - ${trino.version} + runtime - com.alibaba - fastjson - true + org.apache.logging.log4j + log4j-api + runtime org.apache.logging.log4j log4j-core true + runtime org.apache.logging.log4j log4j-slf4j-impl - ${dep.log4j.version} + runtime org.slf4j log4j-over-slf4j + runtime + io.grpc grpc-netty-shaded true + runtime + + + + io.grpc + grpc-netty + runtime io.grpc @@ -179,14 +168,12 @@ true - com.google.protobuf - protobuf-java-util - 3.25.1 + io.grpc + grpc-api - - javax.annotation - javax.annotation-api + com.google.protobuf + protobuf-java @@ -207,16 +194,14 @@ connect-api ${dep.kafka.version} - - com.opencsv - opencsv - ${dep.opencsv.version} + org.slf4j + slf4j-api + com.google.guava guava - 33.2.0-jre @@ -226,6 +211,19 @@ provided + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + org.apache.avro + avro + ${dep.avro.version} + @@ -245,88 +243,73 @@ - - io.debezium - debezium-sink - ${dep.debezium.version} - io.debezium debezium-connector-postgres ${dep.debezium.version} + runtime io.debezium debezium-connector-mysql ${dep.debezium.version} + runtime - junit - junit - test - true - - - org.junit.platform - junit-platform-launcher - test - true - - - org.junit.platform - junit-platform-runner + org.junit.jupiter + junit-jupiter + ${dep.junit.jupiter.version} test - true - org.junit.jupiter - junit-jupiter - 5.8.2 - test + io.apicurio + apicurio-registry-serdes-avro-serde + ${dep.apicurio.version} - org.junit.jupiter - junit-jupiter-api - 5.8.2 - test + io.apicurio + apicurio-registry-serde-common + ${dep.apicurio.version} - io.apicurio - apicurio-registry-serdes-avro-serde - 2.6.2.Final + apicurio-registry-client + ${dep.apicurio.version} io.prometheus simpleclient - 0.16.0 io.prometheus simpleclient_hotspot - 0.16.0 - io.prometheus simpleclient_httpserver - 0.16.0 + ${dep.prometheus.client.version} org.apache.commons commons-math3 - 3.6.1 + ${dep.commons-math3.version} + + + org.apache.commons + commons-lang3 + ${dep.commons-lang3.version} - + io.pixelsdb pixels-storage-localfs + runtime @@ -374,6 +357,35 @@ META-INF/*.SF META-INF/*.DSA META-INF/*.RSA + module-info.class + META-INF/versions/**/module-info.class + + + + io.grpc:grpc-netty + + META-INF/services/io.grpc.ServerProvider + META-INF/services/io.grpc.ManagedChannelProvider + + + + + org.apache.hive:hive-jdbc + + org/apache/logging/log4j/** + org/apache/logging/slf4j/** + com/fasterxml/jackson/** + javax/servlet/** + org/openjdk/jol/** + META-INF/org/apache/logging/log4j/** + META-INF/services/org.apache.logging.log4j.* + META-INF/services/com.fasterxml.jackson.* + META-INF/services/javax.servlet.ServletContainerInitializer + META-INF/maven/org.apache.logging.log4j/** + META-INF/maven/com.fasterxml.jackson.core/** + META-INF/maven/javax.servlet/** + META-INF/maven/org.openjdk.jol/** @@ -429,6 +441,26 @@ + + + hudi-hive + + + org.apache.hive + hive-jdbc + ${dep.hive.jdbc.version} + standalone + runtime + + + * + * + + + + + integration-tests diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java index 297b852..5e53ea9 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java @@ -220,17 +220,17 @@ public class PixelsSinkConfig @ConfigKey(value = "sink.monitor.freshness.timestamp", defaultValue = "false") private boolean sinkMonitorFreshnessTimestamp; - @ConfigKey(value = "trino.url") - private String trinoUrl; + @ConfigKey(value = "sink.query.url") + private String sinkQueryUrl; - @ConfigKey(value = "trino.user") - private String trinoUser; + @ConfigKey(value = "sink.query.user") + private String sinkQueryUser; - @ConfigKey(value = "trino.password") - private String trinoPassword; + @ConfigKey(value = "sink.query.password") + private String sinkQueryPassword; - @ConfigKey(value = "trino.parallel", defaultValue = "1") - private int trinoParallel; + @ConfigKey(value = "sink.query.parallel", defaultValue = "1") + private int sinkQueryParallel; public PixelsSinkConfig(String configFilePath) throws IOException { diff --git a/src/main/java/io/pixelsdb/pixels/sink/freshness/FreshnessClient.java b/src/main/java/io/pixelsdb/pixels/sink/freshness/FreshnessClient.java index da35c54..73ba79a 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/freshness/FreshnessClient.java +++ b/src/main/java/io/pixelsdb/pixels/sink/freshness/FreshnessClient.java @@ -36,8 +36,8 @@ import java.util.concurrent.*; /** - * FreshnessClient is responsible for monitoring data freshness by periodically - * querying the maximum timestamp from a set of dynamically configured tables via Trino JDBC. + * FreshnessClient monitors data freshness by periodically querying max(freshness_ts) + * via JDBC (Trino by default, or HiveServer2 for Hudi when built with {@code -Phudi-hive}). */ public class FreshnessClient { @@ -45,9 +45,10 @@ public class FreshnessClient private static final int QUERY_INTERVAL_SECONDS = 1; private static volatile FreshnessClient instance; // Configuration parameters (should ideally be loaded from a config file) - private final String trinoJdbcUrl; - private final String trinoUser; - private final String trinoPassword; + private final String queryJdbcUrl; + private final String queryUser; + private final String queryPassword; + private final QueryEngine queryEngine; private final int maxConcurrentQueries; private final Semaphore queryPermits; private final ThreadPoolExecutor connectionExecutor; @@ -58,17 +59,35 @@ public class FreshnessClient private final int warmUpSeconds; private final PixelsSinkConfig config; + private enum QueryEngine + { + TRINO("io.trino.jdbc.TrinoDriver", + "Trino JDBC driver not found on classpath."), + HIVE("org.apache.hive.jdbc.HiveDriver", + "Hive JDBC driver not found. Build with -Phudi-hive to enable Hudi freshness via HiveServer2."); + + private final String driverClass; + private final String missingHint; + + QueryEngine(String driverClass, String missingHint) + { + this.driverClass = driverClass; + this.missingHint = missingHint; + } + } + private FreshnessClient() { // Initializes the set with thread safety wrapper this.monitoredTables = Collections.synchronizedSet(new HashSet<>()); this.config = PixelsSinkConfigFactory.getInstance(); - this.trinoUser = config.getTrinoUser(); - this.trinoJdbcUrl = config.getTrinoUrl(); - this.trinoPassword = config.getTrinoPassword(); + this.queryUser = config.getSinkQueryUser(); + this.queryJdbcUrl = config.getSinkQueryUrl(); + this.queryPassword = config.getSinkQueryPassword(); + this.queryEngine = resolveQueryEngine(queryJdbcUrl); this.warmUpSeconds = config.getSinkMonitorFreshnessEmbedWarmupSeconds(); - this.maxConcurrentQueries = config.getTrinoParallel(); + this.maxConcurrentQueries = config.getSinkQueryParallel(); this.queryPermits = new Semaphore(maxConcurrentQueries); // Initializes a single-threaded scheduler for executing freshness queries this.scheduler = Executors.newSingleThreadScheduledExecutor(r -> @@ -113,42 +132,59 @@ public static FreshnessClient getInstance() return instance; } - @Deprecated - protected Connection createNewConnection() throws SQLException + /** + * Unified JDBC entry: resolve engine from URL at construction, load driver, + * apply engine-specific connection properties, then return a ready Connection. + * + * @param queryTimestamp Trino snapshot timestamp when embed.snapshot is enabled; ignored otherwise / for Hive + */ + protected Connection openConnection(Long queryTimestamp) throws SQLException { try { - Class.forName("io.trino.jdbc.TrinoDriver"); - } catch (ClassNotFoundException e) + Class.forName(queryEngine.driverClass); + } + catch (ClassNotFoundException e) { - throw new SQLException(e); + throw new SQLException(queryEngine.missingHint, e); } Properties properties = new Properties(); + properties.setProperty("user", queryUser); + if (queryEngine == QueryEngine.TRINO + && queryTimestamp != null + && config.isSinkMonitorFreshnessEmbedSnapshot()) + { + String sessionPropValue = String.format( + "pixels.query_snapshot_timestamp:%d", queryTimestamp); + properties.setProperty("sessionProperties", sessionPropValue); + } + return DriverManager.getConnection(queryJdbcUrl, properties); + } - - return DriverManager.getConnection(trinoJdbcUrl, trinoUser, null); + @Deprecated + protected Connection createNewConnection() throws SQLException + { + return openConnection(null); } protected Connection createNewConnection(long queryTimestamp) throws SQLException { - try - { - Class.forName("io.trino.jdbc.TrinoDriver"); - } catch (ClassNotFoundException e) + return openConnection(queryTimestamp); + } + + private static QueryEngine resolveQueryEngine(String jdbcUrl) + { + if (jdbcUrl != null && jdbcUrl.startsWith("jdbc:hive2:")) { - throw new SQLException(e); + return QueryEngine.HIVE; } - - Properties properties = new Properties(); - properties.setProperty("user", trinoUser); - if (config.isSinkMonitorFreshnessEmbedSnapshot()) + if (jdbcUrl != null && jdbcUrl.startsWith("jdbc:trino:")) { - String catalogName = "pixels"; - String sessionPropValue = String.format("%s.query_snapshot_timestamp:%d", catalogName, queryTimestamp); - properties.setProperty("sessionProperties", sessionPropValue); + return QueryEngine.TRINO; } - return DriverManager.getConnection(trinoJdbcUrl, properties); + throw new IllegalArgumentException( + "Unsupported freshness JDBC URL (expect jdbc:trino:// or jdbc:hive2://): " + jdbcUrl); } private void closeConnection(Connection conn) @@ -160,7 +196,7 @@ private void closeConnection(Connection conn) conn.close(); } catch (SQLException e) { - LOGGER.warn("Error closing Trino connection.", e); + LOGGER.warn("Error closing freshness JDBC connection.", e); } } } @@ -282,14 +318,13 @@ void queryAndCalculateFreshness() } // Timestamp when the query is sent (t_send) long tSendMillis = System.currentTimeMillis(); + Long snapshotTimestamp = null; if (config.isSinkMonitorFreshnessEmbedSnapshot()) { transContext = TransService.Instance().beginTrans(true); - conn = createNewConnection(transContext.getTimestamp()); - } else - { - conn = createNewConnection(); + snapshotTimestamp = transContext.getTimestamp(); } + conn = openConnection(snapshotTimestamp); String tSendMillisStr = DateUtil.convertDateToString(new Date(tSendMillis)); // Query to find the latest timestamp in the table From 2b9ae21f21d5b172b41a8eb96ab12927a438f1d1 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Fri, 31 Jul 2026 11:24:02 +0800 Subject: [PATCH 07/13] feat: Add sink.storage.mode to select stream or memory storage source --- conf/pixels-sink.aws.properties | 1 + conf/pixels-sink.ch.properties | 1 + conf/pixels-sink.flink.properties.chbench | 1 + conf/pixels-sink.flink.properties.hybench | 1 + conf/pixels-sink.hudi.properties | 1 + conf/pixels-sink.mysql.properties | 1 + conf/pixels-sink.pg.properties | 1 + docs/configuration.md | 1 + .../pixels/sink/config/PixelsSinkConfig.java | 3 + .../sink/config/PixelsSinkDefaultConfig.java | 1 + .../pixels/sink/source/SinkSourceFactory.java | 19 ++++- .../storage/AbstractSinkStorageSource.java | 63 +++++++++++++- .../storage/FasterSinkStorageSource.java | 65 -------------- ...urce.java => MemorySinkStorageSource.java} | 69 ++++----------- .../storage/StreamingSinkStorageSource.java | 84 +++++++++++++++++++ src/main/resources/pixels-sink.aws.properties | 1 + .../resources/pixels-sink.local.properties | 1 + src/main/resources/pixels-sink.properties | 1 + 18 files changed, 192 insertions(+), 123 deletions(-) delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/storage/FasterSinkStorageSource.java rename src/main/java/io/pixelsdb/pixels/sink/source/storage/{AbstractMemorySinkStorageSource.java => MemorySinkStorageSource.java} (58%) create mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/storage/StreamingSinkStorageSource.java diff --git a/conf/pixels-sink.aws.properties b/conf/pixels-sink.aws.properties index b7f34eb..fb1ae99 100644 --- a/conf/pixels-sink.aws.properties +++ b/conf/pixels-sink.aws.properties @@ -1,5 +1,6 @@ # engine | kafka | storage sink.datasource=storage +sink.storage.mode=stream # -1 means no limit, Only implement in retina sink mode yet: sink.datasource.rate.limit=200000 sink.monitor.report.file=/home/ubuntu/pixels-sink/result1k2_feb/rate_8192tile_3.csv diff --git a/conf/pixels-sink.ch.properties b/conf/pixels-sink.ch.properties index 7425a6f..5f68942 100644 --- a/conf/pixels-sink.ch.properties +++ b/conf/pixels-sink.ch.properties @@ -1,5 +1,6 @@ # engine | kafka | storage sink.datasource=storage +sink.storage.mode=stream sink.mode=retina #sink.datasource=engine #sink.mode=proto diff --git a/conf/pixels-sink.flink.properties.chbench b/conf/pixels-sink.flink.properties.chbench index fac352f..c862997 100644 --- a/conf/pixels-sink.flink.properties.chbench +++ b/conf/pixels-sink.flink.properties.chbench @@ -1,5 +1,6 @@ # engine | kafka | storage sink.datasource=storage +sink.storage.mode=stream # -1 means no limit, Only implement in retina sink mode yet sink.datasource.rate.limit=2000 # Sink Config: retina | csv | proto | flink | none diff --git a/conf/pixels-sink.flink.properties.hybench b/conf/pixels-sink.flink.properties.hybench index ea6d956..d809dc6 100644 --- a/conf/pixels-sink.flink.properties.hybench +++ b/conf/pixels-sink.flink.properties.hybench @@ -1,5 +1,6 @@ # engine | kafka | storage sink.datasource=storage +sink.storage.mode=stream # -1 means no limit, Only implement in retina sink mode yet sink.datasource.rate.limit=500 sink.datasource.rate.limit.type=guava diff --git a/conf/pixels-sink.hudi.properties b/conf/pixels-sink.hudi.properties index fb8f27f..801922c 100644 --- a/conf/pixels-sink.hudi.properties +++ b/conf/pixels-sink.hudi.properties @@ -1,5 +1,6 @@ # engine | kafka | storage sink.datasource=storage +sink.storage.mode=stream # -1 means no limit, Only implement in retina sink mode yet sink.datasource.rate.limit=2000 # Sink Config: retina | csv | proto | flink | none diff --git a/conf/pixels-sink.mysql.properties b/conf/pixels-sink.mysql.properties index b3520b1..a119151 100644 --- a/conf/pixels-sink.mysql.properties +++ b/conf/pixels-sink.mysql.properties @@ -1,5 +1,6 @@ # engine | kafka | storage sink.datasource=engine +sink.storage.mode=stream # -1 means no limit, Only implement in retina sink mode yet sink.datasource.rate.limit=100000 # Sink Config: retina | csv | proto | flink | none diff --git a/conf/pixels-sink.pg.properties b/conf/pixels-sink.pg.properties index 079bfa1..c2795ce 100644 --- a/conf/pixels-sink.pg.properties +++ b/conf/pixels-sink.pg.properties @@ -1,5 +1,6 @@ # engine | kafka | storage sink.datasource=engine +sink.storage.mode=stream # -1 means no limit, Only implement in retina sink mode yet sink.datasource.rate.limit=100000 # Sink Config: retina | csv | proto | flink | none diff --git a/docs/configuration.md b/docs/configuration.md index 3ad3326..73d486a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -95,6 +95,7 @@ The Debezium Connector reads the database Binlog or WAL. The local `conversion.d | `sink.proto.dir` | required | Proto output or input directory. | | `sink.proto.data` | `data` | Data set name. | | `sink.proto.maxRecords` | `100000` | Max records per file. | +| `sink.storage.mode` | `stream` | Storage read mode: `stream` reads records incrementally; `memory` preloads all records before replay. | | `sink.storage.loop` | `false` | Whether to loop over stored files. | ### Flink Sink diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java index 5e53ea9..554e517 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java @@ -197,6 +197,9 @@ public class PixelsSinkConfig @ConfigKey(value = "sink.storage.loop", defaultValue = "false") private boolean sinkStorageLoop; + @ConfigKey(value = "sink.storage.mode", defaultValue = PixelsSinkDefaultConfig.STORAGE_MODE) + private String sinkStorageMode; + @ConfigKey(value = "sink.monitor.freshness.level", defaultValue = "row") // row or txn or embed private String sinkMonitorFreshnessLevel; @ConfigKey(value = "sink.monitor.freshness.embed.warmup", defaultValue = "10") diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkDefaultConfig.java b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkDefaultConfig.java index a0667ad..8dfc00d 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkDefaultConfig.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkDefaultConfig.java @@ -53,4 +53,5 @@ public class PixelsSinkDefaultConfig // Mock RPC public static final boolean SINK_RPC_ENABLED = true; public static final String MAX_RECORDS_PER_FILE = "100000"; + public static final String STORAGE_MODE = "stream"; } diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/SinkSourceFactory.java b/src/main/java/io/pixelsdb/pixels/sink/source/SinkSourceFactory.java index c0463d0..5b61ad7 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/SinkSourceFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/SinkSourceFactory.java @@ -25,7 +25,10 @@ import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; import io.pixelsdb.pixels.sink.source.engine.SinkEngineSource; import io.pixelsdb.pixels.sink.source.kafka.SinkKafkaSource; -import io.pixelsdb.pixels.sink.source.storage.FasterSinkStorageSource; +import io.pixelsdb.pixels.sink.source.storage.MemorySinkStorageSource; +import io.pixelsdb.pixels.sink.source.storage.StreamingSinkStorageSource; + +import java.util.Locale; public class SinkSourceFactory { @@ -36,8 +39,20 @@ public static SinkSource createSinkSource() { case "kafka" -> new SinkKafkaSource(); case "engine" -> new SinkEngineSource(); - case "storage" -> new FasterSinkStorageSource(); + case "storage" -> createStorageSource(config); default -> throw new IllegalStateException("Unsupported data source type: " + config.getDataSource()); }; } + + private static SinkSource createStorageSource(PixelsSinkConfig config) + { + String storageMode = config.getSinkStorageMode().trim().toLowerCase(Locale.ROOT); + return switch (storageMode) + { + case "stream" -> new StreamingSinkStorageSource(); + case "memory" -> new MemorySinkStorageSource(); + default -> throw new IllegalStateException( + "Unsupported storage source mode: " + config.getSinkStorageMode()); + }; + } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java index 7c72288..dd1b525 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java @@ -22,6 +22,7 @@ import io.pixelsdb.pixels.common.physical.PhysicalReader; import io.pixelsdb.pixels.core.utils.Pair; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; import io.pixelsdb.pixels.sink.SinkProto; @@ -43,6 +44,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -50,11 +52,13 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; public abstract class AbstractSinkStorageSource implements SinkSource { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractSinkStorageSource.class); + protected static final int RECORD_HEADER_SIZE = Integer.BYTES * 2; protected final AtomicBoolean running = new AtomicBoolean(false); protected final String topic; @@ -89,10 +93,67 @@ protected AbstractSinkStorageSource() this.sourceRateLimiter = FlushRateLimiterFactory.getNewInstance(); } - abstract ProtoType getProtoType(int i); + ProtoType getProtoType(int key) + { + return key == -1 ? ProtoType.TRANS : ProtoType.ROW; + } + + protected static Pair readRecord( + PhysicalReader reader, long offset, long fileLength) throws IOException + { + long remaining = fileLength - offset; + if (remaining < RECORD_HEADER_SIZE) + { + throw new IOException( + "Truncated sink proto header at offset " + offset + + " in " + reader.getPath()); + } + + int key = reader.readInt(ByteOrder.BIG_ENDIAN); + int valueLength = reader.readInt(ByteOrder.BIG_ENDIAN); + long availablePayload = remaining - RECORD_HEADER_SIZE; + if (valueLength < 0 || valueLength > availablePayload) + { + throw new IOException( + "Invalid sink proto payload length " + valueLength + + " at offset " + offset + " in " + reader.getPath()); + } + + return new Pair<>(key, reader.readFully(valueLength)); + } + + protected void submitRecord(int key, ByteBuffer valueBuffer, int recordLoopId) + throws InterruptedException + { + BlockingQueue, Integer>> queue = + queueMap.computeIfAbsent( + key, + ignored -> new LinkedBlockingQueue<>(PixelsSinkConstants.MAX_QUEUE_SIZE) + ); + + consumerThreads.computeIfAbsent(key, ignored -> + { + ProtoType protoType = getProtoType(key); + Thread thread = new Thread(() -> consumeQueue(key, queue, protoType)); + thread.setName("consumer-" + key); + thread.start(); + return thread; + }); + + if (getProtoType(key) == ProtoType.ROW) + { + sourceRateLimiter.acquire(1); + } + + queue.put(new Pair<>( + CompletableFuture.completedFuture(valueBuffer), + recordLoopId + )); + } protected void clean() { + running.set(false); queueMap.values().forEach(q -> { try diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/FasterSinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/FasterSinkStorageSource.java deleted file mode 100644 index 6e76cba..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/source/storage/FasterSinkStorageSource.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.source.storage; - - -import io.pixelsdb.pixels.common.metadata.SchemaTableName; -import io.pixelsdb.pixels.sink.provider.ProtoType; -import io.pixelsdb.pixels.sink.source.SinkSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.ByteBuffer; - -/** - * @package: io.pixelsdb.pixels.sink.source - * @className: FasterSinkStorageSource - * @author: AntiO2 - * @date: 2025/10/5 11:43 - */ -public class FasterSinkStorageSource extends AbstractMemorySinkStorageSource implements SinkSource -{ - private static final Logger LOGGER = LoggerFactory.getLogger(FasterSinkStorageSource.class); - static SchemaTableName transactionSchemaTableName = new SchemaTableName("freak", "transaction"); - - public FasterSinkStorageSource() - { - super(); - } - - private static String readString(ByteBuffer buffer, int len) - { - byte[] bytes = new byte[len]; - buffer.get(bytes); - return new String(bytes); - } - - @Override - ProtoType getProtoType(int i) - { - if (i == -1) - { - return ProtoType.TRANS; - } - return ProtoType.ROW; - } - -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractMemorySinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/MemorySinkStorageSource.java similarity index 58% rename from src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractMemorySinkStorageSource.java rename to src/main/java/io/pixelsdb/pixels/sink/source/storage/MemorySinkStorageSource.java index 78f7e63..9d67189 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractMemorySinkStorageSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/storage/MemorySinkStorageSource.java @@ -1,5 +1,5 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * * This file is part of Pixels. * @@ -24,23 +24,17 @@ import io.pixelsdb.pixels.common.physical.PhysicalReaderUtil; import io.pixelsdb.pixels.common.physical.Storage; import io.pixelsdb.pixels.core.utils.Pair; -import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; -import io.pixelsdb.pixels.sink.provider.ProtoType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.LinkedBlockingQueue; -public abstract class AbstractMemorySinkStorageSource extends AbstractSinkStorageSource +public class MemorySinkStorageSource extends AbstractSinkStorageSource { - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractMemorySinkStorageSource.class); + private static final Logger LOGGER = LoggerFactory.getLogger(MemorySinkStorageSource.class); // All preloaded records, order preserved // key + value buffer @@ -64,27 +58,19 @@ public void start() PhysicalReader reader = PhysicalReaderUtil.newPhysicalReader(scheme, file); readers.add(reader); - while (true) + reader.seek(0); + long offset = 0; + long fileLength = reader.getFileLength(); + while (offset < fileLength) { - int key; - int valueLen; - - try - { - key = reader.readInt(ByteOrder.BIG_ENDIAN); - valueLen = reader.readInt(ByteOrder.BIG_ENDIAN); - } catch (IOException eof) - { - // Reached end of file - break; - } - // Synchronous read and copy to heap buffer - ByteBuffer valueBuffer = reader.readFully(valueLen); + Pair record = readRecord(reader, offset, fileLength); + int valueLength = record.getRight().remaining(); // Store into a single global array - ByteBuffer cleanBuffer = valueBuffer.duplicate(); + ByteBuffer cleanBuffer = record.getRight().duplicate(); cleanBuffer.rewind(); - cleanBuffer.limit(cleanBuffer.position() + valueLen); - preloadedRecords.add(new Pair<>(key, cleanBuffer)); + cleanBuffer.limit(cleanBuffer.position() + valueLength); + preloadedRecords.add(new Pair<>(record.getLeft(), cleanBuffer)); + offset += RECORD_HEADER_SIZE + (long) valueLength; } } @@ -104,34 +90,7 @@ public void start() ByteBuffer copy = ByteBuffer.allocate(src.remaining()); copy.put(src.duplicate().rewind()); copy.flip(); - // Lazily create queue - BlockingQueue, Integer>> queue = - queueMap.computeIfAbsent( - key, - k -> new LinkedBlockingQueue<>(PixelsSinkConstants.MAX_QUEUE_SIZE) - ); - - // Lazily start consumer thread - consumerThreads.computeIfAbsent(key, k -> - { - ProtoType protoType = getProtoType(k); - Thread t = new Thread(() -> consumeQueue(k, queue, protoType)); - t.setName("consumer-" + k); - t.start(); - return t; - }); - - ProtoType protoType = getProtoType(key); - if (protoType == ProtoType.ROW) - { - sourceRateLimiter.acquire(1); - } - - // Use completed future to keep consumer logic unchanged - CompletableFuture future = - CompletableFuture.completedFuture(copy); - - queue.put(new Pair<>(future, loopId)); + submitRecord(key, copy, loopId); } ++loopId; } while (storageLoopEnabled && isRunning()); diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/StreamingSinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/StreamingSinkStorageSource.java new file mode 100644 index 0000000..8201a10 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/storage/StreamingSinkStorageSource.java @@ -0,0 +1,84 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the Affero GNU General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.source.storage; + +import io.pixelsdb.pixels.common.physical.PhysicalReader; +import io.pixelsdb.pixels.common.physical.PhysicalReaderUtil; +import io.pixelsdb.pixels.common.physical.Storage; +import io.pixelsdb.pixels.core.utils.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; + +public class StreamingSinkStorageSource extends AbstractSinkStorageSource +{ + private static final Logger LOGGER = LoggerFactory.getLogger(StreamingSinkStorageSource.class); + + @Override + public void start() + { + this.running.set(true); + this.transactionPipeline.start(); + try + { + for (String file : files) + { + Storage.Scheme scheme = Storage.Scheme.fromPath(file); + readers.add(PhysicalReaderUtil.newPhysicalReader(scheme, file)); + } + + do + { + for (PhysicalReader reader : readers) + { + if (!isRunning()) + { + break; + } + + LOGGER.info("Start reading {}", reader.getPath()); + reader.seek(0); + long offset = 0; + long fileLength = reader.getFileLength(); + while (isRunning() && offset < fileLength) + { + Pair record = readRecord(reader, offset, fileLength); + int valueLength = record.getRight().remaining(); + submitRecord(record.getLeft(), record.getRight(), loopId); + offset += RECORD_HEADER_SIZE + (long) valueLength; + } + } + ++loopId; + } while (storageLoopEnabled && isRunning()); + } catch (IOException e) + { + throw new RuntimeException("Failed to read sink proto storage", e); + } catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + } finally + { + clean(); + } + } +} diff --git a/src/main/resources/pixels-sink.aws.properties b/src/main/resources/pixels-sink.aws.properties index b99cbae..081f5ad 100644 --- a/src/main/resources/pixels-sink.aws.properties +++ b/src/main/resources/pixels-sink.aws.properties @@ -1,5 +1,6 @@ # engine | kafka | storage sink.datasource=storage +sink.storage.mode=stream # Sink Config: retina | csv | proto | none sink.mode=none # Kafka Config diff --git a/src/main/resources/pixels-sink.local.properties b/src/main/resources/pixels-sink.local.properties index 0b24c54..d871cd2 100644 --- a/src/main/resources/pixels-sink.local.properties +++ b/src/main/resources/pixels-sink.local.properties @@ -1,5 +1,6 @@ # engine | kafka | storage sink.datasource=storage +sink.storage.mode=stream # Sink Config: retina | csv | proto | none sink.mode=proto # Kafka Config diff --git a/src/main/resources/pixels-sink.properties b/src/main/resources/pixels-sink.properties index 83247f3..d6c0920 100644 --- a/src/main/resources/pixels-sink.properties +++ b/src/main/resources/pixels-sink.properties @@ -1,5 +1,6 @@ # Source Config sink.datasource=kafka +sink.storage.mode=stream sink.datasource.rate.limit=-1 # Kafka Config From 05e36826a223ab695551f4a8f25685fe3e2775ab Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Fri, 31 Jul 2026 16:02:59 +0800 Subject: [PATCH 08/13] perf: parallelize storage source decoding and add close/abort shutdown --- .../pixelsdb/pixels/sink/PixelsSinkApp.java | 2 +- .../pixels/sink/PixelsSinkProvider.java | 2 +- .../sink/config/PixelsSinkConstants.java | 1 - .../pixels/sink/pipeline/TablePipeline.java | 22 +- .../sink/pipeline/TablePipelineManager.java | 58 ++- .../sink/pipeline/TransactionPipeline.java | 23 +- .../sink/processor/MonitorThreadManager.java | 78 ---- .../sink/processor/StoppableProcessor.java | 27 -- .../pixels/sink/processor/TableProcessor.java | 18 +- .../sink/processor/TransactionProcessor.java | 8 +- .../pixels/sink/source/SinkSource.java | 16 +- .../source/engine/PixelsDebeziumConsumer.java | 12 +- .../sink/source/engine/SinkEngineSource.java | 53 ++- .../sink/source/kafka/KafkaRowSource.java | 6 +- .../source/kafka/KafkaTransactionSource.java | 6 +- .../sink/source/kafka/SinkKafkaSource.java | 93 ++++- .../sink/source/kafka/TopicProcessor.java | 54 ++- .../storage/AbstractSinkStorageSource.java | 346 +++++++++++++++--- .../storage/MemorySinkStorageSource.java | 13 +- .../storage/StreamingSinkStorageSource.java | 7 +- .../sink/util/BlockingBoundedQueue.java | 35 +- .../pixels/sink/pipeline/PipelineTest.java | 17 + .../sink/processor/TableProcessorTest.java | 30 +- .../processor/TransactionProcessorTest.java | 36 +- .../sink/util/BlockingBoundedQueueTest.java | 18 +- .../pixels/sink/writer/RpcEndToEndTest.java | 6 +- 26 files changed, 732 insertions(+), 255 deletions(-) delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/processor/MonitorThreadManager.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/processor/StoppableProcessor.java diff --git a/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkApp.java b/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkApp.java index 0453d6a..bfa253a 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkApp.java +++ b/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkApp.java @@ -53,8 +53,8 @@ public static void main(String[] args) throws IOException Runtime.getRuntime().addShutdownHook(new Thread(() -> { PixelsSinkConfig config = PixelsSinkConfigFactory.getInstance(); + sinkSource.close(); TransactionProxy.staticClose(); - sinkSource.stopProcessor(); LOGGER.info("Pixels Sink Server shutdown complete"); if (config.getSinkMonitorFreshnessLevel().equals("embed") && freshnessClient != null) { diff --git a/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkProvider.java b/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkProvider.java index ecd9364..37f53e0 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkProvider.java +++ b/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkProvider.java @@ -42,7 +42,7 @@ public void start(ConfigFactory config) @Override public void shutdown() { - sinkSource.stopProcessor(); + sinkSource.close(); } @Override diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java index b5d7477..d2269bb 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java @@ -27,7 +27,6 @@ public final class PixelsSinkConstants public static final String ROW_RECORD_CONVERTER_CLASS = "pixels.sink.row.converter.class"; public static final String TRANSACTION_CONVERTER_CLASS = "pixels.sink.transaction.converter.class"; public static final String DEBEZIUM_CONNECTOR_CLASS = "debezium.connector.class"; - public static final int MONITOR_NUM = 3; public static final int MAX_QUEUE_SIZE = 1_000; public static final String SNAPSHOT_TX_PREFIX = "SNAPSHOT-"; diff --git a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipeline.java b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipeline.java index 2b33f72..5fbff2c 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipeline.java +++ b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipeline.java @@ -50,10 +50,30 @@ public void publish(RowChangeEvent event) eventQueue.put(event); } + /** + * Stops accepting events and waits for all pending events to be written. + */ @Override public void close() { - processor.stopProcessor(); eventQueue.close(); + try + { + processor.awaitTermination(); + } catch (InterruptedException e) + { + abort(); + Thread.currentThread().interrupt(); + } + } + + /** + * Discards pending events and interrupts processing. Already written events + * are not rolled back. + */ + public void abort() + { + processor.abort(); + eventQueue.abort(); } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipelineManager.java b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipelineManager.java index f4e40cc..2044536 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipelineManager.java +++ b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipelineManager.java @@ -23,32 +23,74 @@ import io.pixelsdb.pixels.common.metadata.SchemaTableName; import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; public final class TablePipelineManager implements AutoCloseable { private final Map pipelines = new ConcurrentHashMap<>(); + private final Object lifecycleLock = new Object(); + private volatile boolean closed; public void route(RowChangeEvent event) { - if (event == null) + if (event == null || closed) { return; } SchemaTableName table = new SchemaTableName(event.getSchemaName(), event.getTable()); - pipelines.computeIfAbsent(table, ignored -> + TablePipeline pipeline = pipelines.get(table); + if (pipeline == null) { - TablePipeline pipeline = new TablePipeline(); - pipeline.start(); - return pipeline; - }).publish(event); + synchronized (lifecycleLock) + { + if (closed) + { + return; + } + pipeline = pipelines.computeIfAbsent(table, ignored -> + { + TablePipeline newPipeline = new TablePipeline(); + newPipeline.start(); + return newPipeline; + }); + } + } + pipeline.publish(event); } @Override public void close() { - pipelines.values().forEach(TablePipeline::close); - pipelines.clear(); + List pipelinesToClose; + synchronized (lifecycleLock) + { + if (closed) + { + return; + } + closed = true; + pipelinesToClose = new ArrayList<>(pipelines.values()); + pipelines.clear(); + } + pipelinesToClose.forEach(TablePipeline::close); + } + + public void abort() + { + List pipelinesToAbort; + synchronized (lifecycleLock) + { + if (closed) + { + return; + } + closed = true; + pipelinesToAbort = new ArrayList<>(pipelines.values()); + pipelines.clear(); + } + pipelinesToAbort.forEach(TablePipeline::abort); } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TransactionPipeline.java b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TransactionPipeline.java index 1ccdb72..4d146c5 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TransactionPipeline.java +++ b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TransactionPipeline.java @@ -55,11 +55,32 @@ public void publish(SinkProto.TransactionMetadata transaction) eventQueue.put(transaction); } + /** + * Stops accepting transactions and waits for all pending transactions to + * be written. + */ @Override public void close() { - processor.stopProcessor(); eventQueue.close(); + try + { + processorThread.join(); + } catch (InterruptedException e) + { + abort(); + Thread.currentThread().interrupt(); + } + } + + /** + * Discards pending transactions and interrupts processing. Already written + * transactions are not rolled back. + */ + public void abort() + { + processor.abort(); + eventQueue.abort(); processorThread.interrupt(); } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/processor/MonitorThreadManager.java b/src/main/java/io/pixelsdb/pixels/sink/processor/MonitorThreadManager.java deleted file mode 100644 index a3686a4..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/processor/MonitorThreadManager.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.processor; - -import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; - -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -public class MonitorThreadManager -{ - private final List monitors = new CopyOnWriteArrayList<>(); - private final ExecutorService executor = Executors.newFixedThreadPool(PixelsSinkConstants.MONITOR_NUM); - - public void startMonitor(Runnable monitor) - { - monitors.add(monitor); - executor.submit(monitor); - } - - public void shutdown() - { - stopMonitors(); - shutdownExecutor(); - awaitTermination(); - } - - private void stopMonitors() - { - monitors.forEach(monitor -> - { - if (monitor instanceof StoppableProcessor) - { - ((StoppableProcessor) monitor).stopProcessor(); - } - }); - } - - private void shutdownExecutor() - { - executor.shutdown(); - } - - private void awaitTermination() - { - try - { - if (!executor.awaitTermination(10, TimeUnit.SECONDS)) - { - executor.shutdownNow(); - } - } catch (InterruptedException e) - { - executor.shutdownNow(); - } - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/processor/StoppableProcessor.java b/src/main/java/io/pixelsdb/pixels/sink/processor/StoppableProcessor.java deleted file mode 100644 index 9c42a8d..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/processor/StoppableProcessor.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - - -package io.pixelsdb.pixels.sink.processor; - -public interface StoppableProcessor -{ - void stopProcessor(); -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/processor/TableProcessor.java b/src/main/java/io/pixelsdb/pixels/sink/processor/TableProcessor.java index 71b9fb9..4e2583e 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/processor/TableProcessor.java +++ b/src/main/java/io/pixelsdb/pixels/sink/processor/TableProcessor.java @@ -37,7 +37,7 @@ * @author: AntiO2 * @date: 2025/9/26 11:01 */ -public class TableProcessor implements StoppableProcessor, Runnable +public class TableProcessor implements Runnable { private static final Logger LOGGER = LoggerFactory.getLogger(TableProcessor.class); private final AtomicBoolean running = new AtomicBoolean(true); @@ -74,17 +74,25 @@ private void processLoop() RowChangeEvent event = eventQueue.take(); if (event == null) { - continue; + break; } pixelsSinkWriter.writeRow(event); } + running.set(false); LOGGER.info("Processor thread exited"); } - @Override - public void stopProcessor() + public void awaitTermination() throws InterruptedException + { + if (processorThread != null) + { + processorThread.join(); + } + } + + public void abort() { - LOGGER.info("Stopping transaction monitor"); + LOGGER.info("Aborting table processor"); running.set(false); if (processorThread != null) { diff --git a/src/main/java/io/pixelsdb/pixels/sink/processor/TransactionProcessor.java b/src/main/java/io/pixelsdb/pixels/sink/processor/TransactionProcessor.java index a4153ed..175bf5b 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/processor/TransactionProcessor.java +++ b/src/main/java/io/pixelsdb/pixels/sink/processor/TransactionProcessor.java @@ -29,7 +29,7 @@ import java.util.concurrent.atomic.AtomicBoolean; -public class TransactionProcessor implements Runnable, StoppableProcessor +public class TransactionProcessor implements Runnable { private static final Logger LOGGER = LoggerFactory.getLogger(TransactionProcessor.class); private final PixelsSinkWriter sinkWriter; @@ -58,7 +58,6 @@ public void run() SinkProto.TransactionMetadata transaction = eventQueue.take(); if (transaction == null) { - LOGGER.warn("Received null transaction"); running.set(false); break; } @@ -67,10 +66,9 @@ public void run() LOGGER.info("Processor thread exited for transaction"); } - @Override - public void stopProcessor() + public void abort() { - LOGGER.info("Stopping transaction monitor"); + LOGGER.info("Aborting transaction processor"); running.set(false); } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/SinkSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/SinkSource.java index bf2bb4b..6854b7b 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/SinkSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/SinkSource.java @@ -21,18 +21,26 @@ package io.pixelsdb.pixels.sink.source; - -import io.pixelsdb.pixels.sink.processor.StoppableProcessor; - /** * @package: io.pixelsdb.pixels.sink.source * @className: SinkSource * @author: AntiO2 * @date: 2025/9/26 13:45 */ -public interface SinkSource extends StoppableProcessor +public interface SinkSource extends AutoCloseable { void start(); boolean isRunning(); + + /** + * Stops producing new events and waits for pending events to be processed. + */ + @Override + void close(); + + /** + * Stops immediately and discards events that have not been processed. + */ + void abort(); } diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumer.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumer.java index 97bb27e..ce8fff2 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumer.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumer.java @@ -32,7 +32,6 @@ import io.pixelsdb.pixels.sink.exception.SinkException; import io.pixelsdb.pixels.sink.pipeline.TablePipelineManager; import io.pixelsdb.pixels.sink.pipeline.TransactionPipeline; -import io.pixelsdb.pixels.sink.processor.StoppableProcessor; import io.pixelsdb.pixels.sink.source.engine.adapter.DebeziumStructAdapter; import io.pixelsdb.pixels.sink.source.engine.adapter.DebeziumSourceAdapterSelector; import io.pixelsdb.pixels.sink.util.MetricsFacade; @@ -50,7 +49,8 @@ * @author: AntiO2 * @date: 2025/9/25 12:51 */ -public class PixelsDebeziumConsumer implements DebeziumEngine.ChangeConsumer>, StoppableProcessor +public class PixelsDebeziumConsumer + implements DebeziumEngine.ChangeConsumer>, AutoCloseable { private static final Logger LOGGER = LoggerFactory.getLogger(PixelsDebeziumConsumer.class); @@ -233,9 +233,15 @@ private static String schemaName(org.apache.kafka.connect.data.Schema schema) } @Override - public void stopProcessor() + public void close() { tablePipelineManager.close(); transactionPipeline.close(); } + + public void abort() + { + tablePipelineManager.abort(); + transactionPipeline.abort(); + } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/SinkEngineSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/SinkEngineSource.java index cfb69ef..f2fd348 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/engine/SinkEngineSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/SinkEngineSource.java @@ -31,13 +31,16 @@ import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; public class SinkEngineSource implements SinkSource { + private static final long SHUTDOWN_TIMEOUT_SECONDS = 10; + private final PixelsDebeziumConsumer consumer; private DebeziumEngine> engine; private ExecutorService executor; - private volatile boolean running = true; + private volatile boolean running; public SinkEngineSource() { @@ -57,10 +60,11 @@ public void start() this.executor = Executors.newSingleThreadExecutor(); this.executor.execute(engine); + running = true; } @Override - public void stopProcessor() + public void close() { try { @@ -71,10 +75,30 @@ public void stopProcessor() if (executor != null) { executor.shutdown(); + if (!executor.awaitTermination( + SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + { + executor.shutdownNow(); + consumer.abort(); + return; + } } - consumer.stopProcessor(); + consumer.close(); + } catch (InterruptedException e) + { + if (executor != null) + { + executor.shutdownNow(); + } + consumer.abort(); + Thread.currentThread().interrupt(); } catch (Exception e) { + if (executor != null) + { + executor.shutdownNow(); + } + consumer.abort(); throw new RuntimeException("Failed to stop PixelsSinkEngine", e); } finally { @@ -82,6 +106,29 @@ public void stopProcessor() } } + @Override + public void abort() + { + running = false; + if (executor != null) + { + executor.shutdownNow(); + } + try + { + if (engine != null) + { + engine.close(); + } + } catch (Exception e) + { + throw new RuntimeException("Failed to abort PixelsSinkEngine", e); + } finally + { + consumer.abort(); + } + } + @Override public boolean isRunning() { diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaRowSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaRowSource.java index ee391a3..e315021 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaRowSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaRowSource.java @@ -25,7 +25,6 @@ import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; import io.pixelsdb.pixels.sink.event.RowChangeEvent; import io.pixelsdb.pixels.sink.pipeline.TablePipelineManager; -import io.pixelsdb.pixels.sink.processor.StoppableProcessor; import io.pixelsdb.pixels.sink.source.kafka.serde.KafkaRecordConverter; import io.pixelsdb.pixels.sink.util.DataTransform; import io.pixelsdb.pixels.sink.util.MetricsFacade; @@ -42,7 +41,7 @@ import java.util.Properties; import java.util.concurrent.atomic.AtomicBoolean; -public final class KafkaRowSource implements Runnable, StoppableProcessor +public final class KafkaRowSource implements Runnable { private static final Logger LOGGER = LoggerFactory.getLogger(KafkaRowSource.class); @@ -132,8 +131,7 @@ public void run() } } - @Override - public void stopProcessor() + void requestStop() { running.set(false); if (consumer != null) diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaTransactionSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaTransactionSource.java index 2e4ea69..aac33a0 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaTransactionSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaTransactionSource.java @@ -23,7 +23,6 @@ import io.pixelsdb.pixels.sink.SinkProto; import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; import io.pixelsdb.pixels.sink.pipeline.TransactionPipeline; -import io.pixelsdb.pixels.sink.processor.StoppableProcessor; import io.pixelsdb.pixels.sink.source.kafka.serde.KafkaRecordConverter; import io.pixelsdb.pixels.sink.util.MetricsFacade; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -38,7 +37,7 @@ import java.util.Properties; import java.util.concurrent.atomic.AtomicBoolean; -public final class KafkaTransactionSource implements Runnable, StoppableProcessor +public final class KafkaTransactionSource implements Runnable { private static final Logger LOGGER = LoggerFactory.getLogger(KafkaTransactionSource.class); @@ -112,8 +111,7 @@ public void run() } } - @Override - public void stopProcessor() + void requestStop() { running.set(false); consumer.wakeup(); diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/SinkKafkaSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/SinkKafkaSource.java index da9a790..8721b66 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/SinkKafkaSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/SinkKafkaSource.java @@ -24,16 +24,22 @@ import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; import io.pixelsdb.pixels.sink.config.factory.KafkaPropFactorySelector; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import io.pixelsdb.pixels.sink.processor.MonitorThreadManager; import io.pixelsdb.pixels.sink.pipeline.TablePipelineManager; import io.pixelsdb.pixels.sink.pipeline.TransactionPipeline; import io.pixelsdb.pixels.sink.source.SinkSource; import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; public class SinkKafkaSource implements SinkSource { - private MonitorThreadManager manager; + private static final long SHUTDOWN_TIMEOUT_SECONDS = 10; + + private ExecutorService sourceExecutor; + private KafkaTransactionSource transactionSource; + private TopicProcessor topicProcessor; private TablePipelineManager tablePipelineManager; private TransactionPipeline transactionPipeline; private volatile boolean running; @@ -51,42 +57,105 @@ public void start() pixelsSinkConfig.getTransactionTopicSuffix(); tablePipelineManager = new TablePipelineManager(); transactionPipeline = new TransactionPipeline(); - KafkaTransactionSource transactionSource = + transactionSource = new KafkaTransactionSource( transactionKafkaProperties, transactionTopic, transactionPipeline); Properties topicKafkaProperties = kafkaPropFactorySelector .getFactory(PixelsSinkConstants.ROW_RECORD_KAFKA_PROP_FACTORY) .createKafkaProperties(pixelsSinkConfig); - TopicProcessor topicMonitor = new TopicProcessor( + topicProcessor = new TopicProcessor( pixelsSinkConfig, topicKafkaProperties, tablePipelineManager); transactionPipeline.start(); - manager = new MonitorThreadManager(); - manager.startMonitor(transactionSource); - manager.startMonitor(topicMonitor); + sourceExecutor = Executors.newFixedThreadPool(2); + sourceExecutor.submit(transactionSource); + sourceExecutor.submit(topicProcessor); running = true; } @Override - public void stopProcessor() + public void close() { - if (manager != null) + if (transactionSource != null) + { + transactionSource.requestStop(); + } + if (topicProcessor != null) + { + topicProcessor.requestStop(); + } + + boolean sourcesStopped = true; + if (sourceExecutor != null) { - manager.shutdown(); + sourceExecutor.shutdown(); + try + { + if (!sourceExecutor.awaitTermination( + SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + { + sourceExecutor.shutdownNow(); + sourcesStopped = false; + } + } catch (InterruptedException e) + { + sourceExecutor.shutdownNow(); + Thread.currentThread().interrupt(); + sourcesStopped = false; + } } + if (transactionPipeline != null) { - transactionPipeline.close(); + if (sourcesStopped) + { + transactionPipeline.close(); + } else + { + transactionPipeline.abort(); + } } if (tablePipelineManager != null) { - tablePipelineManager.close(); + if (sourcesStopped) + { + tablePipelineManager.close(); + } else + { + tablePipelineManager.abort(); + } } running = false; } + @Override + public void abort() + { + running = false; + if (transactionSource != null) + { + transactionSource.requestStop(); + } + if (topicProcessor != null) + { + topicProcessor.abort(); + } + if (sourceExecutor != null) + { + sourceExecutor.shutdownNow(); + } + if (transactionPipeline != null) + { + transactionPipeline.abort(); + } + if (tablePipelineManager != null) + { + tablePipelineManager.abort(); + } + } + @Override public boolean isRunning() { diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/TopicProcessor.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/TopicProcessor.java index 7867b3c..67dc768 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/TopicProcessor.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/TopicProcessor.java @@ -22,7 +22,6 @@ import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; import io.pixelsdb.pixels.sink.pipeline.TablePipelineManager; -import io.pixelsdb.pixels.sink.processor.StoppableProcessor; import org.apache.kafka.clients.admin.AdminClient; import org.apache.kafka.clients.admin.AdminClientConfig; import org.slf4j.Logger; @@ -43,7 +42,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; -public final class TopicProcessor implements Runnable, StoppableProcessor +public final class TopicProcessor implements Runnable { private static final Logger LOGGER = LoggerFactory.getLogger(TopicProcessor.class); @@ -57,6 +56,7 @@ public final class TopicProcessor implements Runnable, StoppableProcessor private final Map activeSources = new ConcurrentHashMap<>(); private final ExecutorService executor = Executors.newCachedThreadPool(); private final AtomicBoolean running = new AtomicBoolean(true); + private final Object sourceLifecycleLock = new Object(); private AdminClient adminClient; private Timer timer; @@ -101,12 +101,11 @@ public void run() } } finally { - stopProcessor(); + requestStop(); } } - @Override - public void stopProcessor() + void requestStop() { if (!running.compareAndSet(true, false)) { @@ -120,9 +119,12 @@ public void stopProcessor() { adminClient.close(Duration.ofSeconds(5)); } - activeSources.values().forEach(KafkaRowSource::stopProcessor); - activeSources.clear(); - executor.shutdown(); + synchronized (sourceLifecycleLock) + { + activeSources.values().forEach(KafkaRowSource::requestStop); + activeSources.clear(); + executor.shutdown(); + } try { if (!executor.awaitTermination(10, TimeUnit.SECONDS)) @@ -136,6 +138,25 @@ public void stopProcessor() } } + void abort() + { + running.set(false); + if (timer != null) + { + timer.cancel(); + } + if (adminClient != null) + { + adminClient.close(Duration.ofSeconds(5)); + } + synchronized (sourceLifecycleLock) + { + activeSources.values().forEach(KafkaRowSource::requestStop); + activeSources.clear(); + executor.shutdownNow(); + } + } + private class TopicMonitorTask extends TimerTask { @Override @@ -167,11 +188,18 @@ public void run() private void startTopic(String topic) { - KafkaRowSource source = new KafkaRowSource( - kafkaProperties, topic, tablePipelineManager); - activeSources.put(topic, source); - subscribedTopics.add(topic); - executor.submit(source); + synchronized (sourceLifecycleLock) + { + if (!running.get() || executor.isShutdown()) + { + return; + } + KafkaRowSource source = new KafkaRowSource( + kafkaProperties, topic, tablePipelineManager); + activeSources.put(topic, source); + subscribedTopics.add(topic); + executor.submit(source); + } } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java index dd1b525..977ebe1 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java @@ -48,18 +48,34 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; public abstract class AbstractSinkStorageSource implements SinkSource { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractSinkStorageSource.class); + private static final int DECODE_BATCH_SIZE = 64; + private static final int DECODE_THREAD_COUNT = 4; + private static final long DECODE_BATCH_WAIT_MILLIS = 5; + private static final long DECODE_SHUTDOWN_TIMEOUT_SECONDS = 30; protected static final int RECORD_HEADER_SIZE = Integer.BYTES * 2; protected final AtomicBoolean running = new AtomicBoolean(false); + private final AtomicBoolean started = new AtomicBoolean(false); + private final AtomicBoolean abortRequested = new AtomicBoolean(false); + private final CountDownLatch stopped = new CountDownLatch(1); + private volatile Thread sourceThread; protected final String topic; protected final String baseDir; @@ -77,6 +93,17 @@ public abstract class AbstractSinkStorageSource implements SinkSource new TransactionMetadataConverter(); protected final boolean freshnessTimestamp; private final MetricsFacade metricsFacade = MetricsFacade.getInstance(); + private final AtomicInteger decoderThreadId = new AtomicInteger(); + private final ExecutorService decodeExecutor = new ThreadPoolExecutor( + DECODE_THREAD_COUNT, + DECODE_THREAD_COUNT, + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(PixelsSinkConstants.MAX_QUEUE_SIZE), + runnable -> new Thread( + runnable, + "storage-proto-decoder-" + decoderThreadId.incrementAndGet()), + new ThreadPoolExecutor.CallerRunsPolicy()); protected int loopId = 0; protected List readers = new ArrayList<>(); @@ -93,6 +120,17 @@ protected AbstractSinkStorageSource() this.sourceRateLimiter = FlushRateLimiterFactory.getNewInstance(); } + protected void beginProcessing() + { + if (!started.compareAndSet(false, true)) + { + throw new IllegalStateException("Storage source has already been started"); + } + sourceThread = Thread.currentThread(); + running.set(true); + transactionPipeline.start(); + } + ProtoType getProtoType(int key) { return key == -1 ? ProtoType.TRANS : ProtoType.ROW; @@ -153,84 +191,262 @@ protected void submitRecord(int key, ByteBuffer valueBuffer, int recordLoopId) protected void clean() { - running.set(false); - queueMap.values().forEach(q -> + boolean interrupted = false; + try { - try + running.set(false); + if (!abortRequested.get()) { - q.put(new Pair<>(POISON_PILL, loopId)); - } catch (InterruptedException e) + for (BlockingQueue, Integer>> queue + : queueMap.values()) + { + try + { + queue.put(new Pair<>(POISON_PILL, loopId)); + } catch (InterruptedException e) + { + interrupted = true; + abortRequested.set(true); + break; + } + } + } + if (abortRequested.get()) { - Thread.currentThread().interrupt(); + consumerThreads.values().forEach(Thread::interrupt); } - }); - consumerThreads.values().forEach(t -> - { - try + for (Thread thread : consumerThreads.values()) { - t.join(); - } catch (InterruptedException e) + while (thread.isAlive()) + { + try + { + thread.join(); + } catch (InterruptedException e) + { + interrupted = true; + abortRequested.set(true); + consumerThreads.values().forEach(Thread::interrupt); + } + } + } + + if (abortRequested.get()) { - Thread.currentThread().interrupt(); + decodeExecutor.shutdownNow(); + } else + { + shutdownDecodeExecutor(); } - }); - for (PhysicalReader reader : readers) - { - try + for (PhysicalReader reader : readers) { - reader.close(); - } catch (IOException e) + try + { + reader.close(); + } catch (IOException e) + { + LOGGER.warn("Failed to close reader", e); + } + } + if (abortRequested.get()) { - LOGGER.warn("Failed to close reader", e); + tablePipelineManager.abort(); + transactionPipeline.abort(); + } else + { + tablePipelineManager.close(); + transactionPipeline.close(); + } + } finally + { + sourceThread = null; + stopped.countDown(); + if (interrupted) + { + Thread.currentThread().interrupt(); } } - tablePipelineManager.close(); - transactionPipeline.close(); } - protected void handleTransactionSourceRecord(ByteBuffer record, Integer loopId) + private void shutdownDecodeExecutor() { + decodeExecutor.shutdown(); try { - SinkProto.TransactionMetadata metadata = - transactionMetadataConverter.convert(record, loopId); - metricsFacade.recordSerdTxChange(); - transactionPipeline.publish(metadata); - } catch (Exception e) + if (!decodeExecutor.awaitTermination( + DECODE_SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + { + LOGGER.warn("Timed out waiting for storage proto decoders to stop"); + decodeExecutor.shutdownNow(); + } + } catch (InterruptedException e) { - LOGGER.warn("Failed to convert storage transaction metadata", e); + decodeExecutor.shutdownNow(); + Thread.currentThread().interrupt(); } } protected void consumeQueue(int key, BlockingQueue, Integer>> queue, ProtoType protoType) { + boolean stopAfterBatch = false; try { - while (true) + while (!stopAfterBatch) { - Pair, Integer> pair = queue.take(); - CompletableFuture value = pair.getLeft(); - int loopId = pair.getRight(); - if (value == POISON_PILL) + List, Integer>> batch = + new ArrayList<>(DECODE_BATCH_SIZE); + Pair, Integer> first = queue.take(); + if (isPoisonPill(first)) { break; } - ByteBuffer valueBuffer = value.get(); - metricsFacade.recordDebeziumEvent(); - switch (protoType) + batch.add(first); + + long deadline = System.nanoTime() + + TimeUnit.MILLISECONDS.toNanos(DECODE_BATCH_WAIT_MILLIS); + while (batch.size() < DECODE_BATCH_SIZE) { - case ROW -> handleRowChangeSourceRecord(key, valueBuffer, loopId); - case TRANS -> handleTransactionSourceRecord(valueBuffer, loopId); + long remainingNanos = deadline - System.nanoTime(); + if (remainingNanos <= 0) + { + break; + } + + Pair, Integer> next = + queue.poll(remainingNanos, TimeUnit.NANOSECONDS); + if (next == null) + { + break; + } + if (isPoisonPill(next)) + { + stopAfterBatch = true; + break; + } + batch.add(next); } + + processBatch(key, batch, protoType); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); + } + } + + private boolean isPoisonPill( + Pair, Integer> record) + { + return record.getLeft() == POISON_PILL; + } + + private void processBatch( + int key, + List, Integer>> batch, + ProtoType protoType) throws InterruptedException + { + for (int i = 0; i < batch.size(); ++i) + { + metricsFacade.recordDebeziumEvent(); + } + + switch (protoType) + { + case ROW -> + { + List events = decodeInOrder( + decodeExecutor, + batch, + record -> decodeRowChangeSourceRecord(key, record)); + for (RowChangeEvent event : events) + { + if (event != null) + { + metricsFacade.recordSerdRowChange(); + tablePipelineManager.route(event); + } + } + } + case TRANS -> + { + List transactions = decodeInOrder( + decodeExecutor, + batch, + this::decodeTransactionSourceRecord); + for (SinkProto.TransactionMetadata transaction : transactions) + { + if (transaction != null) + { + metricsFacade.recordSerdTxChange(); + transactionPipeline.publish(transaction); + } + } + } + } + } + + static List decodeInOrder( + ExecutorService executor, + List records, + Function decoder) throws InterruptedException + { + List> futures = new ArrayList<>(records.size()); + for (T record : records) + { + futures.add(executor.submit(() -> decoder.apply(record))); + } + + List decodedRecords = new ArrayList<>(records.size()); + for (Future future : futures) + { + try + { + decodedRecords.add(future.get()); + } catch (ExecutionException e) + { + LOGGER.warn("Failed to decode storage record", e.getCause()); + decodedRecords.add(null); + } + } + return decodedRecords; + } + + private RowChangeEvent decodeRowChangeSourceRecord( + int key, + Pair, Integer> record) + { + try + { + return convertRowChangeSourceRecord( + key, record.getLeft().get(), record.getRight()); + } catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + return null; } catch (ExecutionException e) { - LOGGER.error("Error in async processing", e); + LOGGER.warn("Failed to read storage row record", e.getCause()); + return null; + } + } + + private SinkProto.TransactionMetadata decodeTransactionSourceRecord( + Pair, Integer> record) + { + try + { + return transactionMetadataConverter.convert( + record.getLeft().get(), record.getRight()); + } catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + return null; + } catch (Exception e) + { + LOGGER.warn("Failed to convert storage transaction metadata", e); + return null; } } @@ -243,7 +459,8 @@ protected ByteBuffer copyToHeap(ByteBuffer directBuffer) return heapBuffer; } - protected void handleRowChangeSourceRecord(int key, ByteBuffer dataBuffer, int loopId) + protected RowChangeEvent convertRowChangeSourceRecord( + int key, ByteBuffer dataBuffer, int loopId) { try { @@ -260,12 +477,11 @@ protected void handleRowChangeSourceRecord(int key, ByteBuffer dataBuffer, int l .setId(transaction.getId() + "_" + loopId) .build()); } - RowChangeEvent event = rowRecordConverter.convert(builder.build()); - metricsFacade.recordSerdRowChange(); - tablePipelineManager.route(event); + return rowRecordConverter.convert(builder.build()); } catch (Exception e) { LOGGER.warn("Failed to convert storage row record", e); + return null; } } @@ -276,10 +492,46 @@ public boolean isRunning() } @Override - public void stopProcessor() + public void close() { running.set(false); - tablePipelineManager.close(); - transactionPipeline.close(); + if (!started.get() || Thread.currentThread() == sourceThread) + { + return; + } + + boolean interrupted = false; + while (true) + { + try + { + stopped.await(); + break; + } catch (InterruptedException e) + { + interrupted = true; + } + } + if (interrupted) + { + Thread.currentThread().interrupt(); + } + } + + @Override + public void abort() + { + abortRequested.set(true); + running.set(false); + consumerThreads.values().forEach(Thread::interrupt); + decodeExecutor.shutdownNow(); + tablePipelineManager.abort(); + transactionPipeline.abort(); + + Thread thread = sourceThread; + if (thread != null && thread != Thread.currentThread()) + { + thread.interrupt(); + } } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/MemorySinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/MemorySinkStorageSource.java index 9d67189..bc34b5c 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/storage/MemorySinkStorageSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/storage/MemorySinkStorageSource.java @@ -43,8 +43,7 @@ public class MemorySinkStorageSource extends AbstractSinkStorageSource @Override public void start() { - this.running.set(true); - this.transactionPipeline.start(); + beginProcessing(); try { /* ===================================================== @@ -52,6 +51,10 @@ public void start() * ===================================================== */ for (String file : files) { + if (!isRunning()) + { + break; + } Storage.Scheme scheme = Storage.Scheme.fromPath(file); LOGGER.info("Preloading file {}", file); @@ -61,7 +64,7 @@ public void start() reader.seek(0); long offset = 0; long fileLength = reader.getFileLength(); - while (offset < fileLength) + while (isRunning() && offset < fileLength) { Pair record = readRecord(reader, offset, fileLength); int valueLength = record.getRight().remaining(); @@ -85,6 +88,10 @@ public void start() { for (Pair record : preloadedRecords) { + if (!isRunning()) + { + break; + } int key = record.getLeft(); ByteBuffer src = record.getRight(); ByteBuffer copy = ByteBuffer.allocate(src.remaining()); diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/StreamingSinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/StreamingSinkStorageSource.java index 8201a10..81bb151 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/storage/StreamingSinkStorageSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/storage/StreamingSinkStorageSource.java @@ -37,12 +37,15 @@ public class StreamingSinkStorageSource extends AbstractSinkStorageSource @Override public void start() { - this.running.set(true); - this.transactionPipeline.start(); + beginProcessing(); try { for (String file : files) { + if (!isRunning()) + { + break; + } Storage.Scheme scheme = Storage.Scheme.fromPath(file); readers.add(PhysicalReaderUtil.newPhysicalReader(scheme, file)); } diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueue.java b/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueue.java index eb47f2b..5aad626 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueue.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueue.java @@ -27,8 +27,8 @@ /** * A bounded blocking queue with an explicit shutdown signal. * - *

Closing the queue discards pending values and wakes a blocked consumer. - * This is intended for pipeline shutdown, not graceful draining.

+ *

Closing the queue finishes it gracefully. Use {@link #abort()} to + * discard pending values and wake a blocked consumer immediately.

*/ public final class BlockingBoundedQueue implements Closeable { @@ -79,6 +79,12 @@ public T take() } } + /** + * Stops accepting new values and wakes the consumer after all queued values + * have been consumed. + * + *

The caller must stop all producers before invoking this method.

+ */ @Override public void close() { @@ -86,6 +92,31 @@ public void close() { return; } + closed = true; + boolean interrupted = false; + while (true) + { + try + { + queue.put(POISON_PILL); + break; + } catch (InterruptedException e) + { + interrupted = true; + } + } + if (interrupted) + { + Thread.currentThread().interrupt(); + } + } + + /** + * Stops accepting new values, discards pending values, and wakes a blocked + * consumer. Values already consumed are not rolled back. + */ + public void abort() + { closed = true; queue.clear(); queue.offer(POISON_PILL); diff --git a/src/test/java/io/pixelsdb/pixels/sink/pipeline/PipelineTest.java b/src/test/java/io/pixelsdb/pixels/sink/pipeline/PipelineTest.java index 415b690..d31de7a 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/pipeline/PipelineTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/pipeline/PipelineTest.java @@ -59,4 +59,21 @@ void shouldStartTransactionPipelineOnlyOnce() throws Exception }); } + @Test + void shouldAbortPipelines() throws Exception + { + TestConfig.initializeUnitConfig(); + + assertDoesNotThrow(() -> { + try (TablePipeline tablePipeline = new TablePipeline(); + TransactionPipeline transactionPipeline = new TransactionPipeline()) + { + tablePipeline.start(); + transactionPipeline.start(); + tablePipeline.abort(); + transactionPipeline.abort(); + } + }); + } + } diff --git a/src/test/java/io/pixelsdb/pixels/sink/processor/TableProcessorTest.java b/src/test/java/io/pixelsdb/pixels/sink/processor/TableProcessorTest.java index 3ff6ab3..2910840 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/processor/TableProcessorTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/processor/TableProcessorTest.java @@ -30,10 +30,11 @@ import java.io.IOException; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.List; -import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; class TableProcessorTest @@ -42,22 +43,27 @@ class TableProcessorTest void shouldForwardRowsToWriter() throws Exception { TestConfig.initializeUnitConfig(); - BlockingBoundedQueue queue = new BlockingBoundedQueue<>(1); + BlockingBoundedQueue queue = new BlockingBoundedQueue<>(2); RecordingWriter writer = new RecordingWriter(); TableProcessor processor = new TableProcessor(queue, writer); - RowChangeEvent event = rowEvent(); + RowChangeEvent first = rowEvent(); + RowChangeEvent second = rowEvent(); try { processor.run(); - queue.put(event); + queue.put(first); + queue.put(second); + queue.close(); + processor.awaitTermination(); - assertTrue(writer.rowWritten.await(1, TimeUnit.SECONDS)); - assertSame(event, writer.row); + assertTrue(writer.rowsWritten.await(1, TimeUnit.SECONDS)); + assertEquals(List.of(first, second), writer.rows); } finally { - processor.stopProcessor(); - queue.close(); + processor.abort(); + queue.abort(); + processor.awaitTermination(); } } @@ -79,8 +85,8 @@ private static RowChangeEvent rowEvent() throws Exception private static final class RecordingWriter implements PixelsSinkWriter { - private final CountDownLatch rowWritten = new CountDownLatch(1); - private RowChangeEvent row; + private final CountDownLatch rowsWritten = new CountDownLatch(2); + private final List rows = new CopyOnWriteArrayList<>(); @Override public void flush() @@ -90,8 +96,8 @@ public void flush() @Override public boolean writeRow(RowChangeEvent rowChangeEvent) { - row = rowChangeEvent; - rowWritten.countDown(); + rows.add(rowChangeEvent); + rowsWritten.countDown(); return true; } diff --git a/src/test/java/io/pixelsdb/pixels/sink/processor/TransactionProcessorTest.java b/src/test/java/io/pixelsdb/pixels/sink/processor/TransactionProcessorTest.java index a18f9f3..0596d37 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/processor/TransactionProcessorTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/processor/TransactionProcessorTest.java @@ -26,10 +26,13 @@ import org.junit.jupiter.api.Test; import java.io.IOException; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; class TransactionProcessorTest @@ -38,35 +41,42 @@ class TransactionProcessorTest void shouldForwardTransactionsAndStop() throws Exception { BlockingBoundedQueue queue = - new BlockingBoundedQueue<>(1); + new BlockingBoundedQueue<>(2); RecordingWriter writer = new RecordingWriter(); TransactionProcessor processor = new TransactionProcessor(queue, writer); Thread processorThread = new Thread(processor, "transaction-processor-test"); - SinkProto.TransactionMetadata transaction = SinkProto.TransactionMetadata.newBuilder() + SinkProto.TransactionMetadata first = SinkProto.TransactionMetadata.newBuilder() .setId("transaction-1") .build(); + SinkProto.TransactionMetadata second = SinkProto.TransactionMetadata.newBuilder() + .setId("transaction-2") + .build(); try { processorThread.start(); - queue.put(transaction); + queue.put(first); + queue.put(second); + queue.close(); + processorThread.join(1000); - assertTrue(writer.transactionWritten.await(1, TimeUnit.SECONDS)); - assertSame(transaction, writer.transaction); + assertTrue(writer.transactionsWritten.await(1, TimeUnit.SECONDS)); + assertEquals(List.of(first, second), writer.transactions); + assertFalse(processorThread.isAlive()); } finally { - processor.stopProcessor(); - queue.close(); + processor.abort(); + queue.abort(); processorThread.interrupt(); processorThread.join(1000); } - assertTrue(!processorThread.isAlive()); } private static final class RecordingWriter implements PixelsSinkWriter { - private final CountDownLatch transactionWritten = new CountDownLatch(1); - private SinkProto.TransactionMetadata transaction; + private final CountDownLatch transactionsWritten = new CountDownLatch(2); + private final List transactions = + new CopyOnWriteArrayList<>(); @Override public void flush() @@ -82,8 +92,8 @@ public boolean writeRow(io.pixelsdb.pixels.sink.event.RowChangeEvent rowChangeEv @Override public boolean writeTrans(SinkProto.TransactionMetadata transactionMetadata) { - transaction = transactionMetadata; - transactionWritten.countDown(); + transactions.add(transactionMetadata); + transactionsWritten.countDown(); return true; } diff --git a/src/test/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueueTest.java b/src/test/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueueTest.java index 162d1b5..116a033 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueueTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueueTest.java @@ -69,15 +69,29 @@ void shouldWakeConsumerWhenClosed() } @Test - void shouldDiscardPendingValuesWhenClosed() + void shouldDiscardPendingValuesWhenAborted() { BlockingBoundedQueue queue = new BlockingBoundedQueue<>(1); queue.put(7); queue.put(null); - queue.close(); + queue.abort(); assertNull(queue.take()); queue.close(); } + + @Test + void shouldDrainPendingValuesWhenClosed() + { + try (BlockingBoundedQueue queue = new BlockingBoundedQueue<>(2)) + { + queue.put(7); + + queue.close(); + + assertEquals(7, queue.take()); + assertNull(queue.take()); + } + } } diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/RpcEndToEndTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/RpcEndToEndTest.java index 33da52e..8bfaf95 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/RpcEndToEndTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/RpcEndToEndTest.java @@ -229,10 +229,10 @@ public static void main(String[] args) throws InterruptedException, IOException System.out.println("[CLEANUP] Tearing down server components..."); if (sinkSource != null) { - sinkSource.stopProcessor(); - System.out.println("[CLEANUP] SinkSource processor stopped."); + sinkSource.close(); + System.out.println("[CLEANUP] SinkSource closed."); // Note: The writer is part of sinkSource, and its resources - // should be cleaned up by stopProcessor or an equivalent close method. + // should be cleaned up by close. } if (prometheusHttpServer != null) { From 67bc731107b8fab0ef4a0ee4c73c3102dc3ffad5 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Fri, 31 Jul 2026 23:34:19 +0800 Subject: [PATCH 09/13] refactor: parallelize source decode and select Debezium format by config --- conf/pixels-sink.aws.properties | 5 +- conf/pixels-sink.ch.properties | 5 +- conf/pixels-sink.mysql.properties | 4 +- conf/pixels-sink.pg.properties | 6 +- docs/configuration.md | 22 +- docs/overview.md | 32 ++- .../pixels/sink/config/EngineValueFormat.java | 57 ++++ .../pixels/sink/config/KafkaValueFormat.java | 56 ++++ .../pixels/sink/config/PixelsSinkConfig.java | 133 +++++++++- .../sink/config/PixelsSinkConstants.java | 3 +- .../sink/config/PixelsSinkDefaultConfig.java | 1 + .../pixels/sink/config/TransactionConfig.java | 3 - .../factory/RowRecordKafkaPropFactory.java | 3 +- .../factory/TransactionKafkaPropFactory.java | 2 - ...ializer.java => DebeziumRowConverter.java} | 21 +- ...java => DebeziumTransactionConverter.java} | 21 +- .../TransactionMetadataAvroDeserializer.java | 31 --- .../TransactionMetadataJsonDeserializer.java | 31 --- .../avro/DebeziumAvroPayloadDecoder.java | 55 ++++ .../{ => avro}/DebeziumAvroRowConverter.java | 17 +- .../DebeziumAvroTransactionConverter.java | 13 +- .../DebeziumConnectRowConverter.java} | 62 ++--- .../DebeziumConnectTransactionConverter.java | 54 ++++ .../DebeziumSourceAdapter.java | 2 +- .../DebeziumSourceAdapterRegistry.java | 2 +- .../DebeziumSourceMetadata.java | 2 +- .../MySqlSourceAdapter.java | 2 +- .../PostgresSourceAdapter.java | 2 +- .../{ => json}/DebeziumJsonRowConverter.java | 17 +- .../DebeziumJsonTransactionConverter.java | 57 ++++ .../DebeziumEnvelopeNormalizer.java | 8 +- .../{ => support}/DebeziumRecordUtil.java | 2 +- .../DebeziumRowRecordAssembler.java | 2 +- .../DebeziumRowValueConverter.java | 2 +- .../kafka/KafkaRecordConverter.java | 58 ---- .../source/engine/ConnectEventClassifier.java | 110 ++++++++ .../engine/DebeziumEventClassifier.java | 34 +++ .../source/engine/DebeziumRecordType.java | 32 +++ .../source/engine/PixelsDebeziumConsumer.java | 202 +++++++------- .../sink/source/engine/SinkEngineSource.java | 7 +- .../DebeziumSourceAdapterSelector.java | 7 +- .../engine/adapter/DebeziumStructAdapter.java | 58 ---- .../sink/source/kafka/KafkaRowSource.java | 4 +- .../source/kafka/KafkaTransactionSource.java | 4 +- .../kafka/serde/KafkaRecordConverter.java | 185 ++++++++++--- .../serde/RowChangeEventAvroDeserializer.java | 79 ------ .../serde/RowChangeEventJsonDeserializer.java | 78 ------ .../TransactionMetadataAvroDeserializer.java | 78 ------ .../TransactionMetadataJsonDeserializer.java | 78 ------ .../storage/AbstractSinkStorageSource.java | 72 +---- .../sink/util/concurrent/DecodeExecutors.java | 99 +++++++ .../util/concurrent/OrderedBatchDecoder.java | 70 +++++ .../util/concurrent/StreamOrderedDecoder.java | 250 ++++++++++++++++++ ...ion.debezium.dialect.DebeziumSourceAdapter | 2 + ...sion.debezium.source.DebeziumSourceAdapter | 2 - src/main/resources/pixels-sink.aws.properties | 5 +- .../resources/pixels-sink.local.properties | 5 +- src/main/resources/pixels-sink.properties | 3 +- .../resources/pixels-sink.properties.template | 4 +- .../sink/config/EngineValueFormatTest.java | 38 +++ .../sink/config/KafkaValueFormatTest.java | 42 +++ .../LegacyKafkaValueFormatMigrationTest.java | 90 +++++++ .../sink/consumer/AvroConsumerTest.java | 162 +++--------- .../connect/DebeziumConnectConverterTest.java | 202 ++++++++++++++ .../DebeziumJsonConverterTest.java} | 160 ++++++----- .../DebeziumEnvelopeNormalizerTest.java | 53 +--- .../DebeziumRowValueConverterTest.java | 34 +-- .../kafka/KafkaRecordConverterTest.java | 56 ---- ...t.java => ConnectEventClassifierTest.java} | 85 ++---- .../kafka/serde/KafkaRecordConverterTest.java | 79 ++++++ .../concurrent/OrderedBatchDecoderTest.java | 80 ++++++ .../concurrent/StreamOrderedDecoderTest.java | 137 ++++++++++ 72 files changed, 2261 insertions(+), 1218 deletions(-) create mode 100644 src/main/java/io/pixelsdb/pixels/sink/config/EngineValueFormat.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/config/KafkaValueFormat.java rename src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/{RowChangeEventAvroDeserializer.java => DebeziumRowConverter.java} (52%) rename src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/{RowChangeEventJsonDeserializer.java => DebeziumTransactionConverter.java} (52%) delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/TransactionMetadataAvroDeserializer.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/TransactionMetadataJsonDeserializer.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroPayloadDecoder.java rename src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/{ => avro}/DebeziumAvroRowConverter.java (86%) rename src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/{ => avro}/DebeziumAvroTransactionConverter.java (78%) rename src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/{RowChangeEventStructConverter.java => connect/DebeziumConnectRowConverter.java} (76%) create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectTransactionConverter.java rename src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/{source => dialect}/DebeziumSourceAdapter.java (94%) rename src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/{source => dialect}/DebeziumSourceAdapterRegistry.java (96%) rename src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/{source => dialect}/DebeziumSourceMetadata.java (88%) rename src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/{source => dialect}/MySqlSourceAdapter.java (94%) rename src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/{source => dialect}/PostgresSourceAdapter.java (95%) rename src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/{ => json}/DebeziumJsonRowConverter.java (87%) create mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonTransactionConverter.java rename src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/{ => support}/DebeziumEnvelopeNormalizer.java (93%) rename src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/{ => support}/DebeziumRecordUtil.java (98%) rename src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/{ => support}/DebeziumRowRecordAssembler.java (97%) rename src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/{ => support}/DebeziumRowValueConverter.java (99%) delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/conversion/kafka/KafkaRecordConverter.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifier.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumEventClassifier.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumRecordType.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumStructAdapter.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowChangeEventAvroDeserializer.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowChangeEventJsonDeserializer.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataAvroDeserializer.java delete mode 100644 src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataJsonDeserializer.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/util/concurrent/DecodeExecutors.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoder.java create mode 100644 src/main/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoder.java create mode 100644 src/main/resources/META-INF/services/io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter delete mode 100644 src/main/resources/META-INF/services/io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter create mode 100644 src/test/java/io/pixelsdb/pixels/sink/config/EngineValueFormatTest.java create mode 100644 src/test/java/io/pixelsdb/pixels/sink/config/KafkaValueFormatTest.java create mode 100644 src/test/java/io/pixelsdb/pixels/sink/config/LegacyKafkaValueFormatMigrationTest.java create mode 100644 src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectConverterTest.java rename src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/{RowChangeEventJsonDeserializerTest.java => json/DebeziumJsonConverterTest.java} (52%) rename src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/{ => support}/DebeziumEnvelopeNormalizerTest.java (77%) rename src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/{ => support}/DebeziumRowValueConverterTest.java (82%) delete mode 100644 src/test/java/io/pixelsdb/pixels/sink/conversion/kafka/KafkaRecordConverterTest.java rename src/test/java/io/pixelsdb/pixels/sink/source/engine/{PixelsDebeziumConsumerTest.java => ConnectEventClassifierTest.java} (72%) create mode 100644 src/test/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverterTest.java create mode 100644 src/test/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoderTest.java create mode 100644 src/test/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoderTest.java diff --git a/conf/pixels-sink.aws.properties b/conf/pixels-sink.aws.properties index fb1ae99..29bd289 100644 --- a/conf/pixels-sink.aws.properties +++ b/conf/pixels-sink.aws.properties @@ -42,8 +42,7 @@ bootstrap.servers=realtime-kafka-2:29092 group.id=3078 auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -#value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventAvroDeserializer -value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer +sink.kafka.value.format=json # Topic & Database Config topic.prefix=postgresql.oltp_server consumer.capture_database=pixels_bench_sf1x @@ -79,8 +78,6 @@ sink.flink.server.port=9091 sink.registry.url=http://localhost:8080/apis/registry/v2 # Transaction Config transaction.topic.suffix=transaction -#transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataAvroDeserializer -transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer sink.trans.batch.size=100 # Sink Metrics diff --git a/conf/pixels-sink.ch.properties b/conf/pixels-sink.ch.properties index 5f68942..d93fcd4 100644 --- a/conf/pixels-sink.ch.properties +++ b/conf/pixels-sink.ch.properties @@ -43,8 +43,7 @@ bootstrap.servers=realtime-kafka-2:29092 group.id=3078 auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -#value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventAvroDeserializer -value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer +sink.kafka.value.format=json # Topic & Database Config topic.prefix=postgresql.oltp_server consumer.capture_database=pixels_bench_sf1x @@ -76,8 +75,6 @@ sink.flink.server.port=9091 sink.registry.url=http://localhost:8080/apis/registry/v2 # Transaction Config transaction.topic.suffix=transaction -#transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataAvroDeserializer -transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer sink.trans.batch.size=100 # Sink Metrics diff --git a/conf/pixels-sink.mysql.properties b/conf/pixels-sink.mysql.properties index a119151..3887095 100644 --- a/conf/pixels-sink.mysql.properties +++ b/conf/pixels-sink.mysql.properties @@ -1,5 +1,6 @@ # engine | kafka | storage sink.datasource=engine +sink.datasource.engine.format=connect sink.storage.mode=stream # -1 means no limit, Only implement in retina sink mode yet sink.datasource.rate.limit=100000 @@ -31,7 +32,7 @@ bootstrap.servers=realtime-kafka-2:29092 group.id=3078 auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer +sink.kafka.value.format=json # Topic & Database Config topic.prefix=mysql-cdc consumer.capture_database=cdc_verify @@ -64,7 +65,6 @@ sink.flink.server.port=9091 sink.registry.url=http://localhost:8080/apis/registry/v2 # Transaction Config transaction.topic.suffix=transaction -transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer sink.trans.batch.size=100 # Sink Metrics diff --git a/conf/pixels-sink.pg.properties b/conf/pixels-sink.pg.properties index c2795ce..58f599f 100644 --- a/conf/pixels-sink.pg.properties +++ b/conf/pixels-sink.pg.properties @@ -1,5 +1,6 @@ # engine | kafka | storage sink.datasource=engine +sink.datasource.engine.format=connect sink.storage.mode=stream # -1 means no limit, Only implement in retina sink mode yet sink.datasource.rate.limit=100000 @@ -32,8 +33,7 @@ bootstrap.servers=realtime-kafka-2:29092 group.id=3078 auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -#value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventAvroDeserializer -value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer +sink.kafka.value.format=json # Topic & Database Config topic.prefix=postgresql.oltp_server consumer.capture_database=pixels_bench_sf1x @@ -67,8 +67,6 @@ sink.flink.server.port=9091 sink.registry.url=http://localhost:8080/apis/registry/v2 # Transaction Config transaction.topic.suffix=transaction -#transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataAvroDeserializer -transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer sink.trans.batch.size=100 # Sink Metrics diff --git a/docs/configuration.md b/docs/configuration.md index 73d486a..a37562b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -12,12 +12,14 @@ Values are loaded by `PixelsSinkConfig` and mapped from keys in the properties f | --- | --- | --- | | `sink.datasource` | `engine` | Source type: `engine`, `kafka`, or `storage`. | | `sink.mode` | `retina` | Sink type: `retina`, `csv`, `proto`, `flink`, or `none`. | +| `sink.datasource.decode.threads` | `4` | Decode thread pool size. Engine uses `StreamOrderedDecoder` (multi logical stream); Storage uses `OrderedBatchDecoder` (single-key batch). | +| `sink.datasource.engine.format` | `connect` | Engine wire format. Only `connect` is allowed; other values fail fast on resolve. | | `sink.datasource.rate.limit` | `-1` | Rate limit for source ingestion. `-1` disables. | | `sink.datasource.rate.limit.type` | `semaphore` | Rate limiter type used by `FlushRateLimiterFactory`. 'guava' or 'semaphore'| ### Notes on `sink.datasource` -- `engine` reads CDC logs directly from Debezium Engine. +- `engine` reads CDC logs directly from Debezium Engine. Currently requires `sink.datasource.engine.format=connect`. Json/Avro Engine paths are reserved and will reuse the same `conversion.debezium` converter interfaces later. - `storage` reads CDC logs from files dumped by `sink.proto` output; schema reference: [sink.proto](https://github.com/pixelsdb/pixels/blob/master/proto/sink.proto). - `kafka` reads from a set of Kafka topics; this mode is deprecated and not actively tested. - Engine and Kafka records are normalized to the canonical `SinkProto` contract before reaching writers. @@ -52,13 +54,14 @@ Notes on `sink.trans.mode`: | Key | Default | Notes | | --- | --- | --- | +| `sink.datasource.engine.format` | `connect` | Only `connect` is runnable today. Non-`connect` values fail fast. | | `debezium.name` | none | Engine name. | | `debezium.connector.class` | none | Connector class, e.g. PostgreSQL or MySQL connector. | | `debezium.*` | none | Standard Debezium engine properties. | See `conf/pixels-sink.mysql.properties` for a TDSQL MySQL CDC example. -The Debezium Connector reads the database Binlog or WAL. The local `conversion.debezium` package only converts envelopes already emitted by Debezium. +The Debezium Connector reads the database Binlog or WAL. The local `conversion.debezium` package only converts envelopes already emitted by Debezium (`connect` / `json` / `avro` plus shared `support` / `dialect`). ### Retina Sink @@ -115,15 +118,26 @@ Kafka source is deprecated. | `group.id` | required | Consumer group id. | | `auto.offset.reset` | none | Standard Kafka consumer property. | | `key.deserializer` | `org.apache.kafka.common.serialization.StringDeserializer` | Kafka key deserializer. | -| `value.deserializer` | `io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer` | Kafka value deserializer for row events. | +| `sink.kafka.value.format` | `json` | Envelope format: `json` or `avro`. Kafka sources assemble `conversion.debezium` converters from this key. | | `topic.prefix` | required | Topic prefix for table events. | | `consumer.capture_database` | required | Database name used to build topic names. | | `consumer.include_tables` | empty | Comma-separated table list, empty means all. | | `transaction.topic.suffix` | `transaction` | Suffix appended to transaction topics. | -| `transaction.topic.value.deserializer` | `io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer` | Deserializer for transaction messages. | | `transaction.topic.group_id` | `transaction_consumer` | Consumer group for transaction topic. | | `sink.registry.url` | required | Avro Schema registry endpoint. | +**Legacy Kafka deserializer migration** + +Prefer `sink.kafka.value.format`. Old keys `value.deserializer` and +`transaction.topic.value.deserializer` are no longer SPI; `PixelsSinkConfig` +migrates them at startup: + +| Condition | Behavior | +| --- | --- | +| `sink.kafka.value.format` is set; legacy keys still present | Warn and ignore legacy keys | +| Format unset; legacy class name maps to Json/Avro | Infer format and warn | +| Row/tx inferences conflict, or class name unrecognized | Fail fast | + **Reserved Configuration** | Key | Default | Notes | diff --git a/docs/overview.md b/docs/overview.md index e139405..952cfa8 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -15,13 +15,39 @@ Pixels Sink uses a multi-stage pipeline. Each stage communicates via producer/co - Receives a `ConfigFactory` directly and builds the same sink pipeline. **Source** -The source stage owns input lifecycle and invokes the configured converter. +The source stage owns transport lifecycle (Engine / Kafka / Storage). Payload +conversion is separate from transport: + +- Parallel decode infrastructure lives in `util.concurrent` + (`DecodeExecutors`, `StreamOrderedDecoder`, `OrderedBatchDecoder`) and is + sized by `sink.datasource.decode.threads`. +- Debezium envelopes (Engine Connect / Kafka JSON / Kafka Avro) convert through + `conversion.debezium` (`connect` / `json` / `avro` + `support` / `dialect`). +- Storage sink-proto bytes convert through `conversion.sinkproto` and stay + independent of the Debezium package. +- Engine event classification stays in `source.engine` (`ConnectEventClassifier`, + `DebeziumRecordType`). + +**Decode ordering (not enhanced beyond historical contract)** + +| Guaranteed | Not guaranteed | +| --- | --- | +| Same table (Engine) / same Storage key: FIFO relative to source scan or enqueue order | Cross-table global order | +| TX events FIFO on the TX stream | TX BEGIN/END vs row events global arrival order | +| | Same-table cross-PK-bucket write total order; `totalOrder` / `dataCollectionOrder` reorder | + +Two decoders have different roles (not a shared abstraction): + +| Component | Role | Stream / shard key | +| --- | --- | --- | +| `StreamOrderedDecoder` | Engine: ordered publish across multiple logical streams in a mixed batch | ROW = `SchemaTableName`; TX = singleton stream key | +| `OrderedBatchDecoder` | Storage: parallel decode inside one key consumer, then deliver in input order | Sharding already done by `queueMap`; decoder does not re-shard | **Source Inputs** | Source Type | Description | Related Config | | --- | --- | --- | -| `engine` | Debezium Engine reads WAL/binlog directly from a database | `debezium.*` | -| `kafka` | Kafka consumer reads change events from topics | `bootstrap.servers`, `group.id`, `topic.*` | +| `engine` | Debezium Engine reads WAL/binlog directly from a database | `debezium.*`, `sink.datasource.engine.format` | +| `kafka` | Kafka consumer reads change events from topics | `bootstrap.servers`, `group.id`, `topic.*`, `sink.kafka.value.format` | | `storage` | Reads from Pixels storage files containing serialized sink proto records | `sink.proto.*`, `sink.storage.loop` | **Source Outputs** diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/EngineValueFormat.java b/src/main/java/io/pixelsdb/pixels/sink/config/EngineValueFormat.java new file mode 100644 index 0000000..280e636 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/config/EngineValueFormat.java @@ -0,0 +1,57 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.config; + +import java.util.Locale; + +/** + * Resolves {@code sink.datasource.engine.format}. Only {@code connect} is implemented. + */ +public final class EngineValueFormat +{ + public static final String CONNECT = "connect"; + + private EngineValueFormat() + { + } + + public static String resolve(String configuredFormat) + { + if (configuredFormat == null || configuredFormat.isBlank()) + { + return CONNECT; + } + String normalized = normalize(configuredFormat); + if (!CONNECT.equals(normalized)) + { + throw new IllegalArgumentException( + "sink.datasource.engine.format='" + configuredFormat + + "' is not implemented yet; only 'connect' is supported. " + + "Json/Avro Engine paths will reuse conversion.debezium converters later."); + } + return normalized; + } + + private static String normalize(String format) + { + return format.trim().toLowerCase(Locale.ROOT); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/KafkaValueFormat.java b/src/main/java/io/pixelsdb/pixels/sink/config/KafkaValueFormat.java new file mode 100644 index 0000000..5e9c9bb --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/config/KafkaValueFormat.java @@ -0,0 +1,56 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Affero GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.config; + +import java.util.Locale; + +/** + * Resolves {@code sink.kafka.value.format}. Supported values: {@code json}, {@code avro}. + */ +public final class KafkaValueFormat +{ + public static final String JSON = "json"; + public static final String AVRO = "avro"; + + private KafkaValueFormat() + { + } + + public static String resolve(String configuredFormat) + { + if (configuredFormat == null || configuredFormat.isBlank()) + { + return JSON; + } + return normalize(configuredFormat); + } + + private static String normalize(String format) + { + String normalized = format.trim().toLowerCase(Locale.ROOT); + if (!JSON.equals(normalized) && !AVRO.equals(normalized)) + { + throw new IllegalArgumentException( + "sink.kafka.value.format must be json or avro, got: " + format); + } + return normalized; + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java index 554e517..50a684e 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java @@ -21,20 +21,28 @@ package io.pixelsdb.pixels.sink.config; import io.pixelsdb.pixels.common.utils.ConfigFactory; -import io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer; -import io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer; import io.pixelsdb.pixels.sink.writer.PixelsSinkMode; import io.pixelsdb.pixels.sink.writer.retina.RetinaServiceProxy; import io.pixelsdb.pixels.sink.writer.retina.TransactionMode; import lombok.Getter; import org.apache.kafka.common.serialization.StringDeserializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.List; +import java.util.Locale; +import java.util.Properties; @Getter public class PixelsSinkConfig { + private static final Logger LOGGER = LoggerFactory.getLogger(PixelsSinkConfig.class); + private static final String LEGACY_VALUE_DESERIALIZER = "value.deserializer"; + private static final String LEGACY_TX_VALUE_DESERIALIZER = + "transaction.topic.value.deserializer"; + private static final String KAFKA_VALUE_FORMAT_KEY = "sink.kafka.value.format"; + private final ConfigFactory config; @ConfigKey(value = "transaction.timeout", defaultValue = TransactionConfig.DEFAULT_TRANSACTION_TIME_OUT) @@ -152,8 +160,11 @@ public class PixelsSinkConfig @ConfigKey(value = "key.deserializer", defaultClass = StringDeserializer.class) private String keyDeserializer; - @ConfigKey(value = "value.deserializer", defaultClass = RowChangeEventJsonDeserializer.class) - private String valueDeserializer; + /** + * Kafka envelope wire format: {@code json} or {@code avro}. + */ + @ConfigKey(value = "sink.kafka.value.format", defaultValue = KafkaValueFormat.JSON) + private String kafkaValueFormat; @ConfigKey(value = "sink.csv.path", defaultValue = PixelsSinkDefaultConfig.CSV_SINK_PATH) private String csvSinkPath; @@ -161,10 +172,6 @@ public class PixelsSinkConfig @ConfigKey(value = "transaction.topic.suffix", defaultValue = TransactionConfig.DEFAULT_TRANSACTION_TOPIC_SUFFIX) private String transactionTopicSuffix; - @ConfigKey(value = "transaction.topic.value.deserializer", - defaultClass = TransactionMetadataJsonDeserializer.class) - private String transactionTopicValueDeserializer; - @ConfigKey(value = "transaction.topic.group_id", defaultValue = TransactionConfig.DEFAULT_TRANSACTION_TOPIC_GROUP_ID) private String transactionTopicGroupId; @@ -178,6 +185,16 @@ public class PixelsSinkConfig @ConfigKey(value = "sink.datasource", defaultValue = PixelsSinkDefaultConfig.DATA_SOURCE) private String dataSource; + /** + * Engine wire format. Currently only {@code connect} is implemented. + */ + @ConfigKey(value = "sink.datasource.engine.format", defaultValue = "connect") + private String engineFormat; + + @ConfigKey(value = "sink.datasource.decode.threads", + defaultValue = PixelsSinkDefaultConfig.SOURCE_DECODE_THREADS) + private int sourceDecodeThreads; + @ConfigKey(value = "sink.datasource.rate.limit", defaultValue = "-1") private int sourceRateLimit; @@ -255,8 +272,104 @@ public String[] getIncludeTables() private void init() { - ConfigLoader.load(this.config.extractPropertiesByPrefix("", false), this); - + Properties props = this.config.extractPropertiesByPrefix("", false); + ConfigLoader.load(props, this); + + if (this.sourceDecodeThreads <= 0) + { + throw new IllegalArgumentException( + "sink.datasource.decode.threads must be positive"); + } this.enableSourceRateLimit = this.sourceRateLimit >= 0; + this.kafkaValueFormat = resolveKafkaValueFormat(props); + this.engineFormat = EngineValueFormat.resolve(this.engineFormat); + } + + /** + * Migrates legacy {@code value.deserializer} / + * {@code transaction.topic.value.deserializer} class names to + * {@code sink.kafka.value.format}. Package-visible for unit tests. + */ + static String resolveKafkaValueFormat(Properties props) + { + String explicitFormat = trimToNull(props.getProperty(KAFKA_VALUE_FORMAT_KEY)); + String rowDeserializer = trimToNull(props.getProperty(LEGACY_VALUE_DESERIALIZER)); + String txDeserializer = trimToNull(props.getProperty(LEGACY_TX_VALUE_DESERIALIZER)); + boolean hasLegacy = rowDeserializer != null || txDeserializer != null; + + if (explicitFormat != null) + { + if (hasLegacy) + { + LOGGER.warn( + "Ignoring legacy {} / {} because {} is set to '{}'", + LEGACY_VALUE_DESERIALIZER, + LEGACY_TX_VALUE_DESERIALIZER, + KAFKA_VALUE_FORMAT_KEY, + explicitFormat); + } + return KafkaValueFormat.resolve(explicitFormat); + } + + if (!hasLegacy) + { + return KafkaValueFormat.resolve(null); + } + + String inferredRow = inferFormatFromDeserializer(rowDeserializer); + String inferredTx = inferFormatFromDeserializer(txDeserializer); + if (inferredRow == null && inferredTx == null) + { + throw new IllegalArgumentException( + "Unable to migrate legacy Kafka deserializer classes to " + + KAFKA_VALUE_FORMAT_KEY + ": value.deserializer=" + + rowDeserializer + ", transaction.topic.value.deserializer=" + + txDeserializer); + } + if (inferredRow != null && inferredTx != null && !inferredRow.equals(inferredTx)) + { + throw new IllegalArgumentException( + "Conflicting legacy Kafka deserializer formats: value.deserializer=" + + rowDeserializer + " (" + inferredRow + + "), transaction.topic.value.deserializer=" + + txDeserializer + " (" + inferredTx + ")"); + } + String inferred = inferredRow != null ? inferredRow : inferredTx; + LOGGER.warn( + "Migrating legacy Kafka deserializer class names to {}={}", + KAFKA_VALUE_FORMAT_KEY, inferred); + return inferred; + } + + private static String inferFormatFromDeserializer(String className) + { + if (className == null) + { + return null; + } + String simpleName = className; + int lastDot = className.lastIndexOf('.'); + if (lastDot >= 0 && lastDot + 1 < className.length()) + { + simpleName = className.substring(lastDot + 1); + } + String lower = simpleName.toLowerCase(Locale.ROOT); + boolean json = lower.contains("json"); + boolean avro = lower.contains("avro"); + if (json == avro) + { + return null; + } + return json ? KafkaValueFormat.JSON : KafkaValueFormat.AVRO; + } + + private static String trimToNull(String value) + { + if (value == null) + { + return null; + } + String trimmed = value.trim(); + return trimmed.isEmpty() ? null : trimmed; } } \ No newline at end of file diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java index d2269bb..08e5f37 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java @@ -24,8 +24,7 @@ public final class PixelsSinkConstants { public static final String ROW_RECORD_KAFKA_PROP_FACTORY = "row-record"; public static final String TRANSACTION_KAFKA_PROP_FACTORY = "transaction"; - public static final String ROW_RECORD_CONVERTER_CLASS = "pixels.sink.row.converter.class"; - public static final String TRANSACTION_CONVERTER_CLASS = "pixels.sink.transaction.converter.class"; + public static final String KAFKA_VALUE_FORMAT = "pixels.sink.kafka.value.format"; public static final String DEBEZIUM_CONNECTOR_CLASS = "debezium.connector.class"; public static final int MAX_QUEUE_SIZE = 1_000; public static final String SNAPSHOT_TX_PREFIX = "SNAPSHOT-"; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkDefaultConfig.java b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkDefaultConfig.java index 8dfc00d..da08f3f 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkDefaultConfig.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkDefaultConfig.java @@ -23,6 +23,7 @@ public class PixelsSinkDefaultConfig { public static final String DATA_SOURCE = "engine"; + public static final String SOURCE_DECODE_THREADS = "4"; public static final String PROPERTIES_PATH = "pixels-sink.properties"; public static final String CSV_SINK_PATH = "./data"; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/TransactionConfig.java b/src/main/java/io/pixelsdb/pixels/sink/config/TransactionConfig.java index 88c0c5f..1200eb0 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/TransactionConfig.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/TransactionConfig.java @@ -20,12 +20,9 @@ package io.pixelsdb.pixels.sink.config; -import io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer; - public class TransactionConfig { public static final String DEFAULT_TRANSACTION_TOPIC_SUFFIX = "transaction"; - public static final String DEFAULT_TRANSACTION_TOPIC_VALUE_DESERIALIZER = TransactionMetadataJsonDeserializer.class.getName(); public static final String DEFAULT_TRANSACTION_TOPIC_GROUP_ID = "transaction_consumer"; public static final String DEFAULT_TRANSACTION_TIME_OUT = "300"; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/factory/RowRecordKafkaPropFactory.java b/src/main/java/io/pixelsdb/pixels/sink/config/factory/RowRecordKafkaPropFactory.java index 2a82521..d80f8a3 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/factory/RowRecordKafkaPropFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/factory/RowRecordKafkaPropFactory.java @@ -36,6 +36,8 @@ static Properties getCommonKafkaProperties(PixelsSinkConfig config) kafkaProperties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, config.getBootstrapServers()); kafkaProperties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, config.getKeyDeserializer()); kafkaProperties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + kafkaProperties.put( + PixelsSinkConstants.KAFKA_VALUE_FORMAT, config.getKafkaValueFormat()); return kafkaProperties; } @@ -43,7 +45,6 @@ static Properties getCommonKafkaProperties(PixelsSinkConfig config) public Properties createKafkaProperties(PixelsSinkConfig config) { Properties kafkaProperties = getCommonKafkaProperties(config); - kafkaProperties.put(PixelsSinkConstants.ROW_RECORD_CONVERTER_CLASS, config.getValueDeserializer()); if (config.getDebeziumConnectorClass() != null && !config.getDebeziumConnectorClass().isBlank()) { diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/factory/TransactionKafkaPropFactory.java b/src/main/java/io/pixelsdb/pixels/sink/config/factory/TransactionKafkaPropFactory.java index 49c0592..d0049d3 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/factory/TransactionKafkaPropFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/factory/TransactionKafkaPropFactory.java @@ -36,8 +36,6 @@ public class TransactionKafkaPropFactory implements KafkaPropFactory public Properties createKafkaProperties(PixelsSinkConfig config) { Properties kafkaProperties = getCommonKafkaProperties(config); - kafkaProperties.put(PixelsSinkConstants.TRANSACTION_CONVERTER_CLASS, - config.getTransactionTopicValueDeserializer()); if (config.getDebeziumConnectorClass() != null && !config.getDebeziumConnectorClass().isBlank()) { diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventAvroDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowConverter.java similarity index 52% rename from src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventAvroDeserializer.java rename to src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowConverter.java index 53ee7d1..90db45a 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventAvroDeserializer.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowConverter.java @@ -1,31 +1,32 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * * This file is part of Pixels. * * Pixels is free software: you can redistribute it and/or modify * it under the terms of the Affero GNU General Public License as * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * the License, or (at your option) any later version. * * Pixels is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. + * GNU Affero General Public License for more details. * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see + * You should have received a copy of the GNU General Public + * License along with Pixels. If not, see * . */ package io.pixelsdb.pixels.sink.conversion.debezium; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; + /** - * @deprecated Use the Kafka-boundary facade in - * {@code source.kafka.serde.RowChangeEventAvroDeserializer}. + * Converts a Debezium row envelope (any wire format) into a canonical row event. */ -@Deprecated -public class RowChangeEventAvroDeserializer - extends io.pixelsdb.pixels.sink.source.kafka.serde.RowChangeEventAvroDeserializer +@FunctionalInterface +public interface DebeziumRowConverter { + RowChangeEvent convert(I input) throws Exception; } diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventJsonDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumTransactionConverter.java similarity index 52% rename from src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventJsonDeserializer.java rename to src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumTransactionConverter.java index a605720..c412b79 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventJsonDeserializer.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumTransactionConverter.java @@ -1,31 +1,32 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * * This file is part of Pixels. * * Pixels is free software: you can redistribute it and/or modify * it under the terms of the Affero GNU General Public License as * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * the License, or (at your option) any later version. * * Pixels is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. + * GNU Affero General Public License for more details. * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see + * You should have received a copy of the GNU General Public + * License along with Pixels. If not, see * . */ package io.pixelsdb.pixels.sink.conversion.debezium; +import io.pixelsdb.pixels.sink.SinkProto; + /** - * @deprecated Use the Kafka-boundary facade in - * {@code source.kafka.serde.RowChangeEventJsonDeserializer}. + * Converts a Debezium transaction envelope (any wire format) into metadata. */ -@Deprecated -public class RowChangeEventJsonDeserializer - extends io.pixelsdb.pixels.sink.source.kafka.serde.RowChangeEventJsonDeserializer +@FunctionalInterface +public interface DebeziumTransactionConverter { + SinkProto.TransactionMetadata convert(I input) throws Exception; } diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/TransactionMetadataAvroDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/TransactionMetadataAvroDeserializer.java deleted file mode 100644 index ac3ca38..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/TransactionMetadataAvroDeserializer.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.conversion.debezium; - -/** - * @deprecated Use the Kafka-boundary facade in - * {@code source.kafka.serde.TransactionMetadataAvroDeserializer}. - */ -@Deprecated -public class TransactionMetadataAvroDeserializer - extends io.pixelsdb.pixels.sink.source.kafka.serde.TransactionMetadataAvroDeserializer -{ -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/TransactionMetadataJsonDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/TransactionMetadataJsonDeserializer.java deleted file mode 100644 index 2287be1..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/TransactionMetadataJsonDeserializer.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.conversion.debezium; - -/** - * @deprecated Use the Kafka-boundary facade in - * {@code source.kafka.serde.TransactionMetadataJsonDeserializer}. - */ -@Deprecated -public class TransactionMetadataJsonDeserializer - extends io.pixelsdb.pixels.sink.source.kafka.serde.TransactionMetadataJsonDeserializer -{ -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroPayloadDecoder.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroPayloadDecoder.java new file mode 100644 index 0000000..fa76395 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroPayloadDecoder.java @@ -0,0 +1,55 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.conversion.debezium.avro; + +import io.apicurio.registry.serde.SerdeConfig; +import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; +import org.apache.avro.generic.GenericRecord; + +import java.util.HashMap; +import java.util.Map; + +/** + * Decodes Kafka Avro Confluent/Apicurio payloads to {@link GenericRecord}. + */ +public final class DebeziumAvroPayloadDecoder implements AutoCloseable +{ + private final AvroKafkaDeserializer avroDeserializer = + new AvroKafkaDeserializer<>(); + + public void configure(Map configs, boolean isKey) + { + Map enrichedConfig = new HashMap<>(configs); + enrichedConfig.put(SerdeConfig.CHECK_PERIOD_MS, SerdeConfig.CHECK_PERIOD_MS_DEFAULT); + avroDeserializer.configure(enrichedConfig, isKey); + } + + public GenericRecord decode(String topic, byte[] data) + { + return avroDeserializer.deserialize(topic, data); + } + + @Override + public void close() + { + avroDeserializer.close(); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumAvroRowConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroRowConverter.java similarity index 86% rename from src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumAvroRowConverter.java rename to src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroRowConverter.java index d577956..d519fc8 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumAvroRowConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroRowConverter.java @@ -18,29 +18,29 @@ * . */ -package io.pixelsdb.pixels.sink.conversion.debezium; +package io.pixelsdb.pixels.sink.conversion.debezium.avro; import io.pixelsdb.pixels.core.TypeDescription; import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumRowConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumEnvelopeNormalizer; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRecordUtil; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRowRecordAssembler; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRowValueConverter; import io.pixelsdb.pixels.sink.event.RowChangeEvent; import io.pixelsdb.pixels.sink.exception.SinkException; import io.pixelsdb.pixels.sink.metadata.TableMetadata; import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; import org.apache.avro.generic.GenericRecord; -public final class DebeziumAvroRowConverter +public final class DebeziumAvroRowConverter implements DebeziumRowConverter { private final TableMetadataRegistry tableMetadataRegistry; private final DebeziumSourceAdapter configuredAdapter; private final DebeziumRowRecordAssembler rowRecordAssembler = new DebeziumRowRecordAssembler(); - public DebeziumAvroRowConverter(TableMetadataRegistry tableMetadataRegistry) - { - this(tableMetadataRegistry, null); - } - public DebeziumAvroRowConverter( TableMetadataRegistry tableMetadataRegistry, DebeziumSourceAdapter configuredAdapter) @@ -49,6 +49,7 @@ public DebeziumAvroRowConverter( this.configuredAdapter = configuredAdapter; } + @Override public RowChangeEvent convert(GenericRecord record) throws SinkException { SinkProto.OperationType operation = DebeziumRecordUtil.getOperationType( diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumAvroTransactionConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroTransactionConverter.java similarity index 78% rename from src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumAvroTransactionConverter.java rename to src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroTransactionConverter.java index 1a47c12..1d59ac9 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumAvroTransactionConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroTransactionConverter.java @@ -18,26 +18,25 @@ * . */ -package io.pixelsdb.pixels.sink.conversion.debezium; +package io.pixelsdb.pixels.sink.conversion.debezium.avro; import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumTransactionConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumEnvelopeNormalizer; import org.apache.avro.generic.GenericRecord; public final class DebeziumAvroTransactionConverter + implements DebeziumTransactionConverter { private final DebeziumSourceAdapter configuredAdapter; - public DebeziumAvroTransactionConverter() - { - this(null); - } - public DebeziumAvroTransactionConverter(DebeziumSourceAdapter configuredAdapter) { this.configuredAdapter = configuredAdapter; } + @Override public SinkProto.TransactionMetadata convert(GenericRecord record) { return configuredAdapter == null diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventStructConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectRowConverter.java similarity index 76% rename from src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventStructConverter.java rename to src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectRowConverter.java index 0cd5939..52cf60c 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventStructConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectRowConverter.java @@ -1,5 +1,5 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * * This file is part of Pixels. * @@ -11,19 +11,23 @@ * Pixels is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. + * GNU Affero General Public License for more details. * - * You should have received a copy of the Affero GNU General Public + * You should have received a copy of the GNU General Public * License along with Pixels. If not, see * . */ -package io.pixelsdb.pixels.sink.conversion.debezium; - +package io.pixelsdb.pixels.sink.conversion.debezium.connect; import io.pixelsdb.pixels.core.TypeDescription; import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumRowConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumEnvelopeNormalizer; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRecordUtil; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRowRecordAssembler; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRowValueConverter; import io.pixelsdb.pixels.sink.event.RowChangeEvent; import io.pixelsdb.pixels.sink.exception.SinkException; import io.pixelsdb.pixels.sink.metadata.TableMetadata; @@ -35,28 +39,17 @@ import org.slf4j.LoggerFactory; /** - * @package: io.pixelsdb.pixels.sink.conversion.debezium - * @className: RowChangeEventStructConverter - * @author: AntiO2 - * @date: 2025/9/26 12:00 + * Converts Kafka Connect {@link SourceRecord} Debezium envelopes to row events. */ -public class RowChangeEventStructConverter +public final class DebeziumConnectRowConverter implements DebeziumRowConverter { - private static final Logger LOGGER = LoggerFactory.getLogger(RowChangeEventStructConverter.class); + private static final Logger LOGGER = LoggerFactory.getLogger(DebeziumConnectRowConverter.class); + private final TableMetadataRegistry tableMetadataRegistry; private final DebeziumSourceAdapter configuredAdapter; + private final DebeziumRowRecordAssembler rowRecordAssembler = new DebeziumRowRecordAssembler(); - public RowChangeEventStructConverter() - { - this(TableMetadataRegistry.Instance(), null); - } - - public RowChangeEventStructConverter(TableMetadataRegistry tableMetadataRegistry) - { - this(tableMetadataRegistry, null); - } - - public RowChangeEventStructConverter( + public DebeziumConnectRowConverter( TableMetadataRegistry tableMetadataRegistry, DebeziumSourceAdapter configuredAdapter) { @@ -64,11 +57,7 @@ public RowChangeEventStructConverter( this.configuredAdapter = configuredAdapter; } - public static RowChangeEvent convertToRowChangeEvent(SourceRecord sourceRecord) throws SinkException - { - return new RowChangeEventStructConverter().convert(sourceRecord); - } - + @Override public RowChangeEvent convert(SourceRecord sourceRecord) throws SinkException { if (!(sourceRecord.value() instanceof Struct value)) @@ -80,12 +69,9 @@ public RowChangeEvent convert(SourceRecord sourceRecord) throws SinkException return buildRowRecord(value, operationType); } - private RowChangeEvent buildRowRecord(Struct value, - SinkProto.OperationType opType) throws SinkException + private RowChangeEvent buildRowRecord( + Struct value, SinkProto.OperationType opType) throws SinkException { - - String schemaName; - String tableName; DebeziumSourceAdapter adapter; SinkProto.SourceInfo sourceInfo; try @@ -99,15 +85,14 @@ private RowChangeEvent buildRowRecord(Struct value, ? DebeziumEnvelopeNormalizer.adapterForSource(source) : configuredAdapter; sourceInfo = DebeziumEnvelopeNormalizer.normalizeSource(source, adapter); - schemaName = sourceInfo.getDb(); - tableName = sourceInfo.getTable(); } catch (DataException | IllegalArgumentException e) { LOGGER.warn("Missing source field in row record"); throw new SinkException(e); } - TableMetadata metadata = tableMetadataRegistry.getMetadata(schemaName, tableName); + TableMetadata metadata = tableMetadataRegistry.getMetadata( + sourceInfo.getDb(), sourceInfo.getTable()); TypeDescription typeDescription = metadata.getTypeDescription(); DebeziumRowValueConverter rowDataParser = new DebeziumRowValueConverter(typeDescription); @@ -117,7 +102,8 @@ private RowChangeEvent buildRowRecord(Struct value, Struct transaction = value.getStruct("transaction"); if (transaction != null) { - transactionInfo = DebeziumEnvelopeNormalizer.normalizeTransaction(transaction, adapter); + transactionInfo = DebeziumEnvelopeNormalizer.normalizeTransaction( + transaction, adapter); } } catch (DataException e) { @@ -150,7 +136,7 @@ private RowChangeEvent buildRowRecord(Struct value, afterValue = afterBuilder.build(); } - return new DebeziumRowRecordAssembler().assemble( + return rowRecordAssembler.assemble( opType, sourceInfo, transactionInfo, beforeValue, afterValue, typeDescription, metadata); } diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectTransactionConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectTransactionConverter.java new file mode 100644 index 0000000..e8f5b56 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectTransactionConverter.java @@ -0,0 +1,54 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.conversion.debezium.connect; + +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumTransactionConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumEnvelopeNormalizer; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.source.SourceRecord; + +/** + * Converts Kafka Connect {@link SourceRecord} transaction envelopes. + */ +public final class DebeziumConnectTransactionConverter + implements DebeziumTransactionConverter +{ + private final DebeziumSourceAdapter configuredAdapter; + + public DebeziumConnectTransactionConverter(DebeziumSourceAdapter configuredAdapter) + { + this.configuredAdapter = configuredAdapter; + } + + @Override + public SinkProto.TransactionMetadata convert(SourceRecord sourceRecord) + { + if (!(sourceRecord.value() instanceof Struct value)) + { + throw new IllegalArgumentException("Debezium transaction value must be a Struct"); + } + return configuredAdapter == null + ? DebeziumEnvelopeNormalizer.normalizeTransactionMetadata(value) + : DebeziumEnvelopeNormalizer.normalizeTransactionMetadata(value, configuredAdapter); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceAdapter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceAdapter.java similarity index 94% rename from src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceAdapter.java rename to src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceAdapter.java index 26c13e4..37bb481 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceAdapter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceAdapter.java @@ -8,7 +8,7 @@ * published by the Free Software Foundation, either version 3 of * the License, or (at your option) any later version. */ -package io.pixelsdb.pixels.sink.conversion.debezium.source; +package io.pixelsdb.pixels.sink.conversion.debezium.dialect; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceAdapterRegistry.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceAdapterRegistry.java similarity index 96% rename from src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceAdapterRegistry.java rename to src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceAdapterRegistry.java index d6938ad..6510feb 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceAdapterRegistry.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceAdapterRegistry.java @@ -8,7 +8,7 @@ * published by the Free Software Foundation, either version 3 of * the License, or (at your option) any later version. */ -package io.pixelsdb.pixels.sink.conversion.debezium.source; +package io.pixelsdb.pixels.sink.conversion.debezium.dialect; import java.util.List; import java.util.ServiceLoader; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceMetadata.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceMetadata.java similarity index 88% rename from src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceMetadata.java rename to src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceMetadata.java index 2ff3c92..b2efddd 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/DebeziumSourceMetadata.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceMetadata.java @@ -8,7 +8,7 @@ * published by the Free Software Foundation, either version 3 of * the License, or (at your option) any later version. */ -package io.pixelsdb.pixels.sink.conversion.debezium.source; +package io.pixelsdb.pixels.sink.conversion.debezium.dialect; public record DebeziumSourceMetadata( String connector, diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/MySqlSourceAdapter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/MySqlSourceAdapter.java similarity index 94% rename from src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/MySqlSourceAdapter.java rename to src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/MySqlSourceAdapter.java index 4aaa359..4212d99 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/MySqlSourceAdapter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/MySqlSourceAdapter.java @@ -8,7 +8,7 @@ * published by the Free Software Foundation, either version 3 of * the License, or (at your option) any later version. */ -package io.pixelsdb.pixels.sink.conversion.debezium.source; +package io.pixelsdb.pixels.sink.conversion.debezium.dialect; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/PostgresSourceAdapter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/PostgresSourceAdapter.java similarity index 95% rename from src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/PostgresSourceAdapter.java rename to src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/PostgresSourceAdapter.java index 12e3ed3..5852d1b 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/source/PostgresSourceAdapter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/PostgresSourceAdapter.java @@ -8,7 +8,7 @@ * published by the Free Software Foundation, either version 3 of * the License, or (at your option) any later version. */ -package io.pixelsdb.pixels.sink.conversion.debezium.source; +package io.pixelsdb.pixels.sink.conversion.debezium.dialect; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumJsonRowConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonRowConverter.java similarity index 87% rename from src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumJsonRowConverter.java rename to src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonRowConverter.java index 5f04ca9..83d63a5 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumJsonRowConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonRowConverter.java @@ -18,19 +18,24 @@ * . */ -package io.pixelsdb.pixels.sink.conversion.debezium; +package io.pixelsdb.pixels.sink.conversion.debezium.json; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import io.pixelsdb.pixels.core.TypeDescription; import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumRowConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumEnvelopeNormalizer; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRecordUtil; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRowRecordAssembler; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRowValueConverter; import io.pixelsdb.pixels.sink.event.RowChangeEvent; import io.pixelsdb.pixels.sink.exception.SinkException; import io.pixelsdb.pixels.sink.metadata.TableMetadata; import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; -public final class DebeziumJsonRowConverter +public final class DebeziumJsonRowConverter implements DebeziumRowConverter { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -39,11 +44,6 @@ public final class DebeziumJsonRowConverter private final DebeziumRowRecordAssembler rowRecordAssembler = new DebeziumRowRecordAssembler(); - public DebeziumJsonRowConverter(TableMetadataRegistry tableMetadataRegistry) - { - this(tableMetadataRegistry, null); - } - public DebeziumJsonRowConverter( TableMetadataRegistry tableMetadataRegistry, DebeziumSourceAdapter configuredAdapter) @@ -52,6 +52,7 @@ public DebeziumJsonRowConverter( this.configuredAdapter = configuredAdapter; } + @Override public RowChangeEvent convert(byte[] data) throws Exception { JsonNode rootNode = OBJECT_MAPPER.readTree(data); diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonTransactionConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonTransactionConverter.java new file mode 100644 index 0000000..4ddfec5 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonTransactionConverter.java @@ -0,0 +1,57 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.conversion.debezium.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumTransactionConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumEnvelopeNormalizer; + +/** + * Converts Debezium JSON transaction envelopes ({@code byte[]} payload). + */ +public final class DebeziumJsonTransactionConverter + implements DebeziumTransactionConverter +{ + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private final DebeziumSourceAdapter configuredAdapter; + + public DebeziumJsonTransactionConverter(DebeziumSourceAdapter configuredAdapter) + { + this.configuredAdapter = configuredAdapter; + } + + @Override + public SinkProto.TransactionMetadata convert(byte[] data) throws Exception + { + if (configuredAdapter == null) + { + throw new IllegalStateException( + "No Debezium source adapter configured for JSON transaction conversion"); + } + JsonNode payload = OBJECT_MAPPER.readTree(data).path("payload"); + return DebeziumEnvelopeNormalizer.normalizeTransactionMetadata( + payload, configuredAdapter); + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizer.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumEnvelopeNormalizer.java similarity index 93% rename from src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizer.java rename to src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumEnvelopeNormalizer.java index f1e5dcd..41d3a8d 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizer.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumEnvelopeNormalizer.java @@ -8,12 +8,12 @@ * published by the Free Software Foundation, either version 3 of * the License, or (at your option) any later version. */ -package io.pixelsdb.pixels.sink.conversion.debezium; +package io.pixelsdb.pixels.sink.conversion.debezium.support; import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; -import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapterRegistry; -import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceMetadata; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapterRegistry; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceMetadata; public final class DebeziumEnvelopeNormalizer { diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRecordUtil.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRecordUtil.java similarity index 98% rename from src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRecordUtil.java rename to src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRecordUtil.java index 31ba018..d38a65f 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRecordUtil.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRecordUtil.java @@ -8,7 +8,7 @@ * published by the Free Software Foundation, either version 3 of * the License, or (at your option) any later version. */ -package io.pixelsdb.pixels.sink.conversion.debezium; +package io.pixelsdb.pixels.sink.conversion.debezium.support; import com.fasterxml.jackson.databind.JsonNode; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowRecordAssembler.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowRecordAssembler.java similarity index 97% rename from src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowRecordAssembler.java rename to src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowRecordAssembler.java index 9b72956..d2c6d58 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowRecordAssembler.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowRecordAssembler.java @@ -18,7 +18,7 @@ * . */ -package io.pixelsdb.pixels.sink.conversion.debezium; +package io.pixelsdb.pixels.sink.conversion.debezium.support; import io.pixelsdb.pixels.core.TypeDescription; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowValueConverter.java similarity index 99% rename from src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverter.java rename to src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowValueConverter.java index 26a4c78..e510c70 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowValueConverter.java @@ -18,7 +18,7 @@ * . */ -package io.pixelsdb.pixels.sink.conversion.debezium; +package io.pixelsdb.pixels.sink.conversion.debezium.support; import com.fasterxml.jackson.databind.JsonNode; import com.google.protobuf.ByteString; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/kafka/KafkaRecordConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/kafka/KafkaRecordConverter.java deleted file mode 100644 index 3474be8..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/kafka/KafkaRecordConverter.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.conversion.kafka; - -import java.util.Properties; - -/** - * @deprecated Kafka source wiring belongs to {@code source.kafka.serde}. - * This facade keeps the old SPI name usable by existing deployments. - */ -@Deprecated -public final class KafkaRecordConverter implements AutoCloseable -{ - private final io.pixelsdb.pixels.sink.source.kafka.serde.KafkaRecordConverter delegate; - - private KafkaRecordConverter( - io.pixelsdb.pixels.sink.source.kafka.serde.KafkaRecordConverter delegate) - { - this.delegate = delegate; - } - - public static KafkaRecordConverter create( - Properties properties, String converterClassKey) - { - return new KafkaRecordConverter<>( - io.pixelsdb.pixels.sink.source.kafka.serde.KafkaRecordConverter.create( - properties, converterClassKey)); - } - - public T convert(String topic, byte[] data) - { - return delegate.convert(topic, data); - } - - @Override - public void close() - { - delegate.close(); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifier.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifier.java new file mode 100644 index 0000000..4f55356 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifier.java @@ -0,0 +1,110 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.source.engine; + +import io.pixelsdb.pixels.common.metadata.SchemaTableName; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRecordUtil; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.source.SourceRecord; + +import java.util.Locale; + +/** + * Classifies Kafka Connect {@link SourceRecord} Debezium envelopes. + */ +public final class ConnectEventClassifier implements DebeziumEventClassifier +{ + @Override + public DebeziumRecordType classify( + SourceRecord sourceRecord, String transactionTopic) + { + if (sourceRecord == null || sourceRecord.value() == null) + { + return DebeziumRecordType.TOMBSTONE; + } + if (!(sourceRecord.value() instanceof Struct value)) + { + return DebeziumRecordType.UNKNOWN_CONTROL; + } + + if (transactionTopic != null && transactionTopic.equals(sourceRecord.topic())) + { + String status = DebeziumRecordUtil.getStringSafely(value, "status"); + String id = DebeziumRecordUtil.getStringSafely(value, "id"); + return (status.equals("BEGIN") || status.equals("END")) && !id.isBlank() + ? DebeziumRecordType.TRANSACTION + : DebeziumRecordType.UNKNOWN_CONTROL; + } + + return isRowChange(value) + ? DebeziumRecordType.ROW + : DebeziumRecordType.UNKNOWN_CONTROL; + } + + @Override + public SchemaTableName tableOf(SourceRecord record) + { + if (!(record.value() instanceof Struct value)) + { + throw new IllegalArgumentException("Debezium row value must be a Struct"); + } + Object sourceObject = DebeziumRecordUtil.getFieldSafely(value, "source"); + if (!(sourceObject instanceof Struct source)) + { + throw new IllegalArgumentException("Debezium row source must be a Struct"); + } + String database = DebeziumRecordUtil.getStringSafely(source, "db"); + String table = DebeziumRecordUtil.getStringSafely(source, "table"); + if (database.isBlank() || table.isBlank()) + { + throw new IllegalArgumentException( + "Debezium row source is missing db or table"); + } + return new SchemaTableName(database, table); + } + + private static boolean isRowChange(Struct value) + { + String op = DebeziumRecordUtil.getStringSafely(value, "op").toLowerCase(Locale.ROOT); + if (!(op.equals("c") || op.equals("u") || op.equals("d") || op.equals("r"))) + { + return false; + } + + Object sourceObject = DebeziumRecordUtil.getFieldSafely(value, "source"); + if (!(sourceObject instanceof Struct source) || + DebeziumRecordUtil.getStringSafely(source, "db").isBlank() || + DebeziumRecordUtil.getStringSafely(source, "table").isBlank()) + { + return false; + } + + Object before = DebeziumRecordUtil.getFieldSafely(value, "before"); + Object after = DebeziumRecordUtil.getFieldSafely(value, "after"); + return switch (op) + { + case "c", "r" -> after instanceof Struct; + case "u" -> before instanceof Struct && after instanceof Struct; + case "d" -> before instanceof Struct; + default -> false; + }; + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumEventClassifier.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumEventClassifier.java new file mode 100644 index 0000000..73f1071 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumEventClassifier.java @@ -0,0 +1,34 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.source.engine; + +import io.pixelsdb.pixels.common.metadata.SchemaTableName; + +/** + * Engine-side classification of Debezium transport records. + * Lives in {@code source.engine}, not conversion. + */ +public interface DebeziumEventClassifier +{ + DebeziumRecordType classify(I input, String transactionTopic); + + SchemaTableName tableOf(I input); +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumRecordType.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumRecordType.java new file mode 100644 index 0000000..98d8cb5 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumRecordType.java @@ -0,0 +1,32 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.source.engine; + +/** + * Classification of Debezium Engine {@code SourceRecord} envelopes. + */ +public enum DebeziumRecordType +{ + ROW, + TRANSACTION, + TOMBSTONE, + UNKNOWN_CONTROL +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumer.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumer.java index ce8fff2..0e578f8 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumer.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumer.java @@ -26,22 +26,28 @@ import io.pixelsdb.pixels.sink.SinkProto; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumRecordUtil; -import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumRowConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumTransactionConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.connect.DebeziumConnectRowConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.connect.DebeziumConnectTransactionConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.support.DebeziumRecordUtil; import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; import io.pixelsdb.pixels.sink.pipeline.TablePipelineManager; import io.pixelsdb.pixels.sink.pipeline.TransactionPipeline; -import io.pixelsdb.pixels.sink.source.engine.adapter.DebeziumStructAdapter; import io.pixelsdb.pixels.sink.source.engine.adapter.DebeziumSourceAdapterSelector; import io.pixelsdb.pixels.sink.util.MetricsFacade; +import io.pixelsdb.pixels.sink.util.concurrent.StreamOrderedDecoder; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.source.SourceRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.List; -import java.util.Locale; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; /** * @package: io.pixelsdb.pixels.source @@ -53,147 +59,155 @@ public class PixelsDebeziumConsumer implements DebeziumEngine.ChangeConsumer>, AutoCloseable { private static final Logger LOGGER = LoggerFactory.getLogger(PixelsDebeziumConsumer.class); - - public enum RecordType - { - ROW, - TRANSACTION, - TOMBSTONE, - UNKNOWN_CONTROL - } + private static final Object TRANSACTION_STREAM_KEY = new Object(); + private static final ConnectEventClassifier CLASSIFIER = new ConnectEventClassifier(); private final String checkTransactionTopic; private final DebeziumSourceAdapter connectorAdapter; - private final DebeziumStructAdapter structAdapter; - private final TransactionPipeline transactionPipeline = new TransactionPipeline(); - private final TablePipelineManager tablePipelineManager = new TablePipelineManager(); + private final DebeziumRowConverter rowConverter; + private final DebeziumTransactionConverter transactionConverter; + private final TransactionPipeline transactionPipeline; + private final TablePipelineManager tablePipelineManager; + private final StreamOrderedDecoder decodePipeline; private final MetricsFacade metricsFacade = MetricsFacade.getInstance(); private final PixelsSinkConfig pixelsSinkConfig = PixelsSinkConfigFactory.getInstance(); + private record PendingRecord( + RecordChangeEvent record, + CompletableFuture completion) + { + } + public PixelsDebeziumConsumer() { this.checkTransactionTopic = pixelsSinkConfig.getDebeziumTopicPrefix() + ".transaction"; this.connectorAdapter = DebeziumSourceAdapterSelector.configured(); - this.structAdapter = new DebeziumStructAdapter(connectorAdapter); + this.rowConverter = new DebeziumConnectRowConverter( + TableMetadataRegistry.Instance(), connectorAdapter); + this.transactionConverter = new DebeziumConnectTransactionConverter(connectorAdapter); + this.transactionPipeline = new TransactionPipeline(); + this.tablePipelineManager = new TablePipelineManager(); + this.decodePipeline = new StreamOrderedDecoder( + pixelsSinkConfig.getSourceDecodeThreads(), + "debezium-decoder"); } public void start() { transactionPipeline.start(); + decodePipeline.start(); } public void handleBatch(List> event, DebeziumEngine.RecordCommitter> committer) throws InterruptedException { + List pendingRecords = new ArrayList<>(event.size()); for (RecordChangeEvent record : event) { - try + SourceRecord sourceRecord = record.record(); + if (sourceRecord == null) { - SourceRecord sourceRecord = record.record(); - if (sourceRecord == null) - { - continue; - } + pendingRecords.add(new PendingRecord( + record, CompletableFuture.completedFuture(null))); + continue; + } - metricsFacade.recordDebeziumEvent(); - RecordType recordType = classify(sourceRecord, checkTransactionTopic); - logSourceRecord(sourceRecord, recordType); - try - { - switch (recordType) - { - case ROW -> handleRowChangeSourceRecord(sourceRecord); - case TRANSACTION -> handleTransactionSourceRecord(sourceRecord); - case TOMBSTONE, UNKNOWN_CONTROL -> - LOGGER.debug("Skipping Debezium {} event from topic {}", - recordType, sourceRecord.topic()); - } - } catch (RuntimeException e) + metricsFacade.recordDebeziumEvent(); + DebeziumRecordType recordType = + CLASSIFIER.classify(sourceRecord, checkTransactionTopic); + logSourceRecord(sourceRecord, recordType); + CompletableFuture completion = switch (recordType) + { + case ROW -> submitRow(sourceRecord); + case TRANSACTION -> submitTransaction(sourceRecord); + case TOMBSTONE, UNKNOWN_CONTROL -> { - LOGGER.warn("Skipping invalid Debezium {} event from topic {}: {}", - recordType, sourceRecord.topic(), e.getMessage()); + LOGGER.debug("Skipping Debezium {} event from topic {}", + recordType, sourceRecord.topic()); + yield CompletableFuture.completedFuture(null); } - } finally - { - committer.markProcessed(record); - } + }; + pendingRecords.add(new PendingRecord(record, completion)); + } + for (PendingRecord pending : pendingRecords) + { + awaitCompletion(pending.completion()); + committer.markProcessed(pending.record()); } committer.markBatchFinished(); } - private void handleTransactionSourceRecord(SourceRecord sourceRecord) throws InterruptedException + private CompletableFuture submitRow(SourceRecord record) { - SinkProto.TransactionMetadata transaction = structAdapter.toTransactionMetadata(sourceRecord); - metricsFacade.recordSerdTxChange(); - transactionPipeline.publish(transaction); + return decodePipeline.submit( + CLASSIFIER.tableOf(record), + record, + rowConverter::convert, + result -> publishRow(record, result)); } - private void handleRowChangeSourceRecord(SourceRecord sourceRecord) + private CompletableFuture submitTransaction(SourceRecord record) { - try - { - RowChangeEvent event = structAdapter.toRowEvent(sourceRecord); - metricsFacade.recordSerdRowChange(); - tablePipelineManager.route(event); - } catch (SinkException e) - { - throw new IllegalArgumentException("Failed to convert Debezium row event", e); - } + return decodePipeline.submit( + TRANSACTION_STREAM_KEY, + record, + transactionConverter::convert, + result -> publishTransaction(record, result)); } - public static RecordType classify(SourceRecord sourceRecord, String transactionTopic) + private void publishRow( + SourceRecord record, + StreamOrderedDecoder.DecodeResult result) { - if (sourceRecord == null || sourceRecord.value() == null) + if (result.failure() != null) { - return RecordType.TOMBSTONE; + LOGGER.warn("Skipping invalid Debezium ROW event from topic {}", + record.topic(), result.failure()); + return; } - if (!(sourceRecord.value() instanceof Struct value)) + if (result.value() == null) { - return RecordType.UNKNOWN_CONTROL; - } - - if (transactionTopic != null && transactionTopic.equals(sourceRecord.topic())) - { - String status = DebeziumRecordUtil.getStringSafely(value, "status"); - String id = DebeziumRecordUtil.getStringSafely(value, "id"); - return (status.equals("BEGIN") || status.equals("END")) && !id.isBlank() - ? RecordType.TRANSACTION - : RecordType.UNKNOWN_CONTROL; + return; } - - return isRowChange(value) ? RecordType.ROW : RecordType.UNKNOWN_CONTROL; + metricsFacade.recordSerdRowChange(); + tablePipelineManager.route(result.value()); } - private static boolean isRowChange(Struct value) + private void publishTransaction( + SourceRecord record, + StreamOrderedDecoder.DecodeResult result) { - String op = DebeziumRecordUtil.getStringSafely(value, "op").toLowerCase(Locale.ROOT); - if (!(op.equals("c") || op.equals("u") || op.equals("d") || op.equals("r"))) + if (result.failure() != null) { - return false; + LOGGER.warn("Skipping invalid Debezium TRANSACTION event from topic {}", + record.topic(), result.failure()); + return; } - - Object sourceObject = DebeziumRecordUtil.getFieldSafely(value, "source"); - if (!(sourceObject instanceof Struct source) || - DebeziumRecordUtil.getStringSafely(source, "db").isBlank() || - DebeziumRecordUtil.getStringSafely(source, "table").isBlank()) + if (result.value() == null) { - return false; + return; } + metricsFacade.recordSerdTxChange(); + transactionPipeline.publish(result.value()); + } - Object before = DebeziumRecordUtil.getFieldSafely(value, "before"); - Object after = DebeziumRecordUtil.getFieldSafely(value, "after"); - return switch (op) + private void awaitCompletion(CompletableFuture completion) + throws InterruptedException + { + try { - case "c", "r" -> after instanceof Struct; - case "u" -> before instanceof Struct && after instanceof Struct; - case "d" -> before instanceof Struct; - default -> false; - }; + completion.get(); + } catch (ExecutionException e) + { + throw new IllegalStateException( + "Debezium decode pipeline failed before publishing an event", + e.getCause()); + } } - private void logSourceRecord(SourceRecord record, RecordType recordType) + private void logSourceRecord(SourceRecord record, DebeziumRecordType recordType) { if (!LOGGER.isDebugEnabled()) { @@ -204,7 +218,7 @@ private void logSourceRecord(SourceRecord record, RecordType recordType) asStruct(DebeziumRecordUtil.getFieldSafely(value, "source")); Struct transaction = value == null ? null : asStruct(DebeziumRecordUtil.getFieldSafely(value, "transaction")); - String rawTransactionId = recordType == RecordType.TRANSACTION + String rawTransactionId = recordType == DebeziumRecordType.TRANSACTION ? DebeziumRecordUtil.getStringSafely(value, "id") : DebeziumRecordUtil.getStringSafely(transaction, "id"); String canonicalTransactionId = @@ -235,12 +249,14 @@ private static String schemaName(org.apache.kafka.connect.data.Schema schema) @Override public void close() { + decodePipeline.close(); tablePipelineManager.close(); transactionPipeline.close(); } public void abort() { + decodePipeline.abort(); tablePipelineManager.abort(); transactionPipeline.abort(); } diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/SinkEngineSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/SinkEngineSource.java index f2fd348..fa2a6e2 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/engine/SinkEngineSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/SinkEngineSource.java @@ -24,6 +24,7 @@ import io.debezium.engine.DebeziumEngine; import io.debezium.engine.RecordChangeEvent; import io.debezium.engine.format.ChangeEventFormat; +import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; import io.pixelsdb.pixels.sink.source.SinkSource; import org.apache.kafka.connect.source.SourceRecord; @@ -37,6 +38,7 @@ public class SinkEngineSource implements SinkSource { private static final long SHUTDOWN_TIMEOUT_SECONDS = 10; + private final PixelsSinkConfig pixelsSinkConfig; private final PixelsDebeziumConsumer consumer; private DebeziumEngine> engine; private ExecutorService executor; @@ -44,14 +46,15 @@ public class SinkEngineSource implements SinkSource public SinkEngineSource() { + this.pixelsSinkConfig = PixelsSinkConfigFactory.getInstance(); this.consumer = new PixelsDebeziumConsumer(); } public void start() { consumer.start(); - Properties debeziumProps = PixelsSinkConfigFactory.getInstance() - .getConfig().extractPropertiesByPrefix("debezium.", true); + Properties debeziumProps = pixelsSinkConfig.getConfig() + .extractPropertiesByPrefix("debezium.", true); this.engine = DebeziumEngine.create(ChangeEventFormat.of(Connect.class)) .using(debeziumProps) diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumSourceAdapterSelector.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumSourceAdapterSelector.java index 097a8c5..bf5e256 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumSourceAdapterSelector.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumSourceAdapterSelector.java @@ -21,9 +21,12 @@ package io.pixelsdb.pixels.sink.source.engine.adapter; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; -import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapterRegistry; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapterRegistry; +/** + * Wiring-side selector that reads {@code debezium.connector.class}. + */ public final class DebeziumSourceAdapterSelector { private DebeziumSourceAdapterSelector() diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumStructAdapter.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumStructAdapter.java deleted file mode 100644 index 46eb9b0..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumStructAdapter.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.source.engine.adapter; - -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumEnvelopeNormalizer; -import io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventStructConverter; -import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.exception.SinkException; -import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; -import org.apache.kafka.connect.data.Struct; -import org.apache.kafka.connect.source.SourceRecord; - -public final class DebeziumStructAdapter -{ - private final DebeziumSourceAdapter sourceAdapter; - private final RowChangeEventStructConverter rowConverter; - - public DebeziumStructAdapter(DebeziumSourceAdapter sourceAdapter) - { - this.sourceAdapter = sourceAdapter; - this.rowConverter = new RowChangeEventStructConverter( - TableMetadataRegistry.Instance(), sourceAdapter); - } - - public RowChangeEvent toRowEvent(SourceRecord sourceRecord) throws SinkException - { - return rowConverter.convert(sourceRecord); - } - - public SinkProto.TransactionMetadata toTransactionMetadata(SourceRecord sourceRecord) - { - if (!(sourceRecord.value() instanceof Struct value)) - { - throw new IllegalArgumentException("Debezium transaction value must be a Struct"); - } - return DebeziumEnvelopeNormalizer.normalizeTransactionMetadata(value, sourceAdapter); - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaRowSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaRowSource.java index e315021..43134e2 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaRowSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaRowSource.java @@ -21,7 +21,6 @@ package io.pixelsdb.pixels.sink.source.kafka; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; -import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; import io.pixelsdb.pixels.sink.event.RowChangeEvent; import io.pixelsdb.pixels.sink.pipeline.TablePipelineManager; @@ -69,8 +68,7 @@ public KafkaRowSource( this.topic = topic; this.tableName = DataTransform.extractTableName(topic); this.tablePipelineManager = tablePipelineManager; - this.converter = KafkaRecordConverter.create( - this.kafkaProperties, PixelsSinkConstants.ROW_RECORD_CONVERTER_CLASS); + this.converter = KafkaRecordConverter.forRow(this.kafkaProperties); } @Override diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaTransactionSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaTransactionSource.java index aac33a0..5d912a5 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaTransactionSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaTransactionSource.java @@ -21,7 +21,6 @@ package io.pixelsdb.pixels.sink.source.kafka; import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; import io.pixelsdb.pixels.sink.pipeline.TransactionPipeline; import io.pixelsdb.pixels.sink.source.kafka.serde.KafkaRecordConverter; import io.pixelsdb.pixels.sink.util.MetricsFacade; @@ -56,8 +55,7 @@ public KafkaTransactionSource( this.transactionTopic = transactionTopic; Properties consumerProperties = new Properties(); consumerProperties.putAll(kafkaProperties); - this.converter = KafkaRecordConverter.create( - consumerProperties, PixelsSinkConstants.TRANSACTION_CONVERTER_CLASS); + this.converter = KafkaRecordConverter.forTransaction(consumerProperties); this.consumer = new KafkaConsumer<>(consumerProperties); this.transactionPipeline = transactionPipeline; } diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverter.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverter.java index 4fe7e42..1ef2f92 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverter.java @@ -6,81 +6,192 @@ * Pixels is free software: you can redistribute it and/or modify * it under the terms of the Affero GNU General Public License as * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * the License, or (at your option) any later version. * * Pixels is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * Affero GNU General Public License for more details. * - * You should have received a copy of the GNU General Public License + * You should have received a copy of the GNU Affero General Public License * along with Pixels. If not, see * . */ package io.pixelsdb.pixels.sink.source.kafka.serde; -import org.apache.kafka.common.serialization.Deserializer; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.config.KafkaValueFormat; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; +import io.pixelsdb.pixels.sink.conversion.debezium.avro.DebeziumAvroPayloadDecoder; +import io.pixelsdb.pixels.sink.conversion.debezium.avro.DebeziumAvroRowConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.avro.DebeziumAvroTransactionConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapterRegistry; +import io.pixelsdb.pixels.sink.conversion.debezium.json.DebeziumJsonRowConverter; +import io.pixelsdb.pixels.sink.conversion.debezium.json.DebeziumJsonTransactionConverter; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.lang.reflect.InvocationTargetException; import java.util.HashMap; import java.util.Map; import java.util.Properties; +/** + * Assembles Debezium row/tx converters for Kafka by {@code sink.kafka.value.format}. + */ public final class KafkaRecordConverter implements AutoCloseable { - private final Deserializer deserializer; + private static final Logger LOGGER = LoggerFactory.getLogger(KafkaRecordConverter.class); - private KafkaRecordConverter(Deserializer deserializer) + private enum ErrorPolicy { - this.deserializer = deserializer; + SWALLOW, + PROPAGATE } - public static KafkaRecordConverter create( - Properties properties, String converterClassKey) + @FunctionalInterface + private interface ConvertFn { - Object configuredClass = properties.get(converterClassKey); - if (configuredClass == null) - { - throw new IllegalArgumentException( - "Missing Kafka record converter: " + converterClassKey); - } + T convert(String topic, byte[] data) throws Exception; + } - try + private final ConvertFn convertFn; + private final AutoCloseable resource; + private final ErrorPolicy errorPolicy; + + private KafkaRecordConverter( + ConvertFn convertFn, AutoCloseable resource, ErrorPolicy errorPolicy) + { + this.convertFn = convertFn; + this.resource = resource; + this.errorPolicy = errorPolicy; + } + + public static KafkaRecordConverter forRow(Properties properties) + { + String format = resolveFormat(properties); + DebeziumSourceAdapter adapter = resolveAdapter(properties); + return switch (format) { - Class converterClass = configuredClass instanceof Class type - ? type - : Class.forName(configuredClass.toString()); - if (!Deserializer.class.isAssignableFrom(converterClass)) + case KafkaValueFormat.JSON -> + { + DebeziumJsonRowConverter converter = new DebeziumJsonRowConverter( + TableMetadataRegistry.Instance(), adapter); + yield new KafkaRecordConverter<>( + (topic, data) -> converter.convert(data), null, ErrorPolicy.SWALLOW); + } + case KafkaValueFormat.AVRO -> { - throw new IllegalArgumentException( - "Kafka record converter must implement Deserializer: " - + converterClass.getName()); + DebeziumAvroPayloadDecoder decoder = new DebeziumAvroPayloadDecoder(); + decoder.configure(toConfigMap(properties), false); + DebeziumAvroRowConverter converter = new DebeziumAvroRowConverter( + TableMetadataRegistry.Instance(), adapter); + yield new KafkaRecordConverter<>( + (topic, data) -> converter.convert(decoder.decode(topic, data)), + decoder, + ErrorPolicy.SWALLOW); } + default -> throw new IllegalArgumentException( + "Unsupported Kafka value format: " + format); + }; + } - @SuppressWarnings("unchecked") - Deserializer deserializer = - (Deserializer) converterClass.getDeclaredConstructor().newInstance(); - Map configuration = new HashMap<>(); - properties.forEach((key, value) -> configuration.put(key.toString(), value)); - deserializer.configure(configuration, false); - return new KafkaRecordConverter<>(deserializer); - } catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | - IllegalAccessException | InvocationTargetException e) + public static KafkaRecordConverter forTransaction( + Properties properties) + { + String format = resolveFormat(properties); + DebeziumSourceAdapter adapter = resolveAdapter(properties); + return switch (format) { - throw new IllegalArgumentException( - "Failed to create Kafka record converter: " + configuredClass, e); - } + case KafkaValueFormat.JSON -> + { + DebeziumJsonTransactionConverter converter = + new DebeziumJsonTransactionConverter(adapter); + yield new KafkaRecordConverter<>( + (topic, data) -> converter.convert(data), null, ErrorPolicy.PROPAGATE); + } + case KafkaValueFormat.AVRO -> + { + DebeziumAvroPayloadDecoder decoder = new DebeziumAvroPayloadDecoder(); + decoder.configure(toConfigMap(properties), false); + DebeziumAvroTransactionConverter converter = + new DebeziumAvroTransactionConverter(adapter); + yield new KafkaRecordConverter<>( + (topic, data) -> converter.convert(decoder.decode(topic, data)), + decoder, + ErrorPolicy.PROPAGATE); + } + default -> throw new IllegalArgumentException( + "Unsupported Kafka value format: " + format); + }; } public T convert(String topic, byte[] data) { - return deserializer.deserialize(topic, data); + if (data == null || data.length == 0) + { + return null; + } + try + { + return convertFn.convert(topic, data); + } catch (RuntimeException e) + { + if (errorPolicy == ErrorPolicy.SWALLOW) + { + LOGGER.warn("Failed to convert Kafka record from topic {}", topic, e); + return null; + } + throw e; + } catch (Exception e) + { + if (errorPolicy == ErrorPolicy.SWALLOW) + { + LOGGER.warn("Failed to convert Kafka record from topic {}", topic, e); + return null; + } + throw new RuntimeException("Failed to convert Kafka record from " + topic, e); + } } @Override public void close() { - deserializer.close(); + if (resource != null) + { + try + { + resource.close(); + } catch (Exception e) + { + LOGGER.warn("Failed to close Kafka record converter resource", e); + } + } + } + + private static String resolveFormat(Properties properties) + { + Object configured = properties.get(PixelsSinkConstants.KAFKA_VALUE_FORMAT); + return KafkaValueFormat.resolve(configured == null ? null : configured.toString()); + } + + private static DebeziumSourceAdapter resolveAdapter(Properties properties) + { + Object connector = properties.get(PixelsSinkConstants.DEBEZIUM_CONNECTOR_CLASS); + if (connector == null || connector.toString().isBlank()) + { + return null; + } + return DebeziumSourceAdapterRegistry.resolve(connector.toString()); + } + + private static Map toConfigMap(Properties properties) + { + Map configuration = new HashMap<>(); + properties.forEach((key, value) -> configuration.put(key.toString(), value)); + return configuration; } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowChangeEventAvroDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowChangeEventAvroDeserializer.java deleted file mode 100644 index 82490a7..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowChangeEventAvroDeserializer.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.source.kafka.serde; - -import io.apicurio.registry.serde.SerdeConfig; -import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; -import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; -import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumAvroRowConverter; -import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; -import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapterRegistry; -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; -import io.pixelsdb.pixels.sink.source.engine.adapter.DebeziumSourceAdapterSelector; -import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.common.serialization.Deserializer; - -import java.util.HashMap; -import java.util.Map; - -public class RowChangeEventAvroDeserializer implements Deserializer -{ - private final AvroKafkaDeserializer avroDeserializer = - new AvroKafkaDeserializer<>(); - private DebeziumAvroRowConverter converter = - new DebeziumAvroRowConverter( - TableMetadataRegistry.Instance(), - DebeziumSourceAdapterSelector.configuredIfPresent()); - - @Override - public void configure(Map configs, boolean isKey) - { - Map enrichedConfig = new HashMap<>(configs); - enrichedConfig.put(SerdeConfig.CHECK_PERIOD_MS, SerdeConfig.CHECK_PERIOD_MS_DEFAULT); - avroDeserializer.configure(enrichedConfig, isKey); - Object connector = configs.get(PixelsSinkConstants.DEBEZIUM_CONNECTOR_CLASS); - if (connector != null && !connector.toString().isBlank()) - { - DebeziumSourceAdapter adapter = - DebeziumSourceAdapterRegistry.resolve(connector.toString()); - converter = new DebeziumAvroRowConverter( - TableMetadataRegistry.Instance(), adapter); - } - } - - @Override - public RowChangeEvent deserialize(String topic, byte[] data) - { - if (data == null || data.length == 0) - { - return null; - } - try - { - GenericRecord avroRecord = avroDeserializer.deserialize(topic, data); - return converter.convert(avroRecord); - } catch (Exception e) - { - return null; - } - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowChangeEventJsonDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowChangeEventJsonDeserializer.java deleted file mode 100644 index 65428de..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowChangeEventJsonDeserializer.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.source.kafka.serde; - -import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; -import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumJsonRowConverter; -import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; -import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapterRegistry; -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; -import io.pixelsdb.pixels.sink.source.engine.adapter.DebeziumSourceAdapterSelector; -import org.apache.kafka.common.serialization.Deserializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; - -public class RowChangeEventJsonDeserializer implements Deserializer -{ - private static final Logger LOGGER = - LoggerFactory.getLogger(RowChangeEventJsonDeserializer.class); - private DebeziumJsonRowConverter converter; - - public RowChangeEventJsonDeserializer() - { - this.converter = new DebeziumJsonRowConverter( - TableMetadataRegistry.Instance(), - DebeziumSourceAdapterSelector.configuredIfPresent()); - } - - @Override - public void configure(Map configs, boolean isKey) - { - Object connector = configs.get(PixelsSinkConstants.DEBEZIUM_CONNECTOR_CLASS); - if (connector != null && !connector.toString().isBlank()) - { - DebeziumSourceAdapter adapter = - DebeziumSourceAdapterRegistry.resolve(connector.toString()); - converter = new DebeziumJsonRowConverter( - TableMetadataRegistry.Instance(), adapter); - } - } - - @Override - public RowChangeEvent deserialize(String topic, byte[] data) - { - if (data == null || data.length == 0) - { - return null; - } - try - { - return converter.convert(data); - } catch (Exception e) - { - LOGGER.warn("Failed to convert Debezium JSON row from topic {}", topic, e); - return null; - } - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataAvroDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataAvroDeserializer.java deleted file mode 100644 index 0f116b1..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataAvroDeserializer.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.source.kafka.serde; - -import io.apicurio.registry.serde.SerdeConfig; -import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; -import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumAvroTransactionConverter; -import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; -import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapterRegistry; -import io.pixelsdb.pixels.sink.source.engine.adapter.DebeziumSourceAdapterSelector; -import org.apache.avro.generic.GenericRecord; -import org.apache.kafka.common.errors.SerializationException; -import org.apache.kafka.common.serialization.Deserializer; - -import java.util.HashMap; -import java.util.Map; - -public class TransactionMetadataAvroDeserializer - implements Deserializer -{ - private final AvroKafkaDeserializer avroDeserializer = - new AvroKafkaDeserializer<>(); - private DebeziumAvroTransactionConverter converter = - new DebeziumAvroTransactionConverter( - DebeziumSourceAdapterSelector.configuredIfPresent()); - - @Override - public void configure(Map configs, boolean isKey) - { - Map enrichedConfig = new HashMap<>(configs); - enrichedConfig.put(SerdeConfig.CHECK_PERIOD_MS, SerdeConfig.CHECK_PERIOD_MS_DEFAULT); - avroDeserializer.configure(enrichedConfig, isKey); - Object connector = configs.get(PixelsSinkConstants.DEBEZIUM_CONNECTOR_CLASS); - if (connector != null && !connector.toString().isBlank()) - { - DebeziumSourceAdapter adapter = - DebeziumSourceAdapterRegistry.resolve(connector.toString()); - converter = new DebeziumAvroTransactionConverter(adapter); - } - } - - @Override - public SinkProto.TransactionMetadata deserialize(String topic, byte[] data) - { - if (data == null || data.length == 0) - { - return null; - } - try - { - return converter.convert(avroDeserializer.deserialize(topic, data)); - } catch (Exception e) - { - throw new SerializationException( - "Failed to deserialize Avro transaction message", e); - } - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataJsonDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataJsonDeserializer.java deleted file mode 100644 index 6ed769c..0000000 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataJsonDeserializer.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.source.kafka.serde; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; -import io.pixelsdb.pixels.sink.conversion.debezium.DebeziumEnvelopeNormalizer; -import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter; -import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapterRegistry; -import io.pixelsdb.pixels.sink.source.engine.adapter.DebeziumSourceAdapterSelector; -import org.apache.kafka.common.serialization.Deserializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; - -public class TransactionMetadataJsonDeserializer - implements Deserializer -{ - private static final Logger LOGGER = - LoggerFactory.getLogger(TransactionMetadataJsonDeserializer.class); - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private DebeziumSourceAdapter adapter = - DebeziumSourceAdapterSelector.configuredIfPresent(); - - @Override - public void configure(Map configs, boolean isKey) - { - Object connector = configs.get(PixelsSinkConstants.DEBEZIUM_CONNECTOR_CLASS); - if (connector != null && !connector.toString().isBlank()) - { - adapter = DebeziumSourceAdapterRegistry.resolve(connector.toString()); - } - } - - @Override - public SinkProto.TransactionMetadata deserialize(String topic, byte[] data) - { - if (data == null || data.length == 0) - { - return null; - } - try - { - JsonNode payload = OBJECT_MAPPER.readTree(data).path("payload"); - if (adapter == null) - { - throw new IllegalStateException( - "No Debezium source adapter configured for " + topic); - } - return DebeziumEnvelopeNormalizer.normalizeTransactionMetadata(payload, adapter); - } catch (Exception e) - { - LOGGER.error("Failed to deserialize transaction message from {}", topic, e); - throw new RuntimeException("Deserialization error", e); - } - } -} diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java index 977ebe1..48669e4 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java @@ -22,8 +22,8 @@ import io.pixelsdb.pixels.common.physical.PhysicalReader; import io.pixelsdb.pixels.core.utils.Pair; -import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; import io.pixelsdb.pixels.sink.SinkProto; import io.pixelsdb.pixels.sink.conversion.sinkproto.RowRecordConverter; @@ -37,6 +37,8 @@ import io.pixelsdb.pixels.sink.util.EtcdFileRegistry; import io.pixelsdb.pixels.sink.util.DataTransform; import io.pixelsdb.pixels.sink.util.MetricsFacade; +import io.pixelsdb.pixels.sink.util.concurrent.DecodeExecutors; +import io.pixelsdb.pixels.sink.util.concurrent.OrderedBatchDecoder; import io.pixelsdb.pixels.sink.util.rateLimiter.FlushRateLimiter; import io.pixelsdb.pixels.sink.util.rateLimiter.FlushRateLimiterFactory; import org.slf4j.Logger; @@ -48,26 +50,20 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Function; public abstract class AbstractSinkStorageSource implements SinkSource { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractSinkStorageSource.class); private static final int DECODE_BATCH_SIZE = 64; - private static final int DECODE_THREAD_COUNT = 4; private static final long DECODE_BATCH_WAIT_MILLIS = 5; private static final long DECODE_SHUTDOWN_TIMEOUT_SECONDS = 30; protected static final int RECORD_HEADER_SIZE = Integer.BYTES * 2; @@ -93,17 +89,7 @@ public abstract class AbstractSinkStorageSource implements SinkSource new TransactionMetadataConverter(); protected final boolean freshnessTimestamp; private final MetricsFacade metricsFacade = MetricsFacade.getInstance(); - private final AtomicInteger decoderThreadId = new AtomicInteger(); - private final ExecutorService decodeExecutor = new ThreadPoolExecutor( - DECODE_THREAD_COUNT, - DECODE_THREAD_COUNT, - 0L, - TimeUnit.MILLISECONDS, - new ArrayBlockingQueue<>(PixelsSinkConstants.MAX_QUEUE_SIZE), - runnable -> new Thread( - runnable, - "storage-proto-decoder-" + decoderThreadId.incrementAndGet()), - new ThreadPoolExecutor.CallerRunsPolicy()); + private final ExecutorService decodeExecutor; protected int loopId = 0; protected List readers = new ArrayList<>(); @@ -118,6 +104,9 @@ protected AbstractSinkStorageSource() this.rowRecordConverter = new RowRecordConverter(TableMetadataRegistry.Instance()); this.freshnessTimestamp = pixelsSinkConfig.isSinkMonitorFreshnessTimestamp(); this.sourceRateLimiter = FlushRateLimiterFactory.getNewInstance(); + this.decodeExecutor = DecodeExecutors.newFixedCallerRuns( + pixelsSinkConfig.getSourceDecodeThreads(), + "storage-proto-decoder"); } protected void beginProcessing() @@ -272,20 +261,8 @@ protected void clean() private void shutdownDecodeExecutor() { - decodeExecutor.shutdown(); - try - { - if (!decodeExecutor.awaitTermination( - DECODE_SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) - { - LOGGER.warn("Timed out waiting for storage proto decoders to stop"); - decodeExecutor.shutdownNow(); - } - } catch (InterruptedException e) - { - decodeExecutor.shutdownNow(); - Thread.currentThread().interrupt(); - } + DecodeExecutors.shutdownGracefully( + decodeExecutor, DECODE_SHUTDOWN_TIMEOUT_SECONDS, LOGGER); } protected void consumeQueue(int key, BlockingQueue, Integer>> queue, ProtoType protoType) @@ -356,7 +333,7 @@ private void processBatch( { case ROW -> { - List events = decodeInOrder( + List events = OrderedBatchDecoder.decodeInOrder( decodeExecutor, batch, record -> decodeRowChangeSourceRecord(key, record)); @@ -371,7 +348,8 @@ record -> decodeRowChangeSourceRecord(key, record)); } case TRANS -> { - List transactions = decodeInOrder( + List transactions = + OrderedBatchDecoder.decodeInOrder( decodeExecutor, batch, this::decodeTransactionSourceRecord); @@ -387,32 +365,6 @@ record -> decodeRowChangeSourceRecord(key, record)); } } - static List decodeInOrder( - ExecutorService executor, - List records, - Function decoder) throws InterruptedException - { - List> futures = new ArrayList<>(records.size()); - for (T record : records) - { - futures.add(executor.submit(() -> decoder.apply(record))); - } - - List decodedRecords = new ArrayList<>(records.size()); - for (Future future : futures) - { - try - { - decodedRecords.add(future.get()); - } catch (ExecutionException e) - { - LOGGER.warn("Failed to decode storage record", e.getCause()); - decodedRecords.add(null); - } - } - return decodedRecords; - } - private RowChangeEvent decodeRowChangeSourceRecord( int key, Pair, Integer> record) diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/DecodeExecutors.java b/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/DecodeExecutors.java new file mode 100644 index 0000000..f8aaaf9 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/DecodeExecutors.java @@ -0,0 +1,99 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.util.concurrent; + +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; +import org.slf4j.Logger; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Shared fixed-size decode pools with caller-runs backpressure. + * Tasks submitted after shutdown are rejected. + */ +public final class DecodeExecutors +{ + private DecodeExecutors() + { + } + + public static ExecutorService newFixedCallerRuns(int threads, String threadNamePrefix) + { + if (threads <= 0) + { + throw new IllegalArgumentException("decode threads must be positive"); + } + if (threadNamePrefix == null || threadNamePrefix.isBlank()) + { + throw new IllegalArgumentException("threadNamePrefix must be non-blank"); + } + + AtomicInteger threadId = new AtomicInteger(); + return new ThreadPoolExecutor( + threads, + threads, + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(PixelsSinkConstants.MAX_QUEUE_SIZE), + runnable -> new Thread( + runnable, + threadNamePrefix + "-" + threadId.incrementAndGet()), + (task, executor) -> + { + if (executor.isShutdown()) + { + throw new RejectedExecutionException( + "Decode executor is shut down: " + threadNamePrefix); + } + task.run(); + }); + } + + public static void shutdownGracefully( + ExecutorService executor, long timeoutSeconds, Logger logger) + { + if (executor == null) + { + return; + } + executor.shutdown(); + try + { + if (!executor.awaitTermination(timeoutSeconds, TimeUnit.SECONDS)) + { + if (logger != null) + { + logger.warn("Timed out waiting for decode executor to stop"); + } + executor.shutdownNow(); + } + } catch (InterruptedException e) + { + executor.shutdownNow(); + Thread.currentThread().interrupt(); + } + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoder.java b/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoder.java new file mode 100644 index 0000000..efaf28e --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoder.java @@ -0,0 +1,70 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.util.concurrent; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.function.Function; + +/** + * Decodes a batch in parallel and returns results in input order. + * Failed records become {@code null} entries. + */ +public final class OrderedBatchDecoder +{ + private static final Logger LOGGER = LoggerFactory.getLogger(OrderedBatchDecoder.class); + + private OrderedBatchDecoder() + { + } + + public static List decodeInOrder( + ExecutorService executor, + List records, + Function decoder) throws InterruptedException + { + List> futures = new ArrayList<>(records.size()); + for (T record : records) + { + futures.add(executor.submit(() -> decoder.apply(record))); + } + + List decodedRecords = new ArrayList<>(records.size()); + for (Future future : futures) + { + try + { + decodedRecords.add(future.get()); + } catch (ExecutionException e) + { + LOGGER.warn("Failed to decode record", e.getCause()); + decodedRecords.add(null); + } + } + return decodedRecords; + } +} diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoder.java b/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoder.java new file mode 100644 index 0000000..601b930 --- /dev/null +++ b/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoder.java @@ -0,0 +1,250 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * Pixels is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public + * License along with Pixels. If not, see + * . + */ + +package io.pixelsdb.pixels.sink.util.concurrent; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.function.Consumer; + +/** + * Parallel decode with per-stream publish ordering. + * Same {@code streamKey} is published FIFO; different keys may publish in parallel. + */ +public final class StreamOrderedDecoder implements AutoCloseable +{ + private static final Logger LOGGER = LoggerFactory.getLogger(StreamOrderedDecoder.class); + private static final long SHUTDOWN_TIMEOUT_SECONDS = 30; + + @FunctionalInterface + public interface ThrowingFunction + { + O apply(I input) throws Exception; + } + + public record DecodeResult(O value, Throwable failure) + { + } + + private final Object lifecycleLock = new Object(); + private final Map> streamTails = new HashMap<>(); + private final Set> activeCompletions = ConcurrentHashMap.newKeySet(); + private final ExecutorService decodeExecutor; + private final boolean ownsExecutor; + private boolean started; + private boolean accepting; + private boolean closed; + private volatile boolean aborted; + + public StreamOrderedDecoder(int decodeThreads, String threadNamePrefix) + { + this(DecodeExecutors.newFixedCallerRuns(decodeThreads, threadNamePrefix), true); + } + + private StreamOrderedDecoder(ExecutorService decodeExecutor, boolean ownsExecutor) + { + if (decodeExecutor == null) + { + throw new IllegalArgumentException("decodeExecutor must not be null"); + } + this.decodeExecutor = decodeExecutor; + this.ownsExecutor = ownsExecutor; + } + + public void start() + { + synchronized (lifecycleLock) + { + if (started) + { + return; + } + if (closed) + { + throw new IllegalStateException("StreamOrderedDecoder is closed"); + } + started = true; + accepting = true; + } + } + + public CompletableFuture submit( + Object streamKey, + I input, + ThrowingFunction decoder, + Consumer> afterDecode) + { + CompletableFuture previous; + CompletableFuture completion = new CompletableFuture<>(); + synchronized (lifecycleLock) + { + ensureAccepting(); + previous = streamTails.getOrDefault( + streamKey, CompletableFuture.completedFuture(null)); + streamTails.put(streamKey, completion); + activeCompletions.add(completion); + } + + CompletableFuture> decoded = CompletableFuture.supplyAsync( + () -> decode(input, decoder), decodeExecutor); + + previous.thenCombine(decoded, (ignored, result) -> + { + publish(result, afterDecode); + return null; + }) + .whenComplete((ignored, failure) -> + { + if (failure != null) + { + completion.completeExceptionally(failure); + } + else + { + completion.complete(null); + } + removeCompletion(streamKey, completion); + }); + return completion; + } + + private void ensureAccepting() + { + if (!started || !accepting) + { + throw new IllegalStateException( + "StreamOrderedDecoder is not accepting records"); + } + } + + private DecodeResult decode(I input, ThrowingFunction decoder) + { + try + { + return new DecodeResult<>(decoder.apply(input), null); + } catch (Exception e) + { + return new DecodeResult<>(null, e); + } + } + + private void publish(DecodeResult result, Consumer> afterDecode) + { + if (aborted) + { + throw new CancellationException("StreamOrderedDecoder aborted"); + } + afterDecode.accept(result); + } + + private void removeCompletion(Object streamKey, CompletableFuture completion) + { + activeCompletions.remove(completion); + synchronized (lifecycleLock) + { + // Keep a failed tail so later submits on the same stream fail fast + // instead of starting a fresh successful chain. + if (streamTails.get(streamKey) == completion + && !completion.isCompletedExceptionally()) + { + streamTails.remove(streamKey); + } + } + } + + @Override + public void close() + { + List> pending; + synchronized (lifecycleLock) + { + if (closed) + { + return; + } + closed = true; + accepting = false; + pending = new ArrayList<>(activeCompletions); + } + + try + { + awaitCompletions(pending); + if (ownsExecutor) + { + DecodeExecutors.shutdownGracefully( + decodeExecutor, SHUTDOWN_TIMEOUT_SECONDS, LOGGER); + } + } catch (InterruptedException e) + { + abort(); + Thread.currentThread().interrupt(); + } + } + + public void abort() + { + List> pending; + synchronized (lifecycleLock) + { + if (aborted) + { + return; + } + aborted = true; + closed = true; + accepting = false; + pending = new ArrayList<>(activeCompletions); + } + CancellationException cancellation = + new CancellationException("StreamOrderedDecoder aborted"); + pending.forEach(completion -> completion.completeExceptionally(cancellation)); + if (ownsExecutor) + { + decodeExecutor.shutdownNow(); + } + } + + private void awaitCompletions(List> pending) + throws InterruptedException + { + for (CompletableFuture completion : pending) + { + try + { + completion.get(); + } catch (ExecutionException | CancellationException e) + { + LOGGER.warn("StreamOrderedDecoder completed with an error", e); + } + } + } +} diff --git a/src/main/resources/META-INF/services/io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter b/src/main/resources/META-INF/services/io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter new file mode 100644 index 0000000..aa5df8c --- /dev/null +++ b/src/main/resources/META-INF/services/io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapter @@ -0,0 +1,2 @@ +io.pixelsdb.pixels.sink.conversion.debezium.dialect.MySqlSourceAdapter +io.pixelsdb.pixels.sink.conversion.debezium.dialect.PostgresSourceAdapter diff --git a/src/main/resources/META-INF/services/io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter b/src/main/resources/META-INF/services/io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter deleted file mode 100644 index 2ea4ccd..0000000 --- a/src/main/resources/META-INF/services/io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapter +++ /dev/null @@ -1,2 +0,0 @@ -io.pixelsdb.pixels.sink.conversion.debezium.source.MySqlSourceAdapter -io.pixelsdb.pixels.sink.conversion.debezium.source.PostgresSourceAdapter diff --git a/src/main/resources/pixels-sink.aws.properties b/src/main/resources/pixels-sink.aws.properties index 081f5ad..34a3b06 100644 --- a/src/main/resources/pixels-sink.aws.properties +++ b/src/main/resources/pixels-sink.aws.properties @@ -8,8 +8,7 @@ bootstrap.servers=realtime-kafka-2:29092 group.id=3078 auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -#value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventAvroDeserializer -value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer +sink.kafka.value.format=json # Topic & Database Config topic.prefix=postgresql.oltp_server consumer.capture_database=pixels_bench_sf1x @@ -39,8 +38,6 @@ sink.proto.maxRecords=1000000 sink.registry.url=http://localhost:8080/apis/registry/v2 # Transaction Config transaction.topic.suffix=transaction -#transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataAvroDeserializer -transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer sink.trans.batch.size=200 ## Batch or trans or record sink.trans.mode=batch diff --git a/src/main/resources/pixels-sink.local.properties b/src/main/resources/pixels-sink.local.properties index d871cd2..512c38f 100644 --- a/src/main/resources/pixels-sink.local.properties +++ b/src/main/resources/pixels-sink.local.properties @@ -8,8 +8,7 @@ bootstrap.servers=localhost:29092 group.id=3107 auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -#value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventAvroDeserializer -value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer +sink.kafka.value.format=json # Topic & Database Config topic.prefix=postgresql.oltp_server consumer.capture_database=pixels_bench_sf1x @@ -40,8 +39,6 @@ sink.proto.maxRecords=1000000 sink.registry.url=http://localhost:8080/apis/registry/v2 # Transaction Config transaction.topic.suffix=transaction -#transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataAvroDeserializer -transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer sink.trans.batch.size=100 ## Batch or trans or record sink.trans.mode=batch diff --git a/src/main/resources/pixels-sink.properties b/src/main/resources/pixels-sink.properties index d6c0920..71cf117 100644 --- a/src/main/resources/pixels-sink.properties +++ b/src/main/resources/pixels-sink.properties @@ -8,7 +8,7 @@ bootstrap.servers=pixels_kafka:9092 group.id=docker-pixels-table auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer +sink.kafka.value.format=json # Topic & Database Config topic.prefix=oltp_server @@ -41,7 +41,6 @@ sink.registry.url=http://apicurio:8080/apis/registry/v2 # Transaction Config transaction.topic.suffix=transaction transaction.topic.group_id=transaction_consumer -transaction.topic.value.deserializer=io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer sink.trans.batch.size=100 sink.trans.mode=batch transaction.timeout=300 diff --git a/src/main/resources/pixels-sink.properties.template b/src/main/resources/pixels-sink.properties.template index c7b4a2d..4231326 100644 --- a/src/main/resources/pixels-sink.properties.template +++ b/src/main/resources/pixels-sink.properties.template @@ -3,7 +3,7 @@ bootstrap.servers=localhost:9092 group.id= auto.offset.reset=earliest key.deserializer=org.apache.kafka.common.serialization.StringDeserializer -value.deserializer=org.apache.kafka.common.serialization.StringDeserializer +sink.kafka.value.format=json # Topic & Database Config topic.prefix= @@ -11,4 +11,4 @@ consumer.capture_database= consumer.include_tables= # Sink Config -output.directory= \ No newline at end of file +output.directory= diff --git a/src/test/java/io/pixelsdb/pixels/sink/config/EngineValueFormatTest.java b/src/test/java/io/pixelsdb/pixels/sink/config/EngineValueFormatTest.java new file mode 100644 index 0000000..b61347a --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/config/EngineValueFormatTest.java @@ -0,0 +1,38 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + */ + +package io.pixelsdb.pixels.sink.config; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class EngineValueFormatTest +{ + @Test + void shouldDefaultToConnect() + { + assertEquals("connect", EngineValueFormat.resolve(null)); + assertEquals("connect", EngineValueFormat.resolve("")); + assertEquals("connect", EngineValueFormat.resolve("CONNECT")); + } + + @Test + void shouldRejectNonConnectOnResolve() + { + IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, + () -> EngineValueFormat.resolve("json")); + assertTrue(error.getMessage().contains("not implemented")); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/config/KafkaValueFormatTest.java b/src/test/java/io/pixelsdb/pixels/sink/config/KafkaValueFormatTest.java new file mode 100644 index 0000000..2801e4d --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/config/KafkaValueFormatTest.java @@ -0,0 +1,42 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + */ + +package io.pixelsdb.pixels.sink.config; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class KafkaValueFormatTest +{ + @Test + void shouldDefaultToJson() + { + assertEquals("json", KafkaValueFormat.resolve(null)); + assertEquals("json", KafkaValueFormat.resolve("")); + assertEquals("json", KafkaValueFormat.resolve(" ")); + } + + @Test + void shouldNormalizeConfiguredFormat() + { + assertEquals("json", KafkaValueFormat.resolve("JSON")); + assertEquals("avro", KafkaValueFormat.resolve("Avro")); + } + + @Test + void shouldRejectUnknownFormat() + { + assertThrows(IllegalArgumentException.class, + () -> KafkaValueFormat.resolve("protobuf")); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/config/LegacyKafkaValueFormatMigrationTest.java b/src/test/java/io/pixelsdb/pixels/sink/config/LegacyKafkaValueFormatMigrationTest.java new file mode 100644 index 0000000..600861d --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/config/LegacyKafkaValueFormatMigrationTest.java @@ -0,0 +1,90 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + */ + +package io.pixelsdb.pixels.sink.config; + +import org.junit.jupiter.api.Test; + +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class LegacyKafkaValueFormatMigrationTest +{ + @Test + void shouldDefaultToJsonWhenNoFormatOrLegacyKeys() + { + assertEquals("json", PixelsSinkConfig.resolveKafkaValueFormat(new Properties())); + } + + @Test + void shouldInferJsonFromLegacyDeserializerClass() + { + Properties props = new Properties(); + props.setProperty( + "value.deserializer", + "io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer"); + props.setProperty( + "transaction.topic.value.deserializer", + "io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataJsonDeserializer"); + assertEquals("json", PixelsSinkConfig.resolveKafkaValueFormat(props)); + } + + @Test + void shouldInferAvroFromLegacyDeserializerClass() + { + Properties props = new Properties(); + props.setProperty( + "value.deserializer", + "io.pixelsdb.pixels.sink.source.kafka.serde.RowChangeEventAvroDeserializer"); + assertEquals("avro", PixelsSinkConfig.resolveKafkaValueFormat(props)); + } + + @Test + void shouldPreferExplicitFormatAndIgnoreLegacyKeys() + { + Properties props = new Properties(); + props.setProperty("sink.kafka.value.format", "avro"); + props.setProperty( + "value.deserializer", + "io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer"); + assertEquals("avro", PixelsSinkConfig.resolveKafkaValueFormat(props)); + } + + @Test + void shouldFailFastOnConflictingLegacyFormats() + { + Properties props = new Properties(); + props.setProperty( + "value.deserializer", + "io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventJsonDeserializer"); + props.setProperty( + "transaction.topic.value.deserializer", + "io.pixelsdb.pixels.sink.conversion.debezium.TransactionMetadataAvroDeserializer"); + IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, + () -> PixelsSinkConfig.resolveKafkaValueFormat(props)); + assertTrue(error.getMessage().contains("Conflicting")); + } + + @Test + void shouldFailFastOnUnrecognizedLegacyClass() + { + Properties props = new Properties(); + props.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); + IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, + () -> PixelsSinkConfig.resolveKafkaValueFormat(props)); + assertTrue(error.getMessage().contains("Unable to migrate")); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/consumer/AvroConsumerTest.java b/src/test/java/io/pixelsdb/pixels/sink/consumer/AvroConsumerTest.java index d52322f..a1de077 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/consumer/AvroConsumerTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/consumer/AvroConsumerTest.java @@ -17,21 +17,17 @@ package io.pixelsdb.pixels.sink.consumer; -import io.apicurio.registry.rest.client.RegistryClient; -import io.apicurio.registry.rest.client.RegistryClientFactory; import io.apicurio.registry.serde.SerdeConfig; -import io.apicurio.registry.serde.avro.AvroKafkaDeserializer; -import io.pixelsdb.pixels.sink.SinkProto; import io.pixelsdb.pixels.sink.TestConfig; -import io.pixelsdb.pixels.sink.conversion.debezium.RowChangeEventAvroDeserializer; +import io.pixelsdb.pixels.sink.config.KafkaValueFormat; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.exception.SinkException; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; +import io.pixelsdb.pixels.sink.source.kafka.serde.KafkaRecordConverter; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.apache.kafka.common.serialization.StringDeserializer; import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.Tag; @@ -49,140 +45,58 @@ class AvroConsumerTest { private static final int MAX_POLL_CYCLES = 100; - private static KafkaConsumer getRowChangeEventAvroKafkaConsumer() - { - Properties props = new Properties(); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers()); - props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId()); - props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); - props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, RowChangeEventAvroDeserializer.class.getName()); - props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - props.put(SerdeConfig.REGISTRY_URL, registryUrl()); - props.put(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true"); - props.put(SerdeConfig.CHECK_PERIOD_MS, "30000"); - - KafkaConsumer consumer = new KafkaConsumer<>(props); - return consumer; - } - - private static void processRecord(RowChangeEvent event) - { -// RetinaProto.RowValue.Builder builder = RetinaProto.RowValue.newBuilder(); -// for (SinkProto.ColumnValue value : event.getRowRecord().getAfter().getValuesList()) { -// builder.addValues(value.getValue()); -// } -// builder.build(); - } - - private static KafkaConsumer getStringGenericRecordKafkaConsumer() - { - Properties props = new Properties(); - props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers()); - props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId()); - props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); - props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, AvroKafkaDeserializer.class.getName()); - props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - props.put(SerdeConfig.REGISTRY_URL, registryUrl()); - props.put(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true"); - props.put(SerdeConfig.CHECK_PERIOD_MS, "30000"); - - KafkaConsumer consumer = new KafkaConsumer<>(props); - return consumer; - } - - private static RowChangeEvent convertToRowChangeEvent(GenericRecord record, Schema schema) throws SinkException - { - return new RowChangeEvent(SinkProto.RowRecord.newBuilder().build(), null); - } - - private static void processRecord(ConsumerRecord record, RegistryClient registryClient) - { - try - { - GenericRecord avroRecord = record.value(); - Schema schema = avroRecord.getSchema(); - - String schemaId = getSchemaIdFromRegistry(registryClient, schema); - System.out.println("Schema ID: " + schemaId); - - RowChangeEvent event = convertToRowChangeEvent(avroRecord, schema); - - System.out.println("Successfully processed message:"); - System.out.println("Topic: " + record.topic()); - System.out.println("Partition: " + record.partition()); - System.out.println("Offset: " + record.offset()); - System.out.println("Event: " + event); - - } catch (Exception e) - { - System.err.println("Error processing message: " + e.getMessage()); - e.printStackTrace(); - } - } - - private static String getSchemaIdFromRegistry(RegistryClient client, Schema schema) - { - String schemaContent = schema.toString(); - try - { - return ""; - } catch (Exception e) - { - throw new RuntimeException("Schema not found in registry: " + schema.getFullName(), e); - } - } - @Test - void shouldConsumeGenericAvroRecords() + void shouldConsumeRowChangeEvents() throws Exception { - KafkaConsumer consumer = getStringGenericRecordKafkaConsumer(); + TestConfig.initializeIntegrationConfig(); + KafkaConsumer consumer = rowConsumer(); consumer.subscribe(Collections.singletonList(topic())); - RegistryClient registryClient = RegistryClientFactory.create(registryUrl()); - - try + try (KafkaRecordConverter converter = + KafkaRecordConverter.forRow(avroConverterProperties())) { int recordCount = 0; for (int i = 0; i < MAX_POLL_CYCLES; ++i) { - ConsumerRecords records = consumer.poll(Duration.ofMillis(100)); - for (ConsumerRecord record : records) + ConsumerRecords records = consumer.poll(Duration.ofMillis(100)); + for (ConsumerRecord record : records) { - processRecord(record, registryClient); - recordCount++; + RowChangeEvent event = converter.convert(record.topic(), record.value()); + if (event != null) + { + recordCount++; + } } } - assertTrue(recordCount > 0, "No Avro records were consumed"); + assertTrue(recordCount > 0, "No row-change records were consumed"); } finally { consumer.close(); } } - @Test - void shouldConsumeRowChangeEvents() throws Exception + private static KafkaConsumer rowConsumer() { - TestConfig.initializeIntegrationConfig(); - KafkaConsumer consumer = getRowChangeEventAvroKafkaConsumer(); - consumer.subscribe(Collections.singletonList(topic())); + Properties props = new Properties(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers()); + props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId()); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + props.put(SerdeConfig.REGISTRY_URL, registryUrl()); + props.put(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true"); + props.put(SerdeConfig.CHECK_PERIOD_MS, "30000"); + return new KafkaConsumer<>(props); + } - try - { - int recordCount = 0; - for (int i = 0; i < MAX_POLL_CYCLES; ++i) - { - ConsumerRecords records = consumer.poll(Duration.ofMillis(100)); - for (ConsumerRecord record : records) - { - processRecord(record.value()); - recordCount++; - } - } - assertTrue(recordCount > 0, "No row-change records were consumed"); - } finally - { - consumer.close(); - } + private static Properties avroConverterProperties() + { + Properties properties = new Properties(); + properties.put(PixelsSinkConstants.KAFKA_VALUE_FORMAT, KafkaValueFormat.AVRO); + properties.put(SerdeConfig.REGISTRY_URL, registryUrl()); + properties.put(SerdeConfig.AUTO_REGISTER_ARTIFACT, "true"); + properties.put(SerdeConfig.CHECK_PERIOD_MS, "30000"); + return properties; } private static String topic() @@ -213,4 +127,4 @@ private static String requiredProperty(String key) "Set -D" + key + " to run this integration test"); return value; } -} \ No newline at end of file +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectConverterTest.java new file mode 100644 index 0000000..bf024cc --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectConverterTest.java @@ -0,0 +1,202 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + */ + +package io.pixelsdb.pixels.sink.conversion.debezium.connect; + +import io.pixelsdb.pixels.common.metadata.SchemaTableName; +import io.pixelsdb.pixels.common.metadata.domain.Column; +import io.pixelsdb.pixels.common.metadata.domain.Table; +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.TestConfig; +import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.MySqlSourceAdapter; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import io.pixelsdb.pixels.sink.metadata.TableMetadata; +import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.source.SourceRecord; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class DebeziumConnectConverterTest +{ + private static final SchemaTableName TABLE = + new SchemaTableName("cdc_verify", "binlog_test"); + private static final Schema ROW_SCHEMA = SchemaBuilder.struct().optional() + .field("id", Schema.INT64_SCHEMA) + .field("name", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + private static final Schema SOURCE_SCHEMA = SchemaBuilder.struct() + .field("connector", Schema.STRING_SCHEMA) + .field("db", Schema.STRING_SCHEMA) + .field("table", Schema.STRING_SCHEMA) + .build(); + private static final Schema TRANSACTION_INFO_SCHEMA = SchemaBuilder.struct().optional() + .field("id", Schema.STRING_SCHEMA) + .field("total_order", Schema.INT64_SCHEMA) + .field("data_collection_order", Schema.INT64_SCHEMA) + .build(); + + private static TableMetadata previousMetadata; + private static boolean hadMetadata; + + private final DebeziumConnectRowConverter rowConverter = + new DebeziumConnectRowConverter( + TableMetadataRegistry.Instance(), MySqlSourceAdapter.INSTANCE); + private final DebeziumConnectTransactionConverter transactionConverter = + new DebeziumConnectTransactionConverter(MySqlSourceAdapter.INSTANCE); + + @BeforeAll + static void setUpConfig() throws Exception + { + TestConfig.initializeUnitConfig(); + Map registry = metadataRegistry(); + hadMetadata = registry.containsKey(TABLE); + previousMetadata = registry.get(TABLE); + registry.put(TABLE, tableMetadata()); + } + + @AfterAll + static void resetConfig() throws ReflectiveOperationException + { + Map registry = metadataRegistry(); + if (hadMetadata) + { + registry.put(TABLE, previousMetadata); + } + else + { + registry.remove(TABLE); + } + PixelsSinkConfigFactory.reset(); + } + + @Test + void shouldConvertInsertRow() throws Exception + { + RowChangeEvent event = rowConverter.convert(insertRecord()); + + assertTrue(event.isInsert()); + assertEquals("binlog_test", event.getTable()); + assertTrue(event.hasAfterData()); + assertEquals("gtid:1", event.getTransaction().getId()); + } + + @Test + void shouldConvertTransactionBoundary() + { + Schema collectionSchema = SchemaBuilder.struct() + .field("data_collection", Schema.STRING_SCHEMA) + .field("event_count", Schema.INT64_SCHEMA) + .build(); + Schema schema = SchemaBuilder.struct() + .field("status", Schema.STRING_SCHEMA) + .field("id", Schema.STRING_SCHEMA) + .field("event_count", Schema.OPTIONAL_INT64_SCHEMA) + .field("data_collections", SchemaBuilder.array(collectionSchema).optional().build()) + .field("ts_ms", Schema.INT64_SCHEMA) + .build(); + Struct collection = new Struct(collectionSchema) + .put("data_collection", "cdc_verify.binlog_test") + .put("event_count", 1L); + Struct value = new Struct(schema) + .put("status", "END") + .put("id", "gtid:9") + .put("event_count", 1L) + .put("data_collections", List.of(collection)) + .put("ts_ms", 1750000000000L); + + SinkProto.TransactionMetadata transaction = transactionConverter.convert( + new SourceRecord( + Map.of("server", "mysql-cdc"), + Map.of("file", "binlog.000004", "pos", 152L), + "mysql-cdc.transaction", + null, + null, + schema, + value)); + + assertEquals(SinkProto.TransactionStatus.END, transaction.getStatus()); + assertEquals("gtid:9", transaction.getId()); + assertEquals("cdc_verify.binlog_test", + transaction.getDataCollections(0).getDataCollection()); + } + + private SourceRecord insertRecord() + { + Struct source = new Struct(SOURCE_SCHEMA) + .put("connector", "mysql") + .put("db", "cdc_verify") + .put("table", "binlog_test"); + Struct transaction = new Struct(TRANSACTION_INFO_SCHEMA) + .put("id", "gtid:1") + .put("total_order", 1L) + .put("data_collection_order", 1L); + Schema envelopeSchema = SchemaBuilder.struct() + .field("before", ROW_SCHEMA) + .field("after", ROW_SCHEMA) + .field("source", SOURCE_SCHEMA) + .field("transaction", TRANSACTION_INFO_SCHEMA) + .field("op", Schema.STRING_SCHEMA) + .build(); + Struct envelope = new Struct(envelopeSchema) + .put("before", null) + .put("after", new Struct(ROW_SCHEMA).put("id", 1L).put("name", "row-1")) + .put("source", source) + .put("transaction", transaction) + .put("op", "c"); + return new SourceRecord( + Map.of("server", "mysql-cdc"), + Map.of("file", "binlog.000004", "pos", 152L), + "mysql-cdc.cdc_verify.binlog_test", + null, + null, + envelopeSchema, + envelope); + } + + private static TableMetadata tableMetadata() throws Exception + { + Table table = new Table(); + table.setId(1); + table.setName("binlog_test"); + return new TableMetadata(table, null, List.of( + column("id", "bigint"), + column("name", "varchar(64)"))); + } + + private static Column column(String name, String type) + { + Column column = new Column(); + column.setName(name); + column.setType(type); + return column; + } + + @SuppressWarnings("unchecked") + private static Map metadataRegistry() + throws ReflectiveOperationException + { + Field registry = TableMetadataRegistry.class.getDeclaredField("registry"); + registry.setAccessible(true); + return (Map) registry.get(TableMetadataRegistry.Instance()); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventJsonDeserializerTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonConverterTest.java similarity index 52% rename from src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventJsonDeserializerTest.java rename to src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonConverterTest.java index d7e369f..05a79d7 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/RowChangeEventJsonDeserializerTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonConverterTest.java @@ -15,7 +15,7 @@ * */ -package io.pixelsdb.pixels.sink.conversion.debezium; +package io.pixelsdb.pixels.sink.conversion.debezium.json; import io.pixelsdb.pixels.common.metadata.SchemaTableName; import io.pixelsdb.pixels.common.metadata.domain.Column; @@ -23,10 +23,10 @@ import io.pixelsdb.pixels.sink.SinkProto; import io.pixelsdb.pixels.sink.TestConfig; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.MySqlSourceAdapter; import io.pixelsdb.pixels.sink.event.RowChangeEvent; import io.pixelsdb.pixels.sink.metadata.TableMetadata; import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; -import org.apache.kafka.common.serialization.Deserializer; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -34,27 +34,32 @@ import java.io.IOException; import java.lang.reflect.Field; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; -import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import java.util.Objects; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -class RowChangeEventJsonDeserializerTest +class DebeziumJsonConverterTest { private static final SchemaTableName REGION_TABLE = new SchemaTableName("pixels_realtime_crud", "region"); + private static final SchemaTableName NATION_TABLE = + new SchemaTableName("pixels_realtime_crud", "nation"); + private static TableMetadata previousRegionMetadata; + private static TableMetadata previousNationMetadata; private static boolean hadRegionMetadata; + private static boolean hadNationMetadata; - private final Deserializer deserializer = new RowChangeEventJsonDeserializer(); - private final Deserializer transactionDeserializer = - new TransactionMetadataJsonDeserializer(); + private final DebeziumJsonRowConverter rowConverter = + new DebeziumJsonRowConverter(TableMetadataRegistry.Instance(), null); + private final DebeziumJsonTransactionConverter transactionConverter = + new DebeziumJsonTransactionConverter(MySqlSourceAdapter.INSTANCE); @BeforeAll static void setUpConfig() throws Exception @@ -63,83 +68,76 @@ static void setUpConfig() throws Exception Map registry = metadataRegistry(); hadRegionMetadata = registry.containsKey(REGION_TABLE); previousRegionMetadata = registry.get(REGION_TABLE); - registry.put(REGION_TABLE, regionMetadata()); + hadNationMetadata = registry.containsKey(NATION_TABLE); + previousNationMetadata = registry.get(NATION_TABLE); + // MySQL delete/update fixtures use uppercase column names. + registry.put(REGION_TABLE, tableMetadata(2, "region", + List.of("R_REGIONKEY", "R_NAME", "R_COMMENT"), + List.of("int", "string", "string"))); + registry.put(NATION_TABLE, tableMetadata(3, "nation", + List.of("n_nationkey", "n_name", "n_regionkey", "n_comment"), + List.of("int", "string", "int", "string"))); } @AfterAll static void resetConfig() throws ReflectiveOperationException { Map registry = metadataRegistry(); - if (hadRegionMetadata) - { - registry.put(REGION_TABLE, previousRegionMetadata); - } - else - { - registry.remove(REGION_TABLE); - } + restore(registry, REGION_TABLE, hadRegionMetadata, previousRegionMetadata); + restore(registry, NATION_TABLE, hadNationMetadata, previousNationMetadata); PixelsSinkConfigFactory.reset(); } - private String loadSchemaFromFile(String filename) throws IOException, URISyntaxException - { - ClassLoader classLoader = getClass().getClassLoader(); - return Files.readString(Paths.get( - Objects.requireNonNull(classLoader.getResource(filename)).toURI()), - StandardCharsets.UTF_8); - } - - // @ParameterizedTest -// @EnumSource(value = OperationType.class, names = {"INSERT", "UPDATE"}) -// //, , "SNAPSHOT" -// void shouldParseValidOperations(OperationType opType) throws Exception { -// String jsonData = loadSchemaFromFile("records/" + opType.name().toLowerCase() + ".json"); -// RowChangeEvent event = deserializer.deserialize("test_topic", jsonData.getBytes()); -// -// assertNotNull(event); -// assertEquals(opType, event.getOp()); -// assertEquals("region", event.getTable()); -// -// Map data = opType == OperationType.DELETE ? -// event.getBeforeData() : event.getAfterData(); -// assertNotNull(data); -// } - @Test void shouldHandleDeleteOperation() throws Exception { - String jsonData = loadSchemaFromFile("records/delete.json"); - RowChangeEvent event = deserializer.deserialize( - "test_topic", jsonData.getBytes(StandardCharsets.UTF_8)); - + RowChangeEvent event = convertFixture("records/delete.json"); assertTrue(event.isDelete()); -// assertNotNull(event.getBeforeData()); -// assertNull(event.getAfterData()); } @Test void shouldHandleUpdateOperation() throws Exception { - String jsonData = loadSchemaFromFile("records/update.json"); - RowChangeEvent event = deserializer.deserialize( - "test_topic", jsonData.getBytes(StandardCharsets.UTF_8)); - + RowChangeEvent event = convertFixture("records/update.json"); assertTrue(event.isUpdate()); assertEquals("region", event.getTable()); assertTrue(event.hasBeforeData()); assertTrue(event.hasAfterData()); } + @Test + void shouldHandlePostgresInsert() throws Exception + { + Map registry = metadataRegistry(); + TableMetadata mysqlRegion = registry.get(REGION_TABLE); + // Postgres insert fixture uses lowercase column names. + registry.put(REGION_TABLE, tableMetadata(2, "region", + List.of("r_regionkey", "r_name", "r_comment"), + List.of("int", "string", "string"))); + try + { + RowChangeEvent event = convertFixture("records/insert.json"); + assertTrue(event.isInsert()); + assertEquals("public.region", event.getFullTableName()); + assertEquals("779", event.getTransaction().getId()); + assertTrue(event.hasAfterData()); + } finally + { + registry.put(REGION_TABLE, mysqlRegion); + } + } @Test - void shouldHandleEmptyData() + void shouldHandlePostgresSnapshotNation() throws Exception { - RowChangeEvent event = deserializer.deserialize("empty_topic", new byte[0]); - assertNull(event); + RowChangeEvent event = convertFixture("records/nation.json"); + assertTrue(event.isSnapshot()); + assertEquals("public.nation", event.getFullTableName()); + assertTrue(event.hasAfterData()); } @Test - void shouldDeserializeTransactionMetadataWithConfiguredConnector() + void shouldConvertTransactionMetadata() throws Exception { String json = """ { @@ -155,8 +153,8 @@ void shouldDeserializeTransactionMetadataWithConfiguredConnector() } """; - SinkProto.TransactionMetadata transaction = transactionDeserializer.deserialize( - "transaction", json.getBytes(StandardCharsets.UTF_8)); + SinkProto.TransactionMetadata transaction = transactionConverter.convert( + json.getBytes(StandardCharsets.UTF_8)); assertEquals(SinkProto.TransactionStatus.END, transaction.getStatus()); assertEquals("mysql-tx-1", transaction.getId()); @@ -164,15 +162,32 @@ void shouldDeserializeTransactionMetadataWithConfiguredConnector() transaction.getDataCollections(0).getDataCollection()); } - private static TableMetadata regionMetadata() throws Exception + private RowChangeEvent convertFixture(String filename) throws Exception + { + return rowConverter.convert(loadFixture(filename).getBytes(StandardCharsets.UTF_8)); + } + + private String loadFixture(String filename) throws IOException, URISyntaxException + { + ClassLoader classLoader = getClass().getClassLoader(); + return Files.readString(Paths.get( + Objects.requireNonNull(classLoader.getResource(filename)).toURI()), + StandardCharsets.UTF_8); + } + + private static TableMetadata tableMetadata( + long id, String name, List columnNames, List columnTypes) + throws Exception { Table table = new Table(); - table.setId(2); - table.setName("region"); - return new TableMetadata(table, null, List.of( - column("R_REGIONKEY", "int"), - column("R_NAME", "string"), - column("R_COMMENT", "string"))); + table.setId(id); + table.setName(name); + List columns = new java.util.ArrayList<>(columnNames.size()); + for (int i = 0; i < columnNames.size(); i++) + { + columns.add(column(columnNames.get(i), columnTypes.get(i))); + } + return new TableMetadata(table, null, columns); } private static Column column(String name, String type) @@ -183,6 +198,22 @@ private static Column column(String name, String type) return column; } + private static void restore( + Map registry, + SchemaTableName key, + boolean hadPrevious, + TableMetadata previous) + { + if (hadPrevious) + { + registry.put(key, previous); + } + else + { + registry.remove(key); + } + } + @SuppressWarnings("unchecked") private static Map metadataRegistry() throws ReflectiveOperationException @@ -192,4 +223,3 @@ private static Map metadataRegistry() return (Map) registry.get(TableMetadataRegistry.Instance()); } } - diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizerTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumEnvelopeNormalizerTest.java similarity index 77% rename from src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizerTest.java rename to src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumEnvelopeNormalizerTest.java index bb02149..9843278 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumEnvelopeNormalizerTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumEnvelopeNormalizerTest.java @@ -8,7 +8,7 @@ * published by the Free Software Foundation, either version 3 of * the License, or (at your option) any later version. */ -package io.pixelsdb.pixels.sink.conversion.debezium; +package io.pixelsdb.pixels.sink.conversion.debezium.support; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -17,9 +17,9 @@ import io.pixelsdb.pixels.core.TypeDescription; import io.pixelsdb.pixels.sink.SinkProto; import io.pixelsdb.pixels.sink.TestConfig; -import io.pixelsdb.pixels.sink.conversion.debezium.source.DebeziumSourceAdapterRegistry; -import io.pixelsdb.pixels.sink.conversion.debezium.source.MySqlSourceAdapter; -import io.pixelsdb.pixels.sink.conversion.debezium.source.PostgresSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapterRegistry; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.MySqlSourceAdapter; +import io.pixelsdb.pixels.sink.conversion.debezium.dialect.PostgresSourceAdapter; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; import io.pixelsdb.pixels.sink.event.RowChangeEvent; import io.pixelsdb.pixels.sink.exception.SinkException; @@ -46,12 +46,8 @@ class DebeziumEnvelopeNormalizerTest private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private static final SchemaTableName MYSQL_TABLE = new SchemaTableName("cdc_verify", "binlog_test"); - private static final SchemaTableName POSTGRES_TABLE = - new SchemaTableName("pixels_realtime_crud", "region"); private static TableMetadata previousMySqlMetadata; - private static TableMetadata previousPostgresMetadata; private static boolean hadMySqlMetadata; - private static boolean hadPostgresMetadata; @BeforeAll static void setUpConfig() throws Exception @@ -60,10 +56,7 @@ static void setUpConfig() throws Exception Map registry = metadataRegistry(); hadMySqlMetadata = registry.containsKey(MYSQL_TABLE); previousMySqlMetadata = registry.get(MYSQL_TABLE); - hadPostgresMetadata = registry.containsKey(POSTGRES_TABLE); - previousPostgresMetadata = registry.get(POSTGRES_TABLE); registry.put(MYSQL_TABLE, tableMetadata(1, "binlog_test")); - registry.put(POSTGRES_TABLE, tableMetadata(2, "region")); } @AfterAll @@ -78,14 +71,6 @@ static void resetConfig() throws ReflectiveOperationException { registry.remove(MYSQL_TABLE); } - if (hadPostgresMetadata) - { - registry.put(POSTGRES_TABLE, previousPostgresMetadata); - } - else - { - registry.remove(POSTGRES_TABLE); - } PixelsSinkConfigFactory.reset(); } @@ -104,7 +89,7 @@ void shouldNormalizeMySqlSourceWithoutSchema() throws SinkException } @Test - void shouldNormalizePostgresSourceAndTransactionId() throws SinkException + void shouldNormalizePostgresSourceStruct() { Schema sourceSchema = SchemaBuilder.struct() .field("connector", Schema.STRING_SCHEMA) @@ -124,9 +109,6 @@ void shouldNormalizePostgresSourceAndTransactionId() throws SinkException assertEquals("pixels_realtime_crud", normalized.getDb()); assertEquals("public", normalized.getSchema()); assertEquals("region", normalized.getTable()); - assertEquals("public.region", rowChangeEventFor(normalized).getFullTableName()); - assertEquals("779", - PostgresSourceAdapter.INSTANCE.normalizeTransactionId("779:39110592")); assertEquals("779", PostgresSourceAdapter.INSTANCE.normalizeTransactionId("779")); } @@ -196,31 +178,6 @@ void shouldPreserveMySqlGtidAsOpaqueTransactionId() throws Exception transaction.getDataCollections(0).getDataCollection()); } - @Test - void shouldRoundTripCanonicalProto() throws Exception - { - SinkProto.RowRecord record = SinkProto.RowRecord.newBuilder() - .setOp(SinkProto.OperationType.INSERT) - .setSource(DebeziumEnvelopeNormalizer.normalizeSource( - mysqlSource(), MySqlSourceAdapter.INSTANCE)) - .setTransaction(SinkProto.TransactionInfo.newBuilder() - .setId("gtid:8") - .setTotalOrder(1) - .setDataCollectionOrder(1)) - .setAfter(SinkProto.RowValue.newBuilder() - .addValues(SinkProto.ColumnValue.newBuilder() - .setValue(com.google.protobuf.ByteString.copyFromUtf8(""))) - .addValues(SinkProto.ColumnValue.newBuilder() - .setValue(com.google.protobuf.ByteString.EMPTY))) - .build(); - - SinkProto.RowRecord restored = - SinkProto.RowRecord.parseFrom(record.toByteArray()); - - assertEquals(record, restored); - assertEquals(0, restored.getAfter().getValues(1).getValue().size()); - } - private Struct mysqlSource() { Schema sourceSchema = SchemaBuilder.struct() diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowValueConverterTest.java similarity index 82% rename from src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverterTest.java rename to src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowValueConverterTest.java index e078f7f..a193a49 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowValueConverterTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowValueConverterTest.java @@ -15,13 +15,12 @@ * */ -package io.pixelsdb.pixels.sink.conversion.debezium; +package io.pixelsdb.pixels.sink.conversion.debezium.support; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import io.pixelsdb.pixels.core.TypeDescription; import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.util.TestDateUtil; import org.apache.avro.generic.GenericData; import org.apache.kafka.connect.data.Decimal; import org.apache.kafka.connect.data.Schema; @@ -32,31 +31,14 @@ import java.math.BigDecimal; import java.nio.ByteBuffer; import java.util.Base64; -import java.util.Date; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -public class DebeziumRowValueConverterTest +class DebeziumRowValueConverterTest { - // BqQ= 17 - // JbR7 24710.35 -// @ParameterizedTest -// @CsvSource({ -// // encodedValue, expectedValue, precision, scale -// "BqQ=, 17.00, 15, 2", -// "JbR7, 24710.35, 15, 2", -// }) -// void testParseDecimalValid(String encodedValue, String expectedValue, int precision, int scale) { -// JsonNode node = new TextNode(encodedValue); -// TypeDescription type = TypeDescription.createDecimal(precision, scale); -// DebeziumRowValueConverter rowDataParser = new DebeziumRowValueConverter(type); -// BigDecimal result = rowDataParser.parseDecimal(node, type); -// assertEquals(new BigDecimal(expectedValue), result); -// } - @Test - void shouldEncodeCanonicalStructValues() throws Exception + void shouldEncodeCanonicalStructJsonAndAvroValues() throws Exception { TypeDescription typeDescription = TypeDescription.createSchemaFromStrings( List.of("id", "name", "amount", "note", "empty_value"), @@ -146,14 +128,4 @@ void shouldPreserveExistingIntegerWidths() throws Exception assertEquals(2, builder.getValues(1).getValue().size()); assertEquals(4, builder.getValues(2).getValue().size()); } - - @Test - void shouldConvertDebeziumEpochDay() - { - int day = 17059; - Date debeziumDate = TestDateUtil.fromDebeziumDate(day); - String dayString = TestDateUtil.convertDateToDayString(debeziumDate); - - assertEquals("2016-09-15", dayString); - } } diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/kafka/KafkaRecordConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/kafka/KafkaRecordConverterTest.java deleted file mode 100644 index 55559cf..0000000 --- a/src/test/java/io/pixelsdb/pixels/sink/conversion/kafka/KafkaRecordConverterTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * This file is part of Pixels. - * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . - */ - -package io.pixelsdb.pixels.sink.conversion.kafka; - -import org.apache.kafka.common.serialization.Deserializer; -import org.junit.jupiter.api.Test; - -import java.nio.charset.StandardCharsets; -import java.util.Properties; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -class KafkaRecordConverterTest -{ - @Test - void shouldCreateConfiguredDeserializerAndConvertRawBytes() - { - String converterKey = "test.converter"; - Properties properties = new Properties(); - properties.put(converterKey, Utf8Deserializer.class.getName()); - - try (KafkaRecordConverter converter = - KafkaRecordConverter.create(properties, converterKey)) - { - assertEquals("record", converter.convert( - "topic", "record".getBytes(StandardCharsets.UTF_8))); - } - } - - public static class Utf8Deserializer implements Deserializer - { - @Override - public String deserialize(String topic, byte[] data) - { - return new String(data, StandardCharsets.UTF_8); - } - } -} diff --git a/src/test/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumerTest.java b/src/test/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifierTest.java similarity index 72% rename from src/test/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumerTest.java rename to src/test/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifierTest.java index c422ec4..397e8bf 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumerTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifierTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * * This file is part of Pixels. * @@ -8,8 +8,10 @@ * published by the Free Software Foundation, either version 3 of * the License, or (at your option) any later version. */ + package io.pixelsdb.pixels.sink.source.engine; +import io.pixelsdb.pixels.common.metadata.SchemaTableName; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.SchemaBuilder; import org.apache.kafka.connect.data.Struct; @@ -18,13 +20,13 @@ import java.util.Map; -import static io.pixelsdb.pixels.sink.source.engine.PixelsDebeziumConsumer.RecordType.ROW; -import static io.pixelsdb.pixels.sink.source.engine.PixelsDebeziumConsumer.RecordType.TOMBSTONE; -import static io.pixelsdb.pixels.sink.source.engine.PixelsDebeziumConsumer.RecordType.TRANSACTION; -import static io.pixelsdb.pixels.sink.source.engine.PixelsDebeziumConsumer.RecordType.UNKNOWN_CONTROL; +import static io.pixelsdb.pixels.sink.source.engine.DebeziumRecordType.ROW; +import static io.pixelsdb.pixels.sink.source.engine.DebeziumRecordType.TOMBSTONE; +import static io.pixelsdb.pixels.sink.source.engine.DebeziumRecordType.TRANSACTION; +import static io.pixelsdb.pixels.sink.source.engine.DebeziumRecordType.UNKNOWN_CONTROL; import static org.junit.jupiter.api.Assertions.assertEquals; -class PixelsDebeziumConsumerTest +class ConnectEventClassifierTest { private static final String PREFIX = "mysql-cdc"; private static final String ROW_TOPIC = PREFIX + ".cdc_verify.binlog_test"; @@ -42,12 +44,14 @@ class PixelsDebeziumConsumerTest .field("pos", Schema.INT64_SCHEMA) .field("row", Schema.INT32_SCHEMA) .build(); - private static final Schema TRANSACTION_SCHEMA = SchemaBuilder.struct().optional() + private static final Schema TRANSACTION_INFO_SCHEMA = SchemaBuilder.struct().optional() .field("id", Schema.STRING_SCHEMA) .field("total_order", Schema.INT64_SCHEMA) .field("data_collection_order", Schema.INT64_SCHEMA) .build(); + private final ConnectEventClassifier classifier = new ConnectEventClassifier(); + @Test void shouldClassifyMySqlRowOperations() { @@ -57,30 +61,6 @@ void shouldClassifyMySqlRowOperations() assertEquals(ROW, classify(rowRecord("r", null, row(1), "", 0))); } - @Test - void shouldTreatTwoRowsFromOneWriteRowsAsIndependentEvents() - { - SourceRecord first = rowRecord("c", null, row(1), "gtid:4", 1); - SourceRecord second = rowRecord("c", null, row(2), "gtid:4", 2); - - assertEquals(ROW, classify(first)); - assertEquals(ROW, classify(second)); - assertEquals("gtid:4", - ((Struct) ((Struct) first.value()).get("transaction")).getString("id")); - assertEquals("gtid:4", - ((Struct) ((Struct) second.value()).get("transaction")).getString("id")); - } - - @Test - void shouldKeepReplaceDeleteAndInsertInOneTransaction() - { - SourceRecord delete = rowRecord("d", row(2), null, "gtid:7", 1); - SourceRecord insert = rowRecord("c", null, row(2), "gtid:7", 2); - - assertEquals(ROW, classify(delete)); - assertEquals(ROW, classify(insert)); - } - @Test void shouldClassifyTransactionBoundaries() { @@ -109,21 +89,27 @@ void shouldRejectMalformedRows() classify(rowRecord("d", null, null, "gtid:6", 1))); } - private PixelsDebeziumConsumer.RecordType classify(SourceRecord record) + @Test + void shouldExtractTableStreamKey() + { + SchemaTableName table = classifier.tableOf(rowRecord("c", null, row(1), "gtid:1", 1)); + assertEquals("cdc_verify", table.getSchemaName()); + assertEquals("binlog_test", table.getTableName()); + } + + private DebeziumRecordType classify(SourceRecord record) { - return PixelsDebeziumConsumer.classify(record, TRANSACTION_TOPIC); + return classifier.classify(record, TRANSACTION_TOPIC); } private SourceRecord rowRecord( String op, Struct before, Struct after, String transactionId, long order) { - Schema transactionSchema = transactionSchema(); - Struct transaction = transactionId.isEmpty() ? null : new Struct(transactionSchema) + Struct transaction = transactionId.isEmpty() ? null : new Struct(TRANSACTION_INFO_SCHEMA) .put("id", transactionId) .put("total_order", order) .put("data_collection_order", order); - Schema sourceSchema = sourceSchema(); - Struct source = new Struct(sourceSchema) + Struct source = new Struct(SOURCE_SCHEMA) .put("connector", "mysql") .put("db", "cdc_verify") .put("table", "binlog_test") @@ -133,10 +119,10 @@ private SourceRecord rowRecord( .put("row", (int) order); Schema envelopeSchema = SchemaBuilder.struct() .name("mysql-cdc.cdc_verify.binlog_test.Envelope") - .field("before", rowSchema()) - .field("after", rowSchema()) - .field("source", sourceSchema) - .field("transaction", transactionSchema) + .field("before", ROW_SCHEMA) + .field("after", ROW_SCHEMA) + .field("source", SOURCE_SCHEMA) + .field("transaction", TRANSACTION_INFO_SCHEMA) .field("op", Schema.STRING_SCHEMA) .build(); Struct envelope = new Struct(envelopeSchema) @@ -184,21 +170,6 @@ private SourceRecord sourceRecord(String topic, Schema schema, Object value) private Struct row(long id) { - return new Struct(rowSchema()).put("id", id).put("name", "row-" + id); - } - - private Schema rowSchema() - { - return ROW_SCHEMA; - } - - private Schema sourceSchema() - { - return SOURCE_SCHEMA; - } - - private Schema transactionSchema() - { - return TRANSACTION_SCHEMA; + return new Struct(ROW_SCHEMA).put("id", id).put("name", "row-" + id); } } diff --git a/src/test/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverterTest.java new file mode 100644 index 0000000..b0cfae5 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverterTest.java @@ -0,0 +1,79 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + */ + +package io.pixelsdb.pixels.sink.source.kafka.serde; + +import io.pixelsdb.pixels.sink.SinkProto; +import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; +import io.pixelsdb.pixels.sink.event.RowChangeEvent; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class KafkaRecordConverterTest +{ + @Test + void shouldCreateJsonRowAndTransactionConverters() + { + Properties properties = jsonProperties(); + try (KafkaRecordConverter row = KafkaRecordConverter.forRow(properties); + KafkaRecordConverter tx = + KafkaRecordConverter.forTransaction(properties)) + { + assertNotNull(row); + assertNotNull(tx); + } + } + + @Test + void shouldReturnNullForEmptyOrNullRowPayload() + { + try (KafkaRecordConverter converter = + KafkaRecordConverter.forRow(jsonProperties())) + { + assertNull(converter.convert("topic", null)); + assertNull(converter.convert("topic", new byte[0])); + } + } + + @Test + void shouldSwallowInvalidRowPayload() + { + try (KafkaRecordConverter converter = + KafkaRecordConverter.forRow(jsonProperties())) + { + assertNull(converter.convert("topic", "not-json".getBytes(StandardCharsets.UTF_8))); + } + } + + @Test + void shouldPropagateInvalidTransactionPayload() + { + try (KafkaRecordConverter converter = + KafkaRecordConverter.forTransaction(jsonProperties())) + { + assertThrows(RuntimeException.class, + () -> converter.convert("topic", "not-json".getBytes(StandardCharsets.UTF_8))); + } + } + + private static Properties jsonProperties() + { + Properties properties = new Properties(); + properties.put(PixelsSinkConstants.KAFKA_VALUE_FORMAT, "json"); + return properties; + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoderTest.java b/src/test/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoderTest.java new file mode 100644 index 0000000..7b991b2 --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoderTest.java @@ -0,0 +1,80 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + */ + +package io.pixelsdb.pixels.sink.util.concurrent; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +class OrderedBatchDecoderTest +{ + private ExecutorService executor; + + @AfterEach + void tearDown() + { + if (executor != null) + { + DecodeExecutors.shutdownGracefully(executor, 5, null); + } + } + + @Test + void shouldPreserveInputOrder() throws Exception + { + executor = DecodeExecutors.newFixedCallerRuns(4, "ordered-batch-test"); + List input = List.of(3, 1, 2); + List decoded = OrderedBatchDecoder.decodeInOrder( + executor, input, value -> + { + try + { + Thread.sleep(5L * value); + } catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + return value * 10; + }); + assertEquals(List.of(30, 10, 20), decoded); + } + + @Test + void shouldReplaceFailedRecordsWithNull() throws Exception + { + executor = DecodeExecutors.newFixedCallerRuns(2, "ordered-batch-fail"); + AtomicInteger calls = new AtomicInteger(); + List decoded = OrderedBatchDecoder.decodeInOrder( + executor, + List.of(1, 2, 3), + value -> + { + calls.incrementAndGet(); + if (value == 2) + { + throw new IllegalStateException("boom"); + } + return value; + }); + assertEquals(3, calls.get()); + assertEquals(1, decoded.get(0)); + assertNull(decoded.get(1)); + assertEquals(3, decoded.get(2)); + } +} diff --git a/src/test/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoderTest.java b/src/test/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoderTest.java new file mode 100644 index 0000000..60c1f0c --- /dev/null +++ b/src/test/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoderTest.java @@ -0,0 +1,137 @@ +/* + * Copyright 2026 PixelsDB. + * + * This file is part of Pixels. + * + * Pixels is free software: you can redistribute it and/or modify + * it under the terms of the Affero GNU General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + */ + +package io.pixelsdb.pixels.sink.util.concurrent; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class StreamOrderedDecoderTest +{ + @Test + void shouldPublishSameStreamInSubmitOrder() throws Exception + { + try (StreamOrderedDecoder decoder = + new StreamOrderedDecoder(4, "stream-order-test")) + { + decoder.start(); + List published = Collections.synchronizedList(new ArrayList<>()); + CountDownLatch slowStarted = new CountDownLatch(1); + CountDownLatch releaseSlow = new CountDownLatch(1); + + CompletableFuture first = decoder.submit( + "s1", + 1, + value -> + { + slowStarted.countDown(); + assertTrue(releaseSlow.await(5, TimeUnit.SECONDS)); + return value; + }, + result -> published.add(result.value())); + assertTrue(slowStarted.await(5, TimeUnit.SECONDS)); + + CompletableFuture second = decoder.submit( + "s1", + 2, + value -> value, + result -> published.add(result.value())); + + releaseSlow.countDown(); + first.get(5, TimeUnit.SECONDS); + second.get(5, TimeUnit.SECONDS); + assertEquals(List.of(1, 2), published); + } + } + + @Test + void shouldAllowDifferentStreamsToPublishIndependently() throws Exception + { + try (StreamOrderedDecoder decoder = + new StreamOrderedDecoder(4, "stream-parallel-test")) + { + decoder.start(); + List published = Collections.synchronizedList(new ArrayList<>()); + CountDownLatch aStarted = new CountDownLatch(1); + CountDownLatch releaseA = new CountDownLatch(1); + + CompletableFuture a = decoder.submit( + "a", + "a1", + value -> + { + aStarted.countDown(); + assertTrue(releaseA.await(5, TimeUnit.SECONDS)); + return value; + }, + result -> published.add(result.value())); + assertTrue(aStarted.await(5, TimeUnit.SECONDS)); + + CompletableFuture b = decoder.submit( + "b", + "b1", + value -> value, + result -> published.add(result.value())); + b.get(5, TimeUnit.SECONDS); + assertEquals(List.of("b1"), published); + + releaseA.countDown(); + a.get(5, TimeUnit.SECONDS); + assertEquals(List.of("b1", "a1"), published); + } + } + + @Test + void shouldFailFastOnSameStreamWhenPreviousPublishFails() throws Exception + { + try (StreamOrderedDecoder decoder = + new StreamOrderedDecoder(4, "stream-fail-fast-test")) + { + decoder.start(); + List published = Collections.synchronizedList(new ArrayList<>()); + + CompletableFuture first = decoder.submit( + "s1", + 1, + value -> value, + result -> + { + throw new IllegalStateException("publish failed"); + }); + ExecutionException firstError = assertThrows( + ExecutionException.class, () -> first.get(5, TimeUnit.SECONDS)); + assertInstanceOf(IllegalStateException.class, firstError.getCause()); + + // Submit after the stream has already failed; the failed tail must remain. + CompletableFuture second = decoder.submit( + "s1", + 2, + value -> value, + result -> published.add(result.value())); + ExecutionException secondError = assertThrows( + ExecutionException.class, () -> second.get(5, TimeUnit.SECONDS)); + assertInstanceOf(IllegalStateException.class, secondError.getCause()); + assertTrue(published.isEmpty()); + } + } +} From d0c5f7599bced3655da003d92c0be80dca47206a Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Fri, 31 Jul 2026 23:41:28 +0800 Subject: [PATCH 10/13] refactor: align test names and packages with production layout --- .../{TestSplitString.java => SplitStringTest.java} | 2 +- .../debezium/json/DebeziumJsonConverterTest.java | 8 ++++---- ...tFreshnessClient.java => FreshnessClientTest.java} | 2 +- .../{TestIndexService.java => IndexServiceTest.java} | 8 +------- .../{consumer => source/kafka}/AvroConsumerTest.java | 2 +- .../TransServiceTest.java} | 6 +++--- .../ProtoWriterTest.java} | 11 ++--------- .../RetinaWriterIntegrationTest.java} | 10 +++++----- .../retina}/RetinaWriterTest.java | 2 +- .../records/{delete.json => mysql-region-delete.json} | 0 .../records/{update.json => mysql-region-update.json} | 0 .../{nation.json => postgresql-nation-snapshot.json} | 0 .../{insert.json => postgresql-region-insert.json} | 0 13 files changed, 19 insertions(+), 32 deletions(-) rename src/test/java/io/pixelsdb/pixels/sink/{TestSplitString.java => SplitStringTest.java} (98%) rename src/test/java/io/pixelsdb/pixels/sink/freshness/{TestFreshnessClient.java => FreshnessClientTest.java} (99%) rename src/test/java/io/pixelsdb/pixels/sink/metadata/{TestIndexService.java => IndexServiceTest.java} (97%) rename src/test/java/io/pixelsdb/pixels/sink/{consumer => source/kafka}/AvroConsumerTest.java (99%) rename src/test/java/io/pixelsdb/pixels/sink/{concurrent/TransactionServiceTest.java => writer/TransServiceTest.java} (97%) rename src/test/java/io/pixelsdb/pixels/sink/writer/{TestProtoWriter.java => proto/ProtoWriterTest.java} (96%) rename src/test/java/io/pixelsdb/pixels/sink/writer/{TestRetinaWriter.java => retina/RetinaWriterIntegrationTest.java} (99%) rename src/test/java/io/pixelsdb/pixels/sink/{concurrent => writer/retina}/RetinaWriterTest.java (99%) rename src/test/resources/records/{delete.json => mysql-region-delete.json} (100%) rename src/test/resources/records/{update.json => mysql-region-update.json} (100%) rename src/test/resources/records/{nation.json => postgresql-nation-snapshot.json} (100%) rename src/test/resources/records/{insert.json => postgresql-region-insert.json} (100%) diff --git a/src/test/java/io/pixelsdb/pixels/sink/TestSplitString.java b/src/test/java/io/pixelsdb/pixels/sink/SplitStringTest.java similarity index 98% rename from src/test/java/io/pixelsdb/pixels/sink/TestSplitString.java rename to src/test/java/io/pixelsdb/pixels/sink/SplitStringTest.java index 5abe465..f73a1da 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/TestSplitString.java +++ b/src/test/java/io/pixelsdb/pixels/sink/SplitStringTest.java @@ -24,7 +24,7 @@ * Created at: 29/04/2021 * Author: hank */ -class TestSplitString +class SplitStringTest { @Test void shouldSplitPipeDelimitedRecordAndKeepTrailingEmptyField() diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonConverterTest.java index 05a79d7..14371d2 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonConverterTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonConverterTest.java @@ -91,14 +91,14 @@ static void resetConfig() throws ReflectiveOperationException @Test void shouldHandleDeleteOperation() throws Exception { - RowChangeEvent event = convertFixture("records/delete.json"); + RowChangeEvent event = convertFixture("records/mysql-region-delete.json"); assertTrue(event.isDelete()); } @Test void shouldHandleUpdateOperation() throws Exception { - RowChangeEvent event = convertFixture("records/update.json"); + RowChangeEvent event = convertFixture("records/mysql-region-update.json"); assertTrue(event.isUpdate()); assertEquals("region", event.getTable()); assertTrue(event.hasBeforeData()); @@ -116,7 +116,7 @@ void shouldHandlePostgresInsert() throws Exception List.of("int", "string", "string"))); try { - RowChangeEvent event = convertFixture("records/insert.json"); + RowChangeEvent event = convertFixture("records/postgresql-region-insert.json"); assertTrue(event.isInsert()); assertEquals("public.region", event.getFullTableName()); assertEquals("779", event.getTransaction().getId()); @@ -130,7 +130,7 @@ void shouldHandlePostgresInsert() throws Exception @Test void shouldHandlePostgresSnapshotNation() throws Exception { - RowChangeEvent event = convertFixture("records/nation.json"); + RowChangeEvent event = convertFixture("records/postgresql-nation-snapshot.json"); assertTrue(event.isSnapshot()); assertEquals("public.nation", event.getFullTableName()); assertTrue(event.hasAfterData()); diff --git a/src/test/java/io/pixelsdb/pixels/sink/freshness/TestFreshnessClient.java b/src/test/java/io/pixelsdb/pixels/sink/freshness/FreshnessClientTest.java similarity index 99% rename from src/test/java/io/pixelsdb/pixels/sink/freshness/TestFreshnessClient.java rename to src/test/java/io/pixelsdb/pixels/sink/freshness/FreshnessClientTest.java index f6ce0c8..3c537c2 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/freshness/TestFreshnessClient.java +++ b/src/test/java/io/pixelsdb/pixels/sink/freshness/FreshnessClientTest.java @@ -41,7 +41,7 @@ // We extend FreshnessClient to access the protected queryAndCalculateFreshness method @Tag("integration") -class TestFreshnessClient +class FreshnessClientTest { // Mocks for JDBC dependencies diff --git a/src/test/java/io/pixelsdb/pixels/sink/metadata/TestIndexService.java b/src/test/java/io/pixelsdb/pixels/sink/metadata/IndexServiceTest.java similarity index 97% rename from src/test/java/io/pixelsdb/pixels/sink/metadata/TestIndexService.java rename to src/test/java/io/pixelsdb/pixels/sink/metadata/IndexServiceTest.java index efccfc8..01e4b6d 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/metadata/TestIndexService.java +++ b/src/test/java/io/pixelsdb/pixels/sink/metadata/IndexServiceTest.java @@ -36,14 +36,8 @@ import java.nio.ByteBuffer; -/** - * @package: io.pixelsdb.pixels.sink.metadata - * @className: TestIndexService - * @author: AntiO2 - * @date: 2025/8/5 04:34 - */ @Tag("integration") -class TestIndexService +class IndexServiceTest { private final MetadataService metadataService = MetadataService.Instance(); diff --git a/src/test/java/io/pixelsdb/pixels/sink/consumer/AvroConsumerTest.java b/src/test/java/io/pixelsdb/pixels/sink/source/kafka/AvroConsumerTest.java similarity index 99% rename from src/test/java/io/pixelsdb/pixels/sink/consumer/AvroConsumerTest.java rename to src/test/java/io/pixelsdb/pixels/sink/source/kafka/AvroConsumerTest.java index a1de077..ccb0ca3 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/consumer/AvroConsumerTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/source/kafka/AvroConsumerTest.java @@ -15,7 +15,7 @@ * */ -package io.pixelsdb.pixels.sink.consumer; +package io.pixelsdb.pixels.sink.source.kafka; import io.apicurio.registry.serde.SerdeConfig; import io.pixelsdb.pixels.sink.TestConfig; diff --git a/src/test/java/io/pixelsdb/pixels/sink/concurrent/TransactionServiceTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/TransServiceTest.java similarity index 97% rename from src/test/java/io/pixelsdb/pixels/sink/concurrent/TransactionServiceTest.java rename to src/test/java/io/pixelsdb/pixels/sink/writer/TransServiceTest.java index 3ea3b18..e907292 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/concurrent/TransactionServiceTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/TransServiceTest.java @@ -18,7 +18,7 @@ * . */ -package io.pixelsdb.pixels.sink.concurrent; +package io.pixelsdb.pixels.sink.writer; import io.pixelsdb.pixels.common.exception.TransException; @@ -37,9 +37,9 @@ @Slf4j @Tag("integration") -class TransactionServiceTest +class TransServiceTest { - private static final Logger logger = LoggerFactory.getLogger(TransactionServiceTest.class); + private static final Logger logger = LoggerFactory.getLogger(TransServiceTest.class); @Test public void testTransactionService() diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/TestProtoWriter.java b/src/test/java/io/pixelsdb/pixels/sink/writer/proto/ProtoWriterTest.java similarity index 96% rename from src/test/java/io/pixelsdb/pixels/sink/writer/TestProtoWriter.java rename to src/test/java/io/pixelsdb/pixels/sink/writer/proto/ProtoWriterTest.java index dec41bc..34ae0f7 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/TestProtoWriter.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/proto/ProtoWriterTest.java @@ -16,14 +16,13 @@ */ -package io.pixelsdb.pixels.sink.writer; +package io.pixelsdb.pixels.sink.writer.proto; import com.google.protobuf.ByteString; import io.pixelsdb.pixels.common.physical.*; import io.pixelsdb.pixels.sink.SinkProto; import io.pixelsdb.pixels.sink.TestConfig; -import io.pixelsdb.pixels.sink.writer.proto.ProtoWriter; import io.pixelsdb.pixels.storage.localfs.PhysicalLocalReader; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -39,13 +38,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -/** - * @package: io.pixelsdb.pixels.sink.writer - * @className: TestProtoWriter - * @author: AntiO2 - * @date: 2025/10/5 09:24 - */ -class TestProtoWriter +class ProtoWriterTest { private static final String SCHEMA_NAME = "test"; private static final String TABLE_NAME = "ray"; diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/TestRetinaWriter.java b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterIntegrationTest.java similarity index 99% rename from src/test/java/io/pixelsdb/pixels/sink/writer/TestRetinaWriter.java rename to src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterIntegrationTest.java index 0087f08..306eaa6 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/TestRetinaWriter.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterIntegrationTest.java @@ -18,7 +18,7 @@ * . */ -package io.pixelsdb.pixels.sink.writer; +package io.pixelsdb.pixels.sink.writer.retina; import com.google.protobuf.ByteString; import io.pixelsdb.pixels.common.exception.RetinaException; @@ -34,8 +34,8 @@ import io.pixelsdb.pixels.sink.exception.SinkException; import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; import io.pixelsdb.pixels.sink.util.TestDateUtil; -import io.pixelsdb.pixels.sink.writer.retina.RetinaServiceProxy; -import io.pixelsdb.pixels.sink.writer.retina.TransactionProxy; +import io.pixelsdb.pixels.sink.writer.PixelsSinkWriter; +import io.pixelsdb.pixels.sink.writer.PixelsSinkWriterFactory; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; @@ -55,10 +55,10 @@ import java.util.concurrent.Executors; @Tag("integration") -class TestRetinaWriter +class RetinaWriterIntegrationTest { - static Logger logger = LoggerFactory.getLogger(TestRetinaWriter.class.getName()); + static Logger logger = LoggerFactory.getLogger(RetinaWriterIntegrationTest.class.getName()); static RetinaService retinaService; static TableMetadataRegistry metadataRegistry; static TransService transService; diff --git a/src/test/java/io/pixelsdb/pixels/sink/concurrent/RetinaWriterTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterTest.java similarity index 99% rename from src/test/java/io/pixelsdb/pixels/sink/concurrent/RetinaWriterTest.java rename to src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterTest.java index 9325032..193db29 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/concurrent/RetinaWriterTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterTest.java @@ -15,7 +15,7 @@ * */ -package io.pixelsdb.pixels.sink.concurrent; +package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.sink.SinkProto; import io.pixelsdb.pixels.sink.TestConfig; diff --git a/src/test/resources/records/delete.json b/src/test/resources/records/mysql-region-delete.json similarity index 100% rename from src/test/resources/records/delete.json rename to src/test/resources/records/mysql-region-delete.json diff --git a/src/test/resources/records/update.json b/src/test/resources/records/mysql-region-update.json similarity index 100% rename from src/test/resources/records/update.json rename to src/test/resources/records/mysql-region-update.json diff --git a/src/test/resources/records/nation.json b/src/test/resources/records/postgresql-nation-snapshot.json similarity index 100% rename from src/test/resources/records/nation.json rename to src/test/resources/records/postgresql-nation-snapshot.json diff --git a/src/test/resources/records/insert.json b/src/test/resources/records/postgresql-region-insert.json similarity index 100% rename from src/test/resources/records/insert.json rename to src/test/resources/records/postgresql-region-insert.json From 01761dd4626ca449d23896ff1de9c4f7a63d651c Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Fri, 31 Jul 2026 23:56:55 +0800 Subject: [PATCH 11/13] chore: unify Java license headers to Apache-2.0 --- .../pixelsdb/pixels/sink/PixelsSinkApp.java | 23 +++++++--------- .../pixels/sink/PixelsSinkProvider.java | 23 +++++++--------- .../pixels/sink/config/CommandLineConfig.java | 23 +++++++--------- .../pixels/sink/config/ConfigKey.java | 25 +++++++----------- .../pixels/sink/config/ConfigLoader.java | 23 +++++++--------- .../pixels/sink/config/EngineValueFormat.java | 23 +++++++--------- .../pixels/sink/config/KafkaValueFormat.java | 23 +++++++--------- .../pixels/sink/config/PixelsSinkConfig.java | 23 +++++++--------- .../sink/config/PixelsSinkConstants.java | 25 +++++++----------- .../sink/config/PixelsSinkDefaultConfig.java | 23 +++++++--------- .../pixels/sink/config/TransactionConfig.java | 23 +++++++--------- .../sink/config/factory/KafkaPropFactory.java | 25 +++++++----------- .../factory/KafkaPropFactorySelector.java | 23 +++++++--------- .../factory/PixelsSinkConfigFactory.java | 23 +++++++--------- .../factory/RowRecordKafkaPropFactory.java | 23 +++++++--------- .../factory/TransactionKafkaPropFactory.java | 23 +++++++--------- .../debezium/DebeziumRowConverter.java | 25 +++++++----------- .../DebeziumTransactionConverter.java | 25 +++++++----------- .../avro/DebeziumAvroPayloadDecoder.java | 23 +++++++--------- .../avro/DebeziumAvroRowConverter.java | 25 +++++++----------- .../DebeziumAvroTransactionConverter.java | 25 +++++++----------- .../connect/DebeziumConnectRowConverter.java | 23 +++++++--------- .../DebeziumConnectTransactionConverter.java | 23 +++++++--------- .../dialect/DebeziumSourceAdapter.java | 17 +++++++----- .../DebeziumSourceAdapterRegistry.java | 17 +++++++----- .../dialect/DebeziumSourceMetadata.java | 17 +++++++----- .../debezium/dialect/MySqlSourceAdapter.java | 17 +++++++----- .../dialect/PostgresSourceAdapter.java | 17 +++++++----- .../json/DebeziumJsonRowConverter.java | 25 +++++++----------- .../DebeziumJsonTransactionConverter.java | 23 +++++++--------- .../support/DebeziumEnvelopeNormalizer.java | 17 +++++++----- .../debezium/support/DebeziumRecordUtil.java | 15 +++++++---- .../support/DebeziumRowRecordAssembler.java | 25 +++++++----------- .../support/DebeziumRowValueConverter.java | 23 +++++++--------- .../sinkproto/RowRecordConverter.java | 25 +++++++----------- .../TransactionMetadataConverter.java | 25 +++++++----------- .../pixels/sink/event/RowChangeEvent.java | 23 +++++++--------- .../sink/event/RowChangeEventFactory.java | 25 +++++++----------- .../pixels/sink/exception/SinkException.java | 23 +++++++--------- .../sink/freshness/FreshnessClient.java | 23 +++++++--------- .../sink/freshness/FreshnessHistory.java | 23 +++++++--------- .../sink/freshness/OneSecondAverage.java | 23 +++++++--------- .../pixels/sink/metadata/TableMetadata.java | 23 +++++++--------- .../sink/metadata/TableMetadataRegistry.java | 23 +++++++--------- .../pixels/sink/pipeline/TablePipeline.java | 25 +++++++----------- .../sink/pipeline/TablePipelineManager.java | 25 +++++++----------- .../sink/pipeline/TransactionPipeline.java | 25 +++++++----------- .../pixels/sink/processor/TableProcessor.java | 23 +++++++--------- .../sink/processor/TransactionProcessor.java | 23 +++++++--------- .../pixels/sink/provider/ProtoType.java | 25 +++++++----------- .../pixels/sink/source/SinkSource.java | 26 +++++++------------ .../pixels/sink/source/SinkSourceFactory.java | 24 +++++++---------- .../source/engine/ConnectEventClassifier.java | 23 +++++++--------- .../engine/DebeziumEventClassifier.java | 25 +++++++----------- .../source/engine/DebeziumRecordType.java | 25 +++++++----------- .../source/engine/PixelsDebeziumConsumer.java | 23 +++++++--------- .../sink/source/engine/SinkEngineSource.java | 23 +++++++--------- .../DebeziumSourceAdapterSelector.java | 25 +++++++----------- .../sink/source/kafka/KafkaRowSource.java | 25 +++++++----------- .../source/kafka/KafkaTransactionSource.java | 25 +++++++----------- .../sink/source/kafka/SinkKafkaSource.java | 23 +++++++--------- .../sink/source/kafka/TopicProcessor.java | 25 +++++++----------- .../kafka/serde/KafkaRecordConverter.java | 25 +++++++----------- .../kafka/serde/RowRecordDeserializer.java | 25 +++++++----------- .../TransactionMetadataDeserializer.java | 25 +++++++----------- .../storage/AbstractSinkStorageSource.java | 23 +++++++--------- .../storage/MemorySinkStorageSource.java | 25 +++++++----------- .../storage/StreamingSinkStorageSource.java | 23 +++++++--------- .../pixels/sink/util/BlockingBoundedMap.java | 23 +++++++--------- .../sink/util/BlockingBoundedQueue.java | 25 +++++++----------- .../pixels/sink/util/DataTransform.java | 23 +++++++--------- .../pixelsdb/pixels/sink/util/DateUtil.java | 23 +++++++--------- .../pixels/sink/util/EtcdFileRegistry.java | 23 +++++++--------- .../pixels/sink/util/MetricsFacade.java | 23 +++++++--------- .../pixels/sink/util/TableCounters.java | 23 +++++++--------- .../sink/util/concurrent/DecodeExecutors.java | 23 +++++++--------- .../util/concurrent/OrderedBatchDecoder.java | 23 +++++++--------- .../util/concurrent/StreamOrderedDecoder.java | 23 +++++++--------- .../util/rateLimiter/FlushRateLimiter.java | 25 +++++++----------- .../rateLimiter/FlushRateLimiterFactory.java | 23 +++++++--------- .../rateLimiter/GuavaFlushRateLimiter.java | 25 +++++++----------- .../rateLimiter/NoOpFlushRateLimiter.java | 25 +++++++----------- .../SemaphoreFlushRateLimiter.java | 23 +++++++--------- .../sink/writer/AbstractBucketedWriter.java | 23 +++++++--------- .../pixels/sink/writer/NoneWriter.java | 25 +++++++----------- .../pixels/sink/writer/PixelsSinkMode.java | 25 +++++++----------- .../pixels/sink/writer/PixelsSinkWriter.java | 25 +++++++----------- .../sink/writer/PixelsSinkWriterFactory.java | 23 +++++++--------- .../pixels/sink/writer/csv/CsvWriter.java | 23 +++++++--------- .../sink/writer/flink/FlinkPollingWriter.java | 23 +++++++--------- .../flink/PixelsPollingServiceImpl.java | 23 +++++++--------- .../sink/writer/flink/PollingRpcServer.java | 23 +++++++--------- .../pixels/sink/writer/proto/ProtoWriter.java | 23 +++++++--------- .../writer/proto/RotatingWriterManager.java | 23 +++++++--------- .../writer/retina/InFlightControlManager.java | 23 +++++++--------- .../writer/retina/RetinaBucketDispatcher.java | 25 +++++++----------- .../writer/retina/RetinaServiceProxy.java | 23 +++++++--------- .../sink/writer/retina/RetinaWriter.java | 23 +++++++--------- .../sink/writer/retina/SinkContext.java | 23 +++++++--------- .../writer/retina/SinkContextManager.java | 23 +++++++--------- .../writer/retina/TableCrossTxWriter.java | 23 +++++++--------- .../retina/TableSingleRecordWriter.java | 23 +++++++--------- .../writer/retina/TableSingleTxWriter.java | 23 +++++++--------- .../sink/writer/retina/TableWriter.java | 23 +++++++--------- .../sink/writer/retina/TableWriterProxy.java | 23 +++++++--------- .../sink/writer/retina/TransactionMode.java | 25 +++++++----------- .../sink/writer/retina/TransactionProxy.java | 23 +++++++--------- .../pixels/sink/DebeziumEngineTest.java | 3 --- .../pixelsdb/pixels/sink/SplitStringTest.java | 3 +-- .../io/pixelsdb/pixels/sink/TestConfig.java | 3 +-- .../io/pixelsdb/pixels/sink/TestUtils.java | 4 +-- .../sink/config/EngineValueFormatTest.java | 16 +++++++----- .../sink/config/KafkaValueFormatTest.java | 16 +++++++----- .../LegacyKafkaValueFormatMigrationTest.java | 16 +++++++----- .../connect/DebeziumConnectConverterTest.java | 16 +++++++----- .../json/DebeziumJsonConverterTest.java | 4 +-- .../DebeziumEnvelopeNormalizerTest.java | 17 +++++++----- .../DebeziumRowValueConverterTest.java | 4 +-- .../sinkproto/RowRecordConverterTest.java | 25 +++++++----------- .../pixels/sink/event/RowBatchTest.java | 23 +++++++--------- .../sink/event/RowChangeEventFactoryTest.java | 25 +++++++----------- .../pixels/sink/event/RowChangeEventTest.java | 23 +++++++--------- .../sink/freshness/FreshnessClientTest.java | 23 +++++++--------- .../sink/metadata/IndexServiceTest.java | 3 --- .../pixels/sink/pipeline/PipelineTest.java | 25 +++++++----------- .../sink/processor/TableProcessorTest.java | 25 +++++++----------- .../processor/TransactionProcessorTest.java | 25 +++++++----------- .../engine/ConnectEventClassifierTest.java | 16 +++++++----- .../sink/source/kafka/AvroConsumerTest.java | 2 -- .../kafka/serde/KafkaRecordConverterTest.java | 16 +++++++----- .../sink/util/BlockingBoundedQueueTest.java | 25 +++++++----------- .../sink/util/EtcdFileRegistryTest.java | 3 --- .../pixels/sink/util/TestDateUtil.java | 25 +++++++----------- .../concurrent/OrderedBatchDecoderTest.java | 16 +++++++----- .../concurrent/StreamOrderedDecoderTest.java | 16 +++++++----- .../pixels/sink/writer/RpcEndToEndTest.java | 1 - .../pixelsdb/pixels/sink/writer/TpcHTest.java | 23 +++++++--------- .../pixels/sink/writer/TransServiceTest.java | 23 +++++++--------- .../sink/writer/proto/ProtoWriterTest.java | 3 --- .../retina/RetinaWriterIntegrationTest.java | 23 +++++++--------- .../sink/writer/retina/RetinaWriterTest.java | 2 -- .../writer/retina/TableWriterProxyTest.java | 23 +++++++--------- 142 files changed, 1241 insertions(+), 1766 deletions(-) diff --git a/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkApp.java b/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkApp.java index bfa253a..a55afb4 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkApp.java +++ b/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkApp.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink; import io.pixelsdb.pixels.sink.config.CommandLineConfig; diff --git a/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkProvider.java b/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkProvider.java index 37f53e0..2f808db 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkProvider.java +++ b/src/main/java/io/pixelsdb/pixels/sink/PixelsSinkProvider.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink; import io.pixelsdb.pixels.common.sink.SinkProvider; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/CommandLineConfig.java b/src/main/java/io/pixelsdb/pixels/sink/config/CommandLineConfig.java index a8bf062..ab87dd6 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/CommandLineConfig.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/CommandLineConfig.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/ConfigKey.java b/src/main/java/io/pixelsdb/pixels/sink/config/ConfigKey.java index bb4ed66..461ff26 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/ConfigKey.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/ConfigKey.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/ConfigLoader.java b/src/main/java/io/pixelsdb/pixels/sink/config/ConfigLoader.java index 0461535..d5f1a66 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/ConfigLoader.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/ConfigLoader.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/EngineValueFormat.java b/src/main/java/io/pixelsdb/pixels/sink/config/EngineValueFormat.java index 280e636..e1d0303 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/EngineValueFormat.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/EngineValueFormat.java @@ -1,23 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config; import java.util.Locale; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/KafkaValueFormat.java b/src/main/java/io/pixelsdb/pixels/sink/config/KafkaValueFormat.java index 5e9c9bb..0a0721e 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/KafkaValueFormat.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/KafkaValueFormat.java @@ -1,23 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config; import java.util.Locale; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java index 50a684e..c627e3d 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config; import io.pixelsdb.pixels.common.utils.ConfigFactory; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java index 08e5f37..689f1d3 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config; public final class PixelsSinkConstants diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkDefaultConfig.java b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkDefaultConfig.java index da08f3f..585ce0a 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkDefaultConfig.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkDefaultConfig.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config; public class PixelsSinkDefaultConfig diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/TransactionConfig.java b/src/main/java/io/pixelsdb/pixels/sink/config/TransactionConfig.java index 1200eb0..53585fe 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/TransactionConfig.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/TransactionConfig.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config; public class TransactionConfig diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/factory/KafkaPropFactory.java b/src/main/java/io/pixelsdb/pixels/sink/config/factory/KafkaPropFactory.java index cbace9f..e1eceb6 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/factory/KafkaPropFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/factory/KafkaPropFactory.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config.factory; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/factory/KafkaPropFactorySelector.java b/src/main/java/io/pixelsdb/pixels/sink/config/factory/KafkaPropFactorySelector.java index c0343f9..9d2431d 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/factory/KafkaPropFactorySelector.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/factory/KafkaPropFactorySelector.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config.factory; import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/factory/PixelsSinkConfigFactory.java b/src/main/java/io/pixelsdb/pixels/sink/config/factory/PixelsSinkConfigFactory.java index 56b3550..a25dfc3 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/factory/PixelsSinkConfigFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/factory/PixelsSinkConfigFactory.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config.factory; import io.pixelsdb.pixels.common.utils.ConfigFactory; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/factory/RowRecordKafkaPropFactory.java b/src/main/java/io/pixelsdb/pixels/sink/config/factory/RowRecordKafkaPropFactory.java index d80f8a3..19ebd89 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/factory/RowRecordKafkaPropFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/factory/RowRecordKafkaPropFactory.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config.factory; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/factory/TransactionKafkaPropFactory.java b/src/main/java/io/pixelsdb/pixels/sink/config/factory/TransactionKafkaPropFactory.java index d0049d3..c65f16e 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/factory/TransactionKafkaPropFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/factory/TransactionKafkaPropFactory.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config.factory; import io.apicurio.registry.serde.SerdeConfig; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowConverter.java index 90db45a..138e737 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumRowConverter.java @@ -1,23 +1,18 @@ /* - * Copyright 2026 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.conversion.debezium; import io.pixelsdb.pixels.sink.event.RowChangeEvent; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumTransactionConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumTransactionConverter.java index c412b79..4b8893e 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumTransactionConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/DebeziumTransactionConverter.java @@ -1,23 +1,18 @@ /* - * Copyright 2026 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.conversion.debezium; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroPayloadDecoder.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroPayloadDecoder.java index fa76395..4b6924c 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroPayloadDecoder.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroPayloadDecoder.java @@ -1,23 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.conversion.debezium.avro; import io.apicurio.registry.serde.SerdeConfig; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroRowConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroRowConverter.java index d519fc8..34aaf4e 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroRowConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroRowConverter.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.conversion.debezium.avro; import io.pixelsdb.pixels.core.TypeDescription; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroTransactionConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroTransactionConverter.java index 1d59ac9..462e994 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroTransactionConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/avro/DebeziumAvroTransactionConverter.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.conversion.debezium.avro; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectRowConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectRowConverter.java index 52cf60c..ebc61e6 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectRowConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectRowConverter.java @@ -1,23 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.conversion.debezium.connect; import io.pixelsdb.pixels.core.TypeDescription; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectTransactionConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectTransactionConverter.java index e8f5b56..461c2b0 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectTransactionConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectTransactionConverter.java @@ -1,23 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.conversion.debezium.connect; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceAdapter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceAdapter.java index 37bb481..d64eca8 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceAdapter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceAdapter.java @@ -1,12 +1,17 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.pixelsdb.pixels.sink.conversion.debezium.dialect; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceAdapterRegistry.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceAdapterRegistry.java index 6510feb..b002962 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceAdapterRegistry.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceAdapterRegistry.java @@ -1,12 +1,17 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.pixelsdb.pixels.sink.conversion.debezium.dialect; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceMetadata.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceMetadata.java index b2efddd..eda02f0 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceMetadata.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/DebeziumSourceMetadata.java @@ -1,12 +1,17 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.pixelsdb.pixels.sink.conversion.debezium.dialect; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/MySqlSourceAdapter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/MySqlSourceAdapter.java index 4212d99..f35615e 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/MySqlSourceAdapter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/MySqlSourceAdapter.java @@ -1,12 +1,17 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.pixelsdb.pixels.sink.conversion.debezium.dialect; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/PostgresSourceAdapter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/PostgresSourceAdapter.java index 5852d1b..77d6a0f 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/PostgresSourceAdapter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/dialect/PostgresSourceAdapter.java @@ -1,12 +1,17 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.pixelsdb.pixels.sink.conversion.debezium.dialect; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonRowConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonRowConverter.java index 83d63a5..d482131 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonRowConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonRowConverter.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.conversion.debezium.json; import com.fasterxml.jackson.databind.JsonNode; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonTransactionConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonTransactionConverter.java index 4ddfec5..0b5da07 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonTransactionConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonTransactionConverter.java @@ -1,23 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.conversion.debezium.json; import com.fasterxml.jackson.databind.JsonNode; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumEnvelopeNormalizer.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumEnvelopeNormalizer.java index 41d3a8d..960b4a5 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumEnvelopeNormalizer.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumEnvelopeNormalizer.java @@ -1,12 +1,17 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.pixelsdb.pixels.sink.conversion.debezium.support; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRecordUtil.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRecordUtil.java index d38a65f..75eaac2 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRecordUtil.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRecordUtil.java @@ -1,12 +1,17 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.pixelsdb.pixels.sink.conversion.debezium.support; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowRecordAssembler.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowRecordAssembler.java index d2c6d58..8aefa8e 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowRecordAssembler.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowRecordAssembler.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.conversion.debezium.support; import io.pixelsdb.pixels.core.TypeDescription; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowValueConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowValueConverter.java index e510c70..6297e57 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowValueConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowValueConverter.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.conversion.debezium.support; import com.fasterxml.jackson.databind.JsonNode; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverter.java index 79715be..319c657 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverter.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.conversion.sinkproto; import com.google.protobuf.InvalidProtocolBufferException; diff --git a/src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/TransactionMetadataConverter.java b/src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/TransactionMetadataConverter.java index c3d5341..b40a070 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/TransactionMetadataConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/conversion/sinkproto/TransactionMetadataConverter.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.conversion.sinkproto; import com.google.protobuf.InvalidProtocolBufferException; diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEvent.java b/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEvent.java index 8800ad9..19336e7 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEvent.java +++ b/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEvent.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.event; import com.google.protobuf.ByteString; diff --git a/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactory.java b/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactory.java index f69dec0..07eabdc 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactory.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.event; import io.pixelsdb.pixels.core.TypeDescription; diff --git a/src/main/java/io/pixelsdb/pixels/sink/exception/SinkException.java b/src/main/java/io/pixelsdb/pixels/sink/exception/SinkException.java index 5ad1c7a..ce208be 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/exception/SinkException.java +++ b/src/main/java/io/pixelsdb/pixels/sink/exception/SinkException.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.exception; public class SinkException extends Exception diff --git a/src/main/java/io/pixelsdb/pixels/sink/freshness/FreshnessClient.java b/src/main/java/io/pixelsdb/pixels/sink/freshness/FreshnessClient.java index 73ba79a..b94ad43 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/freshness/FreshnessClient.java +++ b/src/main/java/io/pixelsdb/pixels/sink/freshness/FreshnessClient.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.freshness; import io.pixelsdb.pixels.common.exception.TransException; diff --git a/src/main/java/io/pixelsdb/pixels/sink/freshness/FreshnessHistory.java b/src/main/java/io/pixelsdb/pixels/sink/freshness/FreshnessHistory.java index 1037f2e..2843b2c 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/freshness/FreshnessHistory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/freshness/FreshnessHistory.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.freshness; diff --git a/src/main/java/io/pixelsdb/pixels/sink/freshness/OneSecondAverage.java b/src/main/java/io/pixelsdb/pixels/sink/freshness/OneSecondAverage.java index cd4f3c2..b017536 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/freshness/OneSecondAverage.java +++ b/src/main/java/io/pixelsdb/pixels/sink/freshness/OneSecondAverage.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.freshness; import java.util.ArrayDeque; diff --git a/src/main/java/io/pixelsdb/pixels/sink/metadata/TableMetadata.java b/src/main/java/io/pixelsdb/pixels/sink/metadata/TableMetadata.java index 0935d7a..63e1d16 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/metadata/TableMetadata.java +++ b/src/main/java/io/pixelsdb/pixels/sink/metadata/TableMetadata.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.metadata; import io.pixelsdb.pixels.common.exception.MetadataException; diff --git a/src/main/java/io/pixelsdb/pixels/sink/metadata/TableMetadataRegistry.java b/src/main/java/io/pixelsdb/pixels/sink/metadata/TableMetadataRegistry.java index c776e71..9b6121e 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/metadata/TableMetadataRegistry.java +++ b/src/main/java/io/pixelsdb/pixels/sink/metadata/TableMetadataRegistry.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.metadata; import io.pixelsdb.pixels.common.exception.MetadataException; diff --git a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipeline.java b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipeline.java index 5fbff2c..4580679 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipeline.java +++ b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipeline.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.pipeline; import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; diff --git a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipelineManager.java b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipelineManager.java index 2044536..bec3ace 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipelineManager.java +++ b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TablePipelineManager.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.pipeline; import io.pixelsdb.pixels.common.metadata.SchemaTableName; diff --git a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TransactionPipeline.java b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TransactionPipeline.java index 4d146c5..696c002 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/pipeline/TransactionPipeline.java +++ b/src/main/java/io/pixelsdb/pixels/sink/pipeline/TransactionPipeline.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.pipeline; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/processor/TableProcessor.java b/src/main/java/io/pixelsdb/pixels/sink/processor/TableProcessor.java index 4e2583e..98698fc 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/processor/TableProcessor.java +++ b/src/main/java/io/pixelsdb/pixels/sink/processor/TableProcessor.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.processor; diff --git a/src/main/java/io/pixelsdb/pixels/sink/processor/TransactionProcessor.java b/src/main/java/io/pixelsdb/pixels/sink/processor/TransactionProcessor.java index 175bf5b..3a298ff 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/processor/TransactionProcessor.java +++ b/src/main/java/io/pixelsdb/pixels/sink/processor/TransactionProcessor.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.processor; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/provider/ProtoType.java b/src/main/java/io/pixelsdb/pixels/sink/provider/ProtoType.java index ea7a4bf..5eaa4c1 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/provider/ProtoType.java +++ b/src/main/java/io/pixelsdb/pixels/sink/provider/ProtoType.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.provider; diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/SinkSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/SinkSource.java index 6854b7b..50b1014 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/SinkSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/SinkSource.java @@ -1,24 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - - package io.pixelsdb.pixels.sink.source; /** diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/SinkSourceFactory.java b/src/main/java/io/pixelsdb/pixels/sink/source/SinkSourceFactory.java index 5b61ad7..2f7ca8a 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/SinkSourceFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/SinkSourceFactory.java @@ -1,24 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - - package io.pixelsdb.pixels.sink.source; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifier.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifier.java index 4f55356..ad53b97 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifier.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifier.java @@ -1,23 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.engine; import io.pixelsdb.pixels.common.metadata.SchemaTableName; diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumEventClassifier.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumEventClassifier.java index 73f1071..dab32a0 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumEventClassifier.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumEventClassifier.java @@ -1,23 +1,18 @@ /* - * Copyright 2026 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.engine; import io.pixelsdb.pixels.common.metadata.SchemaTableName; diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumRecordType.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumRecordType.java index 98d8cb5..d90cb8f 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumRecordType.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/DebeziumRecordType.java @@ -1,23 +1,18 @@ /* - * Copyright 2026 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.engine; /** diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumer.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumer.java index 0e578f8..2c74e33 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumer.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/PixelsDebeziumConsumer.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.engine; diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/SinkEngineSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/SinkEngineSource.java index fa2a6e2..08f93f1 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/engine/SinkEngineSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/SinkEngineSource.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.engine; import io.debezium.embedded.Connect; diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumSourceAdapterSelector.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumSourceAdapterSelector.java index bf5e256..645f85b 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumSourceAdapterSelector.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumSourceAdapterSelector.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.engine.adapter; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaRowSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaRowSource.java index 43134e2..49fc886 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaRowSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaRowSource.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.kafka; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaTransactionSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaTransactionSource.java index 5d912a5..5624d12 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaTransactionSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/KafkaTransactionSource.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.kafka; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/SinkKafkaSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/SinkKafkaSource.java index 8721b66..3e661b5 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/SinkKafkaSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/SinkKafkaSource.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.kafka; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/TopicProcessor.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/TopicProcessor.java index 67dc768..3929fee 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/TopicProcessor.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/TopicProcessor.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.kafka; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverter.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverter.java index 1ef2f92..81f4b57 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverter.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.kafka.serde; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowRecordDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowRecordDeserializer.java index 6dcb2ef..28902ba 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowRecordDeserializer.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/RowRecordDeserializer.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.kafka.serde; import com.google.protobuf.InvalidProtocolBufferException; diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataDeserializer.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataDeserializer.java index f0237ca..9b4ac1b 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataDeserializer.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/TransactionMetadataDeserializer.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.kafka.serde; import com.google.protobuf.InvalidProtocolBufferException; diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java index 48669e4..1e39f60 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/storage/AbstractSinkStorageSource.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.storage; import io.pixelsdb.pixels.common.physical.PhysicalReader; diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/MemorySinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/MemorySinkStorageSource.java index bc34b5c..b8e1b79 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/storage/MemorySinkStorageSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/storage/MemorySinkStorageSource.java @@ -1,23 +1,18 @@ /* - * Copyright 2026 PixelsDB. + * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.storage; import io.pixelsdb.pixels.common.physical.PhysicalReader; diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/storage/StreamingSinkStorageSource.java b/src/main/java/io/pixelsdb/pixels/sink/source/storage/StreamingSinkStorageSource.java index 81bb151..a9100c6 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/storage/StreamingSinkStorageSource.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/storage/StreamingSinkStorageSource.java @@ -1,23 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.storage; import io.pixelsdb.pixels.common.physical.PhysicalReader; diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedMap.java b/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedMap.java index 4f649aa..7c3825c 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedMap.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedMap.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util; import java.util.Set; diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueue.java b/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueue.java index 5aad626..30de2cb 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueue.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueue.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util; import java.io.Closeable; diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/DataTransform.java b/src/main/java/io/pixelsdb/pixels/sink/util/DataTransform.java index d0c1f24..c5c9536 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/DataTransform.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/DataTransform.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util; import com.google.protobuf.ByteString; diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/DateUtil.java b/src/main/java/io/pixelsdb/pixels/sink/util/DateUtil.java index e630270..5aedd51 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/DateUtil.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/DateUtil.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util; diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistry.java b/src/main/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistry.java index b06b8b8..b3ac9ac 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistry.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistry.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util; diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/MetricsFacade.java b/src/main/java/io/pixelsdb/pixels/sink/util/MetricsFacade.java index 734319b..860a6db 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/MetricsFacade.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/MetricsFacade.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util; import com.google.protobuf.ByteString; diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/TableCounters.java b/src/main/java/io/pixelsdb/pixels/sink/util/TableCounters.java index b6c0850..bfd597b 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/TableCounters.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/TableCounters.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util; /** diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/DecodeExecutors.java b/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/DecodeExecutors.java index f8aaaf9..43edf91 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/DecodeExecutors.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/DecodeExecutors.java @@ -1,23 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util.concurrent; import io.pixelsdb.pixels.sink.config.PixelsSinkConstants; diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoder.java b/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoder.java index efaf28e..38555c1 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoder.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoder.java @@ -1,23 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util.concurrent; import org.slf4j.Logger; diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoder.java b/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoder.java index 601b930..56a04d8 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoder.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoder.java @@ -1,23 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util.concurrent; import org.slf4j.Logger; diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/FlushRateLimiter.java b/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/FlushRateLimiter.java index d62c25a..bcef26a 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/FlushRateLimiter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/FlushRateLimiter.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util.rateLimiter; public interface FlushRateLimiter diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/FlushRateLimiterFactory.java b/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/FlushRateLimiterFactory.java index 1956d31..54c6917 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/FlushRateLimiterFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/FlushRateLimiterFactory.java @@ -1,23 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util.rateLimiter; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/GuavaFlushRateLimiter.java b/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/GuavaFlushRateLimiter.java index 9c4ebd0..83a1f82 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/GuavaFlushRateLimiter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/GuavaFlushRateLimiter.java @@ -1,23 +1,18 @@ /* - * Copyright 2026 PixelsDB. + * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util.rateLimiter; import com.google.common.util.concurrent.RateLimiter; diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/NoOpFlushRateLimiter.java b/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/NoOpFlushRateLimiter.java index 05745c1..374d28a 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/NoOpFlushRateLimiter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/NoOpFlushRateLimiter.java @@ -1,23 +1,18 @@ /* - * Copyright 2026 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util.rateLimiter; public class NoOpFlushRateLimiter implements FlushRateLimiter diff --git a/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/SemaphoreFlushRateLimiter.java b/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/SemaphoreFlushRateLimiter.java index 52c2c5a..1a37653 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/SemaphoreFlushRateLimiter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/util/rateLimiter/SemaphoreFlushRateLimiter.java @@ -1,23 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util.rateLimiter; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/AbstractBucketedWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/AbstractBucketedWriter.java index 52bd4b3..3723fe4 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/AbstractBucketedWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/AbstractBucketedWriter.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/NoneWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/NoneWriter.java index f40bae0..aa9d5ec 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/NoneWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/NoneWriter.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkMode.java b/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkMode.java index c5c6e15..5f53498 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkMode.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkMode.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkWriter.java index 8e2744a..11cb3f5 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkWriter.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkWriterFactory.java b/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkWriterFactory.java index 5697ecb..cc901d2 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkWriterFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/PixelsSinkWriterFactory.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/csv/CsvWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/csv/CsvWriter.java index 540d0f7..0165a52 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/csv/CsvWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/csv/CsvWriter.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.csv; import com.google.common.util.concurrent.ThreadFactoryBuilder; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/flink/FlinkPollingWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/flink/FlinkPollingWriter.java index 33881bc..72c5985 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/flink/FlinkPollingWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/flink/FlinkPollingWriter.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.flink; import io.pixelsdb.pixels.common.metadata.SchemaTableName; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/flink/PixelsPollingServiceImpl.java b/src/main/java/io/pixelsdb/pixels/sink/writer/flink/PixelsPollingServiceImpl.java index 9c33848..a6db288 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/flink/PixelsPollingServiceImpl.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/flink/PixelsPollingServiceImpl.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.flink; import io.grpc.stub.StreamObserver; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/flink/PollingRpcServer.java b/src/main/java/io/pixelsdb/pixels/sink/writer/flink/PollingRpcServer.java index 333f21b..5233667 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/flink/PollingRpcServer.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/flink/PollingRpcServer.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.flink; import io.grpc.Server; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/proto/ProtoWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/proto/ProtoWriter.java index 23f5dd0..f5273f2 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/proto/ProtoWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/proto/ProtoWriter.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.proto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/proto/RotatingWriterManager.java b/src/main/java/io/pixelsdb/pixels/sink/writer/proto/RotatingWriterManager.java index 5d9ff13..b45d305 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/proto/RotatingWriterManager.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/proto/RotatingWriterManager.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.proto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/InFlightControlManager.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/InFlightControlManager.java index 7298506..9d0fb9b 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/InFlightControlManager.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/InFlightControlManager.java @@ -1,23 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaBucketDispatcher.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaBucketDispatcher.java index 9dbb492..4055fce 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaBucketDispatcher.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaBucketDispatcher.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.sink.event.RowChangeEvent; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaServiceProxy.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaServiceProxy.java index 7ded27f..6d33416 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaServiceProxy.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaServiceProxy.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.common.exception.RetinaException; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriter.java index 32cbee3..8e20f91 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriter.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/SinkContext.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/SinkContext.java index dbea99f..d66b7ac 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/SinkContext.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/SinkContext.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.common.transaction.TransContext; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/SinkContextManager.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/SinkContextManager.java index 491afb2..01ba91b 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/SinkContextManager.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/SinkContextManager.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.common.transaction.TransContext; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableCrossTxWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableCrossTxWriter.java index 040966c..cd29f0f 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableCrossTxWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableCrossTxWriter.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableSingleRecordWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableSingleRecordWriter.java index 9f02226..737f4c8 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableSingleRecordWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableSingleRecordWriter.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.common.transaction.TransContext; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableSingleTxWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableSingleTxWriter.java index 6ccea6e..1ff9846 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableSingleTxWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableSingleTxWriter.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.retina.RetinaProto; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableWriter.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableWriter.java index 3472121..9508a30 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableWriter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableWriter.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableWriterProxy.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableWriterProxy.java index 071011c..bd04800 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableWriterProxy.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TableWriterProxy.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.common.node.BucketCache; diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TransactionMode.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TransactionMode.java index 1ff0347..1d7de8d 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TransactionMode.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TransactionMode.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; public enum TransactionMode diff --git a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TransactionProxy.java b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TransactionProxy.java index 8514dbd..5c8713d 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TransactionProxy.java +++ b/src/main/java/io/pixelsdb/pixels/sink/writer/retina/TransactionProxy.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.common.exception.TransException; diff --git a/src/test/java/io/pixelsdb/pixels/sink/DebeziumEngineTest.java b/src/test/java/io/pixelsdb/pixels/sink/DebeziumEngineTest.java index d4f9c80..32ae53e 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/DebeziumEngineTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/DebeziumEngineTest.java @@ -12,10 +12,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ - - package io.pixelsdb.pixels.sink; diff --git a/src/test/java/io/pixelsdb/pixels/sink/SplitStringTest.java b/src/test/java/io/pixelsdb/pixels/sink/SplitStringTest.java index f73a1da..51f31c5 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/SplitStringTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/SplitStringTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package io.pixelsdb.pixels.sink; diff --git a/src/test/java/io/pixelsdb/pixels/sink/TestConfig.java b/src/test/java/io/pixelsdb/pixels/sink/TestConfig.java index ab32852..ef1be78 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/TestConfig.java +++ b/src/test/java/io/pixelsdb/pixels/sink/TestConfig.java @@ -1,5 +1,5 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package io.pixelsdb.pixels.sink; import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; diff --git a/src/test/java/io/pixelsdb/pixels/sink/TestUtils.java b/src/test/java/io/pixelsdb/pixels/sink/TestUtils.java index 7713401..45d8c44 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/TestUtils.java +++ b/src/test/java/io/pixelsdb/pixels/sink/TestUtils.java @@ -1,5 +1,5 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2023 PixelsDB. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,9 +12,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ - package io.pixelsdb.pixels.sink; import java.util.concurrent.ExecutorService; diff --git a/src/test/java/io/pixelsdb/pixels/sink/config/EngineValueFormatTest.java b/src/test/java/io/pixelsdb/pixels/sink/config/EngineValueFormatTest.java index b61347a..1b08a53 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/config/EngineValueFormatTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/config/EngineValueFormatTest.java @@ -1,14 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config; import org.junit.jupiter.api.Test; diff --git a/src/test/java/io/pixelsdb/pixels/sink/config/KafkaValueFormatTest.java b/src/test/java/io/pixelsdb/pixels/sink/config/KafkaValueFormatTest.java index 2801e4d..4c7294b 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/config/KafkaValueFormatTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/config/KafkaValueFormatTest.java @@ -1,14 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config; import org.junit.jupiter.api.Test; diff --git a/src/test/java/io/pixelsdb/pixels/sink/config/LegacyKafkaValueFormatMigrationTest.java b/src/test/java/io/pixelsdb/pixels/sink/config/LegacyKafkaValueFormatMigrationTest.java index 600861d..152c806 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/config/LegacyKafkaValueFormatMigrationTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/config/LegacyKafkaValueFormatMigrationTest.java @@ -1,14 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.config; import org.junit.jupiter.api.Test; diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectConverterTest.java index bf024cc..8be88ba 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectConverterTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/connect/DebeziumConnectConverterTest.java @@ -1,14 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.conversion.debezium.connect; import io.pixelsdb.pixels.common.metadata.SchemaTableName; diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonConverterTest.java index 14371d2..e1c50d5 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonConverterTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/json/DebeziumJsonConverterTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,9 +12,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ - package io.pixelsdb.pixels.sink.conversion.debezium.json; import io.pixelsdb.pixels.common.metadata.SchemaTableName; diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumEnvelopeNormalizerTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumEnvelopeNormalizerTest.java index 9843278..9c6844b 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumEnvelopeNormalizerTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumEnvelopeNormalizerTest.java @@ -1,12 +1,17 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package io.pixelsdb.pixels.sink.conversion.debezium.support; diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowValueConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowValueConverterTest.java index a193a49..53b6e79 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowValueConverterTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/debezium/support/DebeziumRowValueConverterTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,9 +12,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ - package io.pixelsdb.pixels.sink.conversion.debezium.support; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/src/test/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverterTest.java index 2dc280e..60cade7 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverterTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/conversion/sinkproto/RowRecordConverterTest.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.conversion.sinkproto; import io.pixelsdb.pixels.common.metadata.SchemaTableName; diff --git a/src/test/java/io/pixelsdb/pixels/sink/event/RowBatchTest.java b/src/test/java/io/pixelsdb/pixels/sink/event/RowBatchTest.java index 932b1f3..d86cc55 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/event/RowBatchTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/event/RowBatchTest.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.event; import io.pixelsdb.pixels.core.TypeDescription; diff --git a/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactoryTest.java b/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactoryTest.java index 8cfbfac..bcc9d04 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactoryTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventFactoryTest.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.event; import com.google.protobuf.ByteString; diff --git a/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventTest.java b/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventTest.java index afe807c..d26cba2 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/event/RowChangeEventTest.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.event; import com.google.protobuf.ByteString; diff --git a/src/test/java/io/pixelsdb/pixels/sink/freshness/FreshnessClientTest.java b/src/test/java/io/pixelsdb/pixels/sink/freshness/FreshnessClientTest.java index 3c537c2..3d35472 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/freshness/FreshnessClientTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/freshness/FreshnessClientTest.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.freshness; import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; diff --git a/src/test/java/io/pixelsdb/pixels/sink/metadata/IndexServiceTest.java b/src/test/java/io/pixelsdb/pixels/sink/metadata/IndexServiceTest.java index 01e4b6d..70a9067 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/metadata/IndexServiceTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/metadata/IndexServiceTest.java @@ -12,10 +12,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ - - package io.pixelsdb.pixels.sink.metadata; import com.google.protobuf.ByteString; diff --git a/src/test/java/io/pixelsdb/pixels/sink/pipeline/PipelineTest.java b/src/test/java/io/pixelsdb/pixels/sink/pipeline/PipelineTest.java index d31de7a..50ee917 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/pipeline/PipelineTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/pipeline/PipelineTest.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.pipeline; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/test/java/io/pixelsdb/pixels/sink/processor/TableProcessorTest.java b/src/test/java/io/pixelsdb/pixels/sink/processor/TableProcessorTest.java index 2910840..9bcba2c 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/processor/TableProcessorTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/processor/TableProcessorTest.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.processor; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/test/java/io/pixelsdb/pixels/sink/processor/TransactionProcessorTest.java b/src/test/java/io/pixelsdb/pixels/sink/processor/TransactionProcessorTest.java index 0596d37..15c8767 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/processor/TransactionProcessorTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/processor/TransactionProcessorTest.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.processor; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/test/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifierTest.java b/src/test/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifierTest.java index 397e8bf..ac63342 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifierTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/source/engine/ConnectEventClassifierTest.java @@ -1,14 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.engine; import io.pixelsdb.pixels.common.metadata.SchemaTableName; diff --git a/src/test/java/io/pixelsdb/pixels/sink/source/kafka/AvroConsumerTest.java b/src/test/java/io/pixelsdb/pixels/sink/source/kafka/AvroConsumerTest.java index ccb0ca3..fe5b7cf 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/source/kafka/AvroConsumerTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/source/kafka/AvroConsumerTest.java @@ -12,9 +12,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ - package io.pixelsdb.pixels.sink.source.kafka; import io.apicurio.registry.serde.SerdeConfig; diff --git a/src/test/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverterTest.java index b0cfae5..dc95138 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverterTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverterTest.java @@ -1,14 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.source.kafka.serde; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/test/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueueTest.java b/src/test/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueueTest.java index 116a033..d8cb81e 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueueTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/util/BlockingBoundedQueueTest.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util; import org.junit.jupiter.api.Test; diff --git a/src/test/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistryTest.java b/src/test/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistryTest.java index 1d65203..e6eb2a1 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistryTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/util/EtcdFileRegistryTest.java @@ -12,10 +12,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ - - package io.pixelsdb.pixels.sink.util; diff --git a/src/test/java/io/pixelsdb/pixels/sink/util/TestDateUtil.java b/src/test/java/io/pixelsdb/pixels/sink/util/TestDateUtil.java index aea207d..538374e 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/util/TestDateUtil.java +++ b/src/test/java/io/pixelsdb/pixels/sink/util/TestDateUtil.java @@ -1,23 +1,18 @@ /* - * Copyright 2025 PixelsDB. + * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the license, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util; import io.pixelsdb.pixels.core.utils.DatetimeUtils; diff --git a/src/test/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoderTest.java b/src/test/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoderTest.java index 7b991b2..724b2af 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoderTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/util/concurrent/OrderedBatchDecoderTest.java @@ -1,14 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util.concurrent; import org.junit.jupiter.api.AfterEach; diff --git a/src/test/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoderTest.java b/src/test/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoderTest.java index 60c1f0c..d53e763 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoderTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/util/concurrent/StreamOrderedDecoderTest.java @@ -1,14 +1,18 @@ /* * Copyright 2026 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.util.concurrent; import org.junit.jupiter.api.Test; diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/RpcEndToEndTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/RpcEndToEndTest.java index 8bfaf95..47231e3 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/RpcEndToEndTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/RpcEndToEndTest.java @@ -12,7 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package io.pixelsdb.pixels.sink.writer; diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/TpcHTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/TpcHTest.java index 6b1b220..23dab0b 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/TpcHTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/TpcHTest.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer; import io.pixelsdb.pixels.common.exception.RetinaException; diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/TransServiceTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/TransServiceTest.java index e907292..9fc30ac 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/TransServiceTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/TransServiceTest.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer; diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/proto/ProtoWriterTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/proto/ProtoWriterTest.java index 34ae0f7..9609268 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/proto/ProtoWriterTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/proto/ProtoWriterTest.java @@ -12,10 +12,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ - - package io.pixelsdb.pixels.sink.writer.proto; diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterIntegrationTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterIntegrationTest.java index 306eaa6..53171b3 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterIntegrationTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterIntegrationTest.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import com.google.protobuf.ByteString; diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterTest.java index 193db29..7e80512 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterTest.java @@ -12,9 +12,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.sink.SinkProto; diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/retina/TableWriterProxyTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/TableWriterProxyTest.java index bdb78a0..53f0d40 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/retina/TableWriterProxyTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/TableWriterProxyTest.java @@ -1,23 +1,18 @@ /* * Copyright 2025 PixelsDB. * - * This file is part of Pixels. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Pixels is free software: you can redistribute it and/or modify - * it under the terms of the Affero GNU General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. + * http://www.apache.org/licenses/LICENSE-2.0 * - * Pixels is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Affero GNU General Public License for more details. - * - * You should have received a copy of the Affero GNU General Public - * License along with Pixels. If not, see - * . + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package io.pixelsdb.pixels.sink.writer.retina; import io.pixelsdb.pixels.sink.TestConfig; From 6a3640f8e7ca12dff368a33b7a71dcf831bcc57a Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Sat, 1 Aug 2026 00:21:22 +0800 Subject: [PATCH 12/13] fix: require sink.debezium.dialect for Kafka transaction decoding --- conf/pixels-sink.aws.properties | 2 ++ conf/pixels-sink.ch.properties | 2 ++ conf/pixels-sink.mysql.properties | 2 ++ conf/pixels-sink.pg.properties | 2 ++ docs/configuration.md | 4 ++- .../pixels/sink/config/PixelsSinkConfig.java | 25 +++++++++++++++++++ .../sink/config/PixelsSinkConstants.java | 2 ++ .../factory/RowRecordKafkaPropFactory.java | 8 +++--- .../factory/TransactionKafkaPropFactory.java | 8 +++--- .../DebeziumSourceAdapterSelector.java | 13 +++++----- .../kafka/serde/KafkaRecordConverter.java | 22 +++++++++++++--- src/main/resources/pixels-sink.aws.properties | 2 ++ .../resources/pixels-sink.local.properties | 2 ++ .../kafka/serde/KafkaRecordConverterTest.java | 14 +++++++++++ .../resources/pixels-sink-test.properties | 1 + 15 files changed, 88 insertions(+), 21 deletions(-) diff --git a/conf/pixels-sink.aws.properties b/conf/pixels-sink.aws.properties index 29bd289..a6ef8b3 100644 --- a/conf/pixels-sink.aws.properties +++ b/conf/pixels-sink.aws.properties @@ -1,6 +1,8 @@ # engine | kafka | storage sink.datasource=storage sink.storage.mode=stream +# CDC dialect for envelope normalization: mysql | postgresql +sink.debezium.dialect=postgresql # -1 means no limit, Only implement in retina sink mode yet: sink.datasource.rate.limit=200000 sink.monitor.report.file=/home/ubuntu/pixels-sink/result1k2_feb/rate_8192tile_3.csv diff --git a/conf/pixels-sink.ch.properties b/conf/pixels-sink.ch.properties index d93fcd4..7312a01 100644 --- a/conf/pixels-sink.ch.properties +++ b/conf/pixels-sink.ch.properties @@ -1,6 +1,8 @@ # engine | kafka | storage sink.datasource=storage sink.storage.mode=stream +# CDC dialect for envelope normalization: mysql | postgresql +sink.debezium.dialect=postgresql sink.mode=retina #sink.datasource=engine #sink.mode=proto diff --git a/conf/pixels-sink.mysql.properties b/conf/pixels-sink.mysql.properties index 3887095..58e5e1e 100644 --- a/conf/pixels-sink.mysql.properties +++ b/conf/pixels-sink.mysql.properties @@ -1,6 +1,8 @@ # engine | kafka | storage sink.datasource=engine sink.datasource.engine.format=connect +# CDC dialect for envelope normalization: mysql | postgresql +sink.debezium.dialect=mysql sink.storage.mode=stream # -1 means no limit, Only implement in retina sink mode yet sink.datasource.rate.limit=100000 diff --git a/conf/pixels-sink.pg.properties b/conf/pixels-sink.pg.properties index 58f599f..741aa2c 100644 --- a/conf/pixels-sink.pg.properties +++ b/conf/pixels-sink.pg.properties @@ -1,6 +1,8 @@ # engine | kafka | storage sink.datasource=engine sink.datasource.engine.format=connect +# CDC dialect for envelope normalization: mysql | postgresql +sink.debezium.dialect=postgresql sink.storage.mode=stream # -1 means no limit, Only implement in retina sink mode yet sink.datasource.rate.limit=100000 diff --git a/docs/configuration.md b/docs/configuration.md index a37562b..79cd195 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -55,8 +55,9 @@ Notes on `sink.trans.mode`: | Key | Default | Notes | | --- | --- | --- | | `sink.datasource.engine.format` | `connect` | Only `connect` is runnable today. Non-`connect` values fail fast. | +| `sink.debezium.dialect` | none | Sink-side CDC dialect for envelope normalization: `mysql` or `postgresql`. Used by Engine and Kafka conversion. If unset, falls back to inferring from `debezium.connector.class`. | | `debezium.name` | none | Engine name. | -| `debezium.connector.class` | none | Connector class, e.g. PostgreSQL or MySQL connector. | +| `debezium.connector.class` | none | Debezium Engine connector class only (which DB the engine connects to). Not a Kafka setting. | | `debezium.*` | none | Standard Debezium engine properties. | See `conf/pixels-sink.mysql.properties` for a TDSQL MySQL CDC example. @@ -119,6 +120,7 @@ Kafka source is deprecated. | `auto.offset.reset` | none | Standard Kafka consumer property. | | `key.deserializer` | `org.apache.kafka.common.serialization.StringDeserializer` | Kafka key deserializer. | | `sink.kafka.value.format` | `json` | Envelope format: `json` or `avro`. Kafka sources assemble `conversion.debezium` converters from this key. | +| `sink.debezium.dialect` | none | Upstream CDC dialect (`mysql` / `postgresql`). Required for Kafka transaction decoding; row events can also infer from `source.connector` when unset. | | `topic.prefix` | required | Topic prefix for table events. | | `consumer.capture_database` | required | Database name used to build topic names. | | `consumer.include_tables` | empty | Comma-separated table list, empty means all. | diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java index c627e3d..02108b7 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConfig.java @@ -137,6 +137,13 @@ public class PixelsSinkConfig @ConfigKey("debezium.topic.prefix") private String debeziumTopicPrefix; + /** + * Sink-side CDC dialect for envelope normalization: {@code mysql} or {@code postgresql}. + * Independent of {@code debezium.connector.class}, which is only for Debezium Engine. + */ + @ConfigKey(value = "sink.debezium.dialect", defaultValue = "") + private String debeziumDialect; + @ConfigKey(value = "debezium.connector.class", defaultValue = "") private String debeziumConnectorClass; @@ -265,6 +272,24 @@ public String[] getIncludeTables() return includeTablesRaw.isEmpty() ? new String[0] : includeTablesRaw.split(","); } + /** + * Token used to select a {@code DebeziumSourceAdapter}. + * Prefers {@code sink.debezium.dialect}; falls back to {@code debezium.connector.class} + * so Engine-only configs keep working without duplicating the dialect. + */ + public String resolveDebeziumSourceDialect() + { + if (debeziumDialect != null && !debeziumDialect.isBlank()) + { + return debeziumDialect.trim(); + } + if (debeziumConnectorClass != null && !debeziumConnectorClass.isBlank()) + { + return debeziumConnectorClass.trim(); + } + return ""; + } + private void init() { Properties props = this.config.extractPropertiesByPrefix("", false); diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java index 689f1d3..4e5e1c5 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/PixelsSinkConstants.java @@ -20,6 +20,8 @@ public final class PixelsSinkConstants public static final String ROW_RECORD_KAFKA_PROP_FACTORY = "row-record"; public static final String TRANSACTION_KAFKA_PROP_FACTORY = "transaction"; public static final String KAFKA_VALUE_FORMAT = "pixels.sink.kafka.value.format"; + /** Sink-side CDC dialect for envelope normalization: {@code mysql} or {@code postgresql}. */ + public static final String SINK_DEBEZIUM_DIALECT = "sink.debezium.dialect"; public static final String DEBEZIUM_CONNECTOR_CLASS = "debezium.connector.class"; public static final int MAX_QUEUE_SIZE = 1_000; public static final String SNAPSHOT_TX_PREFIX = "SNAPSHOT-"; diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/factory/RowRecordKafkaPropFactory.java b/src/main/java/io/pixelsdb/pixels/sink/config/factory/RowRecordKafkaPropFactory.java index 19ebd89..be04556 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/factory/RowRecordKafkaPropFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/factory/RowRecordKafkaPropFactory.java @@ -40,12 +40,10 @@ static Properties getCommonKafkaProperties(PixelsSinkConfig config) public Properties createKafkaProperties(PixelsSinkConfig config) { Properties kafkaProperties = getCommonKafkaProperties(config); - if (config.getDebeziumConnectorClass() != null && - !config.getDebeziumConnectorClass().isBlank()) + String dialect = config.resolveDebeziumSourceDialect(); + if (!dialect.isBlank()) { - kafkaProperties.put( - PixelsSinkConstants.DEBEZIUM_CONNECTOR_CLASS, - config.getDebeziumConnectorClass()); + kafkaProperties.put(PixelsSinkConstants.SINK_DEBEZIUM_DIALECT, dialect); } if (config.getRegistryUrl() != null && !config.getRegistryUrl().isBlank()) { diff --git a/src/main/java/io/pixelsdb/pixels/sink/config/factory/TransactionKafkaPropFactory.java b/src/main/java/io/pixelsdb/pixels/sink/config/factory/TransactionKafkaPropFactory.java index c65f16e..464f162 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/config/factory/TransactionKafkaPropFactory.java +++ b/src/main/java/io/pixelsdb/pixels/sink/config/factory/TransactionKafkaPropFactory.java @@ -31,12 +31,10 @@ public class TransactionKafkaPropFactory implements KafkaPropFactory public Properties createKafkaProperties(PixelsSinkConfig config) { Properties kafkaProperties = getCommonKafkaProperties(config); - if (config.getDebeziumConnectorClass() != null && - !config.getDebeziumConnectorClass().isBlank()) + String dialect = config.resolveDebeziumSourceDialect(); + if (!dialect.isBlank()) { - kafkaProperties.put( - PixelsSinkConstants.DEBEZIUM_CONNECTOR_CLASS, - config.getDebeziumConnectorClass()); + kafkaProperties.put(PixelsSinkConstants.SINK_DEBEZIUM_DIALECT, dialect); } if (config.getRegistryUrl() != null && !config.getRegistryUrl().isBlank()) { diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumSourceAdapterSelector.java b/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumSourceAdapterSelector.java index 645f85b..2ac349f 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumSourceAdapterSelector.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/engine/adapter/DebeziumSourceAdapterSelector.java @@ -20,7 +20,8 @@ import io.pixelsdb.pixels.sink.conversion.debezium.dialect.DebeziumSourceAdapterRegistry; /** - * Wiring-side selector that reads {@code debezium.connector.class}. + * Wiring-side selector for sink CDC dialect ({@code sink.debezium.dialect}, + * with fallback to {@code debezium.connector.class}). */ public final class DebeziumSourceAdapterSelector { @@ -31,15 +32,15 @@ private DebeziumSourceAdapterSelector() public static DebeziumSourceAdapter configured() { return DebeziumSourceAdapterRegistry.resolve( - PixelsSinkConfigFactory.getInstance().getDebeziumConnectorClass()); + PixelsSinkConfigFactory.getInstance().resolveDebeziumSourceDialect()); } public static DebeziumSourceAdapter configuredIfPresent() { - String connector = - PixelsSinkConfigFactory.getInstance().getDebeziumConnectorClass(); - return connector == null || connector.isBlank() + String dialect = + PixelsSinkConfigFactory.getInstance().resolveDebeziumSourceDialect(); + return dialect == null || dialect.isBlank() ? null - : DebeziumSourceAdapterRegistry.resolve(connector); + : DebeziumSourceAdapterRegistry.resolve(dialect); } } diff --git a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverter.java b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverter.java index 81f4b57..44f3d86 100644 --- a/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverter.java +++ b/src/main/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverter.java @@ -98,7 +98,8 @@ public static KafkaRecordConverter forTransaction Properties properties) { String format = resolveFormat(properties); - DebeziumSourceAdapter adapter = resolveAdapter(properties); + // Transaction envelopes lack source.connector; dialect must be configured up front. + DebeziumSourceAdapter adapter = requireAdapter(properties); return switch (format) { case KafkaValueFormat.JSON -> @@ -175,12 +176,25 @@ private static String resolveFormat(Properties properties) private static DebeziumSourceAdapter resolveAdapter(Properties properties) { - Object connector = properties.get(PixelsSinkConstants.DEBEZIUM_CONNECTOR_CLASS); - if (connector == null || connector.toString().isBlank()) + Object dialect = properties.get(PixelsSinkConstants.SINK_DEBEZIUM_DIALECT); + if (dialect == null || dialect.toString().isBlank()) { return null; } - return DebeziumSourceAdapterRegistry.resolve(connector.toString()); + return DebeziumSourceAdapterRegistry.resolve(dialect.toString()); + } + + private static DebeziumSourceAdapter requireAdapter(Properties properties) + { + DebeziumSourceAdapter adapter = resolveAdapter(properties); + if (adapter == null) + { + throw new IllegalStateException( + PixelsSinkConstants.SINK_DEBEZIUM_DIALECT + + " is required for Kafka transaction decoding" + + " (mysql or postgresql)"); + } + return adapter; } private static Map toConfigMap(Properties properties) diff --git a/src/main/resources/pixels-sink.aws.properties b/src/main/resources/pixels-sink.aws.properties index 34a3b06..badfca5 100644 --- a/src/main/resources/pixels-sink.aws.properties +++ b/src/main/resources/pixels-sink.aws.properties @@ -1,6 +1,8 @@ # engine | kafka | storage sink.datasource=storage sink.storage.mode=stream +# CDC dialect for envelope normalization: mysql | postgresql +sink.debezium.dialect=postgresql # Sink Config: retina | csv | proto | none sink.mode=none # Kafka Config diff --git a/src/main/resources/pixels-sink.local.properties b/src/main/resources/pixels-sink.local.properties index 512c38f..51fff2d 100644 --- a/src/main/resources/pixels-sink.local.properties +++ b/src/main/resources/pixels-sink.local.properties @@ -1,6 +1,8 @@ # engine | kafka | storage sink.datasource=storage sink.storage.mode=stream +# CDC dialect for envelope normalization: mysql | postgresql +sink.debezium.dialect=postgresql # Sink Config: retina | csv | proto | none sink.mode=proto # Kafka Config diff --git a/src/test/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverterTest.java b/src/test/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverterTest.java index dc95138..8793f3b 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverterTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/source/kafka/serde/KafkaRecordConverterTest.java @@ -26,6 +26,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; class KafkaRecordConverterTest { @@ -74,10 +75,23 @@ void shouldPropagateInvalidTransactionPayload() } } + @Test + void shouldRejectTransactionConverterWithoutDialect() + { + Properties properties = new Properties(); + properties.put(PixelsSinkConstants.KAFKA_VALUE_FORMAT, "json"); + + IllegalStateException error = assertThrows( + IllegalStateException.class, + () -> KafkaRecordConverter.forTransaction(properties)); + assertTrue(error.getMessage().contains(PixelsSinkConstants.SINK_DEBEZIUM_DIALECT)); + } + private static Properties jsonProperties() { Properties properties = new Properties(); properties.put(PixelsSinkConstants.KAFKA_VALUE_FORMAT, "json"); + properties.put(PixelsSinkConstants.SINK_DEBEZIUM_DIALECT, "postgresql"); return properties; } } diff --git a/src/test/resources/pixels-sink-test.properties b/src/test/resources/pixels-sink-test.properties index 54cff79..c7e33b3 100644 --- a/src/test/resources/pixels-sink-test.properties +++ b/src/test/resources/pixels-sink-test.properties @@ -1,4 +1,5 @@ sink.monitor.enable=false sink.monitor.report.enable=false sink.mode=none +sink.debezium.dialect=mysql debezium.connector.class=io.debezium.connector.mysql.MySqlConnector From 93868e52751caab0af277443b3940670f539698f Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Sat, 1 Aug 2026 00:32:31 +0800 Subject: [PATCH 13/13] chore: remove unused test helpers and dead manual harnesses --- .../pixelsdb/pixels/sink/SplitStringTest.java | 47 ---- .../io/pixelsdb/pixels/sink/TestUtils.java | 32 --- .../pixels/sink/util/TestDateUtil.java | 52 ---- .../pixels/sink/writer/RpcEndToEndTest.java | 258 ------------------ .../retina/RetinaWriterIntegrationTest.java | 19 +- .../sink/writer/retina/RetinaWriterTest.java | 10 +- 6 files changed, 21 insertions(+), 397 deletions(-) delete mode 100644 src/test/java/io/pixelsdb/pixels/sink/SplitStringTest.java delete mode 100644 src/test/java/io/pixelsdb/pixels/sink/TestUtils.java delete mode 100644 src/test/java/io/pixelsdb/pixels/sink/util/TestDateUtil.java delete mode 100644 src/test/java/io/pixelsdb/pixels/sink/writer/RpcEndToEndTest.java diff --git a/src/test/java/io/pixelsdb/pixels/sink/SplitStringTest.java b/src/test/java/io/pixelsdb/pixels/sink/SplitStringTest.java deleted file mode 100644 index 51f31c5..0000000 --- a/src/test/java/io/pixelsdb/pixels/sink/SplitStringTest.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2023 PixelsDB. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.pixelsdb.pixels.sink; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; - -/** - * Created at: 29/04/2021 - * Author: hank - */ -class SplitStringTest -{ - @Test - void shouldSplitPipeDelimitedRecordAndKeepTrailingEmptyField() - { - String s = "1|3689999|O|224560.83|1996-01-02|5-LOW|Clerk#000095055|0|nstructions sleep furiously among |"; - String[] splits = s.split("\\|", -1); - - assertArrayEquals(new String[] { - "1", - "3689999", - "O", - "224560.83", - "1996-01-02", - "5-LOW", - "Clerk#000095055", - "0", - "nstructions sleep furiously among ", - "" - }, splits); - } -} diff --git a/src/test/java/io/pixelsdb/pixels/sink/TestUtils.java b/src/test/java/io/pixelsdb/pixels/sink/TestUtils.java deleted file mode 100644 index 45d8c44..0000000 --- a/src/test/java/io/pixelsdb/pixels/sink/TestUtils.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2023 PixelsDB. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.pixelsdb.pixels.sink; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -public class TestUtils -{ - public static ExecutorService synchronousExecutor() - { - return Executors.newSingleThreadExecutor(runnable -> - { - Thread thread = new Thread(runnable); - thread.setDaemon(true); - return thread; - }); - } -} diff --git a/src/test/java/io/pixelsdb/pixels/sink/util/TestDateUtil.java b/src/test/java/io/pixelsdb/pixels/sink/util/TestDateUtil.java deleted file mode 100644 index 538374e..0000000 --- a/src/test/java/io/pixelsdb/pixels/sink/util/TestDateUtil.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2026 PixelsDB. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.pixelsdb.pixels.sink.util; - -import io.pixelsdb.pixels.core.utils.DatetimeUtils; - -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.Calendar; -import java.util.Date; - -public final class TestDateUtil -{ - private TestDateUtil() - { - } - - public static Date fromDebeziumDate(int epochDay) - { - Calendar calendar = Calendar.getInstance(); - calendar.clear(); - calendar.set(1970, Calendar.JANUARY, 1); - calendar.add(Calendar.DAY_OF_MONTH, epochDay); - return calendar.getTime(); - } - - public static String convertDateToDayString(Date date) - { - return new java.text.SimpleDateFormat("yyyy-MM-dd").format(date); - } - - public static String convertDebeziumTimestampToString(long epochTs) - { - LocalDateTime dateTime = LocalDateTime.ofInstant( - Instant.ofEpochMilli(epochTs), ZoneId.systemDefault()); - return dateTime.format(DatetimeUtils.SQL_LOCAL_DATE_TIME); - } -} diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/RpcEndToEndTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/RpcEndToEndTest.java deleted file mode 100644 index 47231e3..0000000 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/RpcEndToEndTest.java +++ /dev/null @@ -1,258 +0,0 @@ -/* - * Copyright 2025 PixelsDB. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.pixelsdb.pixels.sink.writer; - -import com.google.protobuf.ByteString; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.pixelsdb.pixels.sink.PixelsPollingServiceGrpc; -import io.pixelsdb.pixels.sink.SinkProto; -import io.pixelsdb.pixels.sink.config.PixelsSinkConfig; -import io.pixelsdb.pixels.sink.config.factory.PixelsSinkConfigFactory; -import io.pixelsdb.pixels.sink.event.RowChangeEvent; -import io.pixelsdb.pixels.sink.exception.SinkException; -import io.pixelsdb.pixels.sink.source.SinkSource; -import io.pixelsdb.pixels.sink.source.SinkSourceFactory; -import io.pixelsdb.pixels.sink.util.MetricsFacade; -import io.pixelsdb.pixels.sink.writer.retina.SinkContextManager; -import io.prometheus.client.exporter.HTTPServer; -import io.prometheus.client.hotspot.DefaultExports; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -public class RpcEndToEndTest -{ - private static final String TEST_SCHEMA = "pixels_bench_sf10x"; - private static final String TEST_TABLE = "savingaccount"; - private static final String FULL_TABLE_NAME = TEST_SCHEMA + "." + TEST_TABLE; - private static SinkSource sinkSource; // Keep a reference to stop it later - private static HTTPServer prometheusHttpServer; - - // This method contains the setup logic from PixelsSinkApp - private static void startServer() - { - System.out.println("[SETUP] Starting server with full initialization sequence..."); - try - { - // === 1. Mimic init() method from PixelsSinkApp === - String configPath = requiredProperty("pixels.sink.integration.config"); - int testPort = configuredPort(); - System.out.println("[SETUP] Initializing configuration from " + configPath + "..."); - PixelsSinkConfigFactory.initialize(configPath); - - System.out.println("[SETUP] Initializing MetricsFacade..."); - MetricsFacade.getInstance().setSinkContextManager(SinkContextManager.getInstance()); - - // For determinism in testing, override the port from the config file - System.setProperty("sink.flink.server.port", String.valueOf(testPort)); - // === 2. Mimic main() method from PixelsSinkApp === - PixelsSinkConfig config = PixelsSinkConfigFactory.getInstance(); - System.out.println("[SETUP] Creating SinkSource application engine..."); - sinkSource = SinkSourceFactory.createSinkSource(); - System.out.println("[SETUP] Setting up Prometheus monitoring server..."); - if (config.isMonitorEnabled()) - { - DefaultExports.initialize(); - // To avoid port conflicts during tests, you could use port 0 for a random port - // For this example, we'll assume the config port is fine. - prometheusHttpServer = new HTTPServer(config.getMonitorPort()); - System.out.println("[SETUP] Prometheus server started on port: " + config.getMonitorPort()); - } - // === 3. THE MOST CRITICAL STEP: Start the processor === - System.out.println("[SETUP] Starting SinkSource processor threads..."); - sinkSource.start(); - System.out.println("[SETUP] Server is fully initialized and running."); - } catch (IOException e) - { - // If setup fails, we cannot run the test. - throw new RuntimeException("Failed to start the server for the test", e); - } - } - - public static void main(String[] args) throws InterruptedException, IOException - { - // ========== 1. SETUP PHASE ========== - // Start the server components within this same process. - startServer(); - - // Give the server a moment to fully start up and bind to the port. - Thread.sleep(2000); // 2 seconds, adjust if needed - // [REFACTORED] To ensure the FlinkPollingWriter uses our test port, - // we set it as a system property before the writer is created. - // The PixelsSinkConfigFactory should be configured to read this property. - System.setProperty("sink.flink.server.port", String.valueOf(configuredPort())); - - // [REFACTORED] The setup is now dramatically simpler. - // Instantiating FlinkPollingWriter is the ONLY step needed. - // Its constructor automatically creates the service and starts the gRPC server. - System.out.println("[SETUP] Initializing FlinkPollingWriter, which will start the gRPC server..."); - PixelsSinkWriter writer = PixelsSinkWriterFactory.getWriter(); - - // [REFACTORED] The following lines are no longer needed and have been removed: - // PixelsPollingServiceImpl serviceImpl = new PixelsPollingServiceImpl(writer); - // PollingRpcServer server = new PollingRpcServer(serviceImpl, TEST_PORT); - // server.start(); - - ExecutorService executor = Executors.newFixedThreadPool(2); - - try - { - // 3. Start the mock data producer (in a separate thread) - executor.submit(() -> - { - try - { - System.out.println("[PRODUCER] Starting mock data producer..."); - // Simulate producing 5 INSERT records - for (int i = 1; i <= 5; i++) - { - Thread.sleep(1000); // Simulate data production interval - - SinkProto.SourceInfo sourceInfo = SinkProto.SourceInfo.newBuilder() - .setDb(TEST_SCHEMA) - .setTable(TEST_TABLE) - .build(); - - byte[] idBytes = String.valueOf(i).getBytes(StandardCharsets.UTF_8); - SinkProto.ColumnValue idColumnValue = SinkProto.ColumnValue.newBuilder() - .setValue(ByteString.copyFrom(idBytes)) - .build(); - - SinkProto.RowValue afterImage = SinkProto.RowValue.newBuilder() - .addValues(idColumnValue) - .build(); - - SinkProto.RowRecord record = SinkProto.RowRecord.newBuilder() - .setSource(sourceInfo) - .setOp(SinkProto.OperationType.INSERT) - .setAfter(afterImage) - .build(); - - RowChangeEvent event = new RowChangeEvent(record); - System.out.printf("[PRODUCER] Writing INSERT record %d for table '%s.%s'%n", i, TEST_SCHEMA, TEST_TABLE); - writer.writeRow(event); - } - System.out.println("[PRODUCER] Finished writing data."); - } catch (InterruptedException e) - { - System.err.println("[PRODUCER] Producer thread was interrupted."); - Thread.currentThread().interrupt(); - } catch (SinkException e) - { - System.err.println("[PRODUCER] SinkException occurred: %s" + e.getMessage()); - } - }); - - // 4. Start the gRPC client (in another thread) - executor.submit(() -> - { - System.out.println("[CLIENT] Starting gRPC client..."); - ManagedChannel channel = ManagedChannelBuilder - .forAddress(requiredProperty("pixels.sink.test.host"), configuredPort()) - .usePlaintext() - .build(); - - try - { - PixelsPollingServiceGrpc.PixelsPollingServiceBlockingStub client = PixelsPollingServiceGrpc.newBlockingStub(channel); - int recordsPolled = 0; - int maxRetries = 10; - int retryCount = 0; - while (recordsPolled < 5 && retryCount < maxRetries) - { - System.out.println("[CLIENT] Sending poll request for table '" + FULL_TABLE_NAME + "'..."); - SinkProto.PollRequest request = SinkProto.PollRequest.newBuilder() - .setSchemaName(TEST_SCHEMA) - .setTableName(TEST_TABLE) - .build(); - SinkProto.PollResponse response = client.pollEvents(request); - - if (response.getRecordsCount() > 0) - { - System.out.printf("[CLIENT] SUCCESS: Polled %d record(s).%n", response.getRecordsCount()); - response.getRecordsList().forEach(record -> System.out.println(" -> " + record.toString().trim())); - recordsPolled += response.getRecordsCount(); - } else - { - System.out.println("[CLIENT] Polled 0 records, will retry..."); - retryCount++; - } - } - if (recordsPolled == 5) - { - System.out.println("[CLIENT] Test finished successfully. Polled all 5 records."); - } else - { - System.err.println("[CLIENT] Test FAILED. Did not poll all records in time."); - } - } finally - { - channel.shutdownNow(); - } - }); - } finally - { - // 5. Wait for the test to complete and clean up resources - executor.shutdown(); - if (!executor.awaitTermination(1, TimeUnit.MINUTES)) - { - System.err.println("[CLEANUP] Test timed out."); - executor.shutdownNow(); - } else - { - System.out.println("[CLEANUP] Test completed."); - } - - // [REFACTORED] The correct way to stop the server is now by closing the writer. - writer.close(); - System.out.println("[CLEANUP] FlinkPollingWriter closed and server stopped."); - } - // ========== 3. CLEANUP PHASE ========== - System.out.println("[CLEANUP] Tearing down server components..."); - if (sinkSource != null) - { - sinkSource.close(); - System.out.println("[CLEANUP] SinkSource closed."); - // Note: The writer is part of sinkSource, and its resources - // should be cleaned up by close. - } - if (prometheusHttpServer != null) - { - prometheusHttpServer.close(); - System.out.println("[CLEANUP] Prometheus server stopped."); - } - System.out.println("[CLEANUP] Test finished."); - } - - private static int configuredPort() - { - return Integer.parseInt(requiredProperty("pixels.sink.test.port")); - } - - private static String requiredProperty(String key) - { - String value = System.getProperty(key); - if (value == null || value.isBlank()) - { - throw new IllegalStateException("Set -D" + key + " to run this integration test"); - } - return value; - } -} diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterIntegrationTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterIntegrationTest.java index 53171b3..32e4bb9 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterIntegrationTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterIntegrationTest.java @@ -27,8 +27,8 @@ import io.pixelsdb.pixels.sink.TestConfig; import io.pixelsdb.pixels.sink.event.RowChangeEvent; import io.pixelsdb.pixels.sink.exception.SinkException; +import io.pixelsdb.pixels.core.utils.DatetimeUtils; import io.pixelsdb.pixels.sink.metadata.TableMetadataRegistry; -import io.pixelsdb.pixels.sink.util.TestDateUtil; import io.pixelsdb.pixels.sink.writer.PixelsSinkWriter; import io.pixelsdb.pixels.sink.writer.PixelsSinkWriterFactory; import org.junit.jupiter.api.Assertions; @@ -41,6 +41,9 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; import java.util.ArrayList; import java.util.List; import java.util.Random; @@ -383,14 +386,14 @@ public void testCheckingAccountUpdatePerformance() throws .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Integer.toString(userID))).build()) .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Float.toString(oldBalance))).build()) .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Integer.toString(isBlocked))).build()) - .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(TestDateUtil.convertDebeziumTimestampToString(oldTs))).build()); + .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(formatTimestamp(oldTs))).build()); SinkProto.RowValue.Builder afterValueBuilder = SinkProto.RowValue.newBuilder() .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Integer.toString(accountID))).build()) .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Integer.toString(userID))).build()) .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Float.toString(newBalance))).build()) .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(Integer.toString(isBlocked))).build()) - .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(TestDateUtil.convertDebeziumTimestampToString(newTs))).build()); + .addValues(SinkProto.ColumnValue.newBuilder().setValue(ByteString.copyFromUtf8(formatTimestamp(newTs))).build()); SinkProto.RowRecord.Builder rowBuilder = SinkProto.RowRecord.newBuilder() .setOp(SinkProto.OperationType.UPDATE) @@ -412,7 +415,7 @@ public void testCheckingAccountUpdatePerformance() throws .addColValues(ByteString.copyFromUtf8(Integer.toString(userID))) .addColValues(ByteString.copyFromUtf8(Float.toString(newBalance))) .addColValues(ByteString.copyFromUtf8(Integer.toString(isBlocked))) - .addColValues(ByteString.copyFromUtf8(TestDateUtil.convertDebeziumTimestampToString(newTs))) + .addColValues(ByteString.copyFromUtf8(formatTimestamp(newTs))) .addIndexKeys(rowChangeEvent.getAfterKey()); tableUpdateDataBuilder.addInsertData(insertDataBuilder.build()); } @@ -502,7 +505,7 @@ public void testInsertTwoTablePerformance() throws cols[1] = Integer.toString(userID).getBytes(StandardCharsets.UTF_8); cols[2] = Float.toString(balance).getBytes(StandardCharsets.UTF_8); cols[3] = Integer.toString(isBlocked).getBytes(StandardCharsets.UTF_8); - cols[4] = TestDateUtil.convertDebeziumTimestampToString(ts).getBytes(StandardCharsets.UTF_8); + cols[4] = formatTimestamp(ts).getBytes(StandardCharsets.UTF_8); // cols[4] = Long.toString(ts).getBytes(StandardCharsets.UTF_8); // after row SinkProto.RowValue.Builder afterValueBuilder = SinkProto.RowValue.newBuilder() @@ -595,4 +598,10 @@ public void testInsertTwoTablePerformance() throws double transPerSec = batchCount * 2 / seconds; logger.info("Inserted " + totalInserts + " rows in " + seconds + "s, rate=" + insertsPerSec + " inserts/s," + transPerSec + "trans/s"); } + + private static String formatTimestamp(long epochTs) + { + return LocalDateTime.ofInstant(Instant.ofEpochMilli(epochTs), ZoneId.systemDefault()) + .format(DatetimeUtils.SQL_LOCAL_DATE_TIME); + } } diff --git a/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterTest.java b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterTest.java index 7e80512..2bfd9f5 100644 --- a/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterTest.java +++ b/src/test/java/io/pixelsdb/pixels/sink/writer/retina/RetinaWriterTest.java @@ -17,10 +17,8 @@ import io.pixelsdb.pixels.sink.SinkProto; import io.pixelsdb.pixels.sink.TestConfig; -import io.pixelsdb.pixels.sink.TestUtils; import io.pixelsdb.pixels.sink.event.RowChangeEvent; import io.pixelsdb.pixels.sink.exception.SinkException; -import io.pixelsdb.pixels.sink.writer.retina.RetinaWriter; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @@ -37,6 +35,7 @@ import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -56,7 +55,12 @@ void setUp() throws Exception { TestConfig.initializeIntegrationConfig(); - testExecutor = TestUtils.synchronousExecutor(); + testExecutor = Executors.newSingleThreadExecutor(runnable -> + { + Thread thread = new Thread(runnable); + thread.setDaemon(true); + return thread; + }); dispatchedEvents = Collections.synchronizedList(new ArrayList<>()); coordinator = new RetinaWriter();