From 16bf48ddc0011a9ecfa13a74353f79f3c49bff92 Mon Sep 17 00:00:00 2001 From: Celia King Date: Tue, 28 Jul 2026 17:44:32 +1000 Subject: [PATCH] Update with get started instructions I'm hoping it will make it a bit clearer on how to start navigating this code There were also a few little things from when I've been working on these that were worth committing --- .gitignore | 3 ++ README.md | 6 +++ datagen/README.md | 2 +- features/log-clients/consumer.py | 2 +- features/offset-management/README.md | 2 + features/offset-management/consumer.py | 2 +- features/offset-management/simple_consumer.py | 4 +- features/rapid-kafka-diagnostics/README.md | 12 ++--- .../compose-consumer.yml | 4 +- fh-local-labs/lab-01/consumer.py | 2 +- getting-started/README.md | 50 +++++++++++++++++++ 11 files changed, 75 insertions(+), 14 deletions(-) create mode 100644 getting-started/README.md diff --git a/.gitignore b/.gitignore index aca670e..dda7d04 100644 --- a/.gitignore +++ b/.gitignore @@ -185,3 +185,6 @@ bin/ ### Mac OS ### .DS_Store + +.idea/ +integrations/oci-streaming/deps/ diff --git a/README.md b/README.md index 89d69a6..a50c450 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ LinkedIn Business (1) +## Getting Started + +New here? Start with **[Getting Started](./getting-started/)**, a single `docker-compose.yml` gets Kpow and/or Flex running locally. + +Once you've got that running, [Factor House Local](https://github.com/factorhouse/factorhouse-local) is the full local development environment (Kafka, Flink, Spark, Iceberg, Pinot) that the rest of this repo, the [Labs](./fh-local-labs/), [Features](#product-features), and [Projects](#projects) below, builds on. + ## Product Features
diff --git a/datagen/README.md b/datagen/README.md index 11da79b..3612efa 100644 --- a/datagen/README.md +++ b/datagen/README.md @@ -6,5 +6,5 @@ We recommend using these scripts in conjunction with the local development envir Explore the available data generators: -- [theLook eCommerce Data Generator](./look-ecomm/) +- [theLook eCommerce Data Generator](./thelook-ecomm/) - [Mobile Game User Score](./mobile-game/) diff --git a/features/log-clients/consumer.py b/features/log-clients/consumer.py index edc871b..0c2fb5f 100644 --- a/features/log-clients/consumer.py +++ b/features/log-clients/consumer.py @@ -28,7 +28,7 @@ def on_assign(consumer: Consumer, partitions: List[TopicPartition]): client_conf = { "bootstrap.servers": BOOTSTRAP_SERVERS, - "group.id": f"{TOPIC_NAME}-group", + "group.id": f"{TOPIC_NAME}-regular-consumer-group", "auto.offset.reset": "earliest", "enable.auto.commit": False, } diff --git a/features/offset-management/README.md b/features/offset-management/README.md index 205b9f4..6830bd0 100644 --- a/features/offset-management/README.md +++ b/features/offset-management/README.md @@ -51,6 +51,8 @@ Start the producer and consumer. python features/offset-management/producer.py # consumer python features/offset-management/consumer.py +# simple-consumer +python features/offset-management/simple_consumer.py ``` ## Shutdown environment diff --git a/features/offset-management/consumer.py b/features/offset-management/consumer.py index 0d17a06..b44e461 100644 --- a/features/offset-management/consumer.py +++ b/features/offset-management/consumer.py @@ -37,7 +37,7 @@ def assignment_callback(_: Consumer, partitions: List[TopicPartition]): value = message.value().decode("utf8") partition = message.partition() logging.info( - f"Reveived {value} from partition {message.partition()}, offset {message.offset()}." + f"Reveived {value} from partition {message.partition()}, offset {message.offset()} for regular consumer." ) consumer.commit(message) except KeyboardInterrupt: diff --git a/features/offset-management/simple_consumer.py b/features/offset-management/simple_consumer.py index 4fabff6..37555de 100644 --- a/features/offset-management/simple_consumer.py +++ b/features/offset-management/simple_consumer.py @@ -23,7 +23,7 @@ def assignment_callback(_: Consumer, partitions: List[TopicPartition]): conf = { "bootstrap.servers": BOOTSTRAP_SERVERS, - "group.id": f"{TOPIC_NAME}-group", # group.id is still needed for committing offsets + "group.id": f"{TOPIC_NAME}-simple-consumer-group", # group.id is still needed for committing offsets "auto.offset.reset": "earliest", "enable.auto.commit": False, } @@ -49,7 +49,7 @@ def assignment_callback(_: Consumer, partitions: List[TopicPartition]): else: value = message.value().decode("utf8") logging.info( - f"Received {value} from partition {message.partition()}, offset {message.offset()}." + f"Received {value} from partition {message.partition()}, offset {message.offset()} from a simple consumer." ) consumer.commit(message) except KeyboardInterrupt: diff --git a/features/rapid-kafka-diagnostics/README.md b/features/rapid-kafka-diagnostics/README.md index b80cc27..cee4507 100644 --- a/features/rapid-kafka-diagnostics/README.md +++ b/features/rapid-kafka-diagnostics/README.md @@ -2,7 +2,7 @@ This demo simulates a **"Silent Stall"** production incident. In this scenario, a Kafka producer injects a "poison pill" (malformed) message into a specific topic partition. -The environment runs three consumer instances. When one of the instances consumes the malicious message, it enters a "Zombie" state—it continuously crashes and retries the same message logic while successfully sending heartbeats to the Kafka cluster. This mimics a difficult-to-diagnose issue where infrastructure looks healthy, but data processing has halted on a specific partition. +The environment runs six consumer instances (one per partition). When one of the instances consumes the malicious message, it enters a "Zombie" state—it continuously crashes and retries the same message logic while successfully sending heartbeats to the Kafka cluster. This mimics a difficult-to-diagnose issue where infrastructure looks healthy, but data processing has halted on a specific partition. ### Getting Started @@ -23,7 +23,7 @@ git clone https://github.com/factorhouse/factorhouse-local.git ## Export your license details export KPOW_SUFFIX="-ce" -export KPOW_LICENSE_FILE= +export KPOW_LICENSE= ## Start Kafka and Kpow docker compose -p kpow -f ./factorhouse-local/compose-kpow.yml up -d @@ -33,7 +33,7 @@ docker compose -p kpow -f ./factorhouse-local/compose-kpow.yml up -d ### Launch the Demo -We will launch a producer to generate traffic and a consumer group scaled to 3 instances to process it. +We will launch a producer to generate traffic and a consumer group scaled to 6 instances (one per partition) to process it. #### Start the Producer @@ -45,7 +45,7 @@ docker compose -p producer -f ./features/rapid-kafka-diagnostics/compose-produce #### Start the Consumers -Scale the consumer application to 3 instances to ensure all partitions are covered. +Scale the consumer application to 6 instances to ensure all partitions are covered. ```bash docker compose -p consumer -f ./features/rapid-kafka-diagnostics/compose-consumer.yml up -d --scale consumer=6 @@ -55,7 +55,7 @@ docker compose -p consumer -f ./features/rapid-kafka-diagnostics/compose-consume 1. Open Kpow. 2. Observe that the **Consumer Group** state is `Stable` (Green). -3. Drill down into the Consumer Group details. You will see that **Partition 2** has a growing lag, while Partitions 0 and 1 are processing normally. +3. Drill down into the Consumer Group details. You will see that **Partition 2** has a growing lag, while Partitions 0, 1, 3, 4, and 5 are processing normally. 4. Use **Data Inspect** to view the message at the head of the lag to identify the schema mismatch. ### Shutdown Environment @@ -69,5 +69,5 @@ docker compose -p consumer -f ./features/rapid-kafka-diagnostics/compose-consume && docker compose -p kpow -f ./factorhouse-local/compose-kpow.yml down # Clean up variables -unset KPOW_SUFFIX KPOW_LICENSE_FILE +unset KPOW_SUFFIX KPOW_LICENSE ``` diff --git a/features/rapid-kafka-diagnostics/compose-consumer.yml b/features/rapid-kafka-diagnostics/compose-consumer.yml index 5c7ba38..937304f 100644 --- a/features/rapid-kafka-diagnostics/compose-consumer.yml +++ b/features/rapid-kafka-diagnostics/compose-consumer.yml @@ -1,5 +1,5 @@ services: - # consumer - scale to 3 instances + # consumer - scale to 6 instances (one per partition) consumer: image: rapid-kafka-diagnostics:consumer build: . @@ -7,7 +7,7 @@ services: networks: - factorhouse deploy: - replicas: 3 + replicas: 6 environment: BOOTSTRAP_SERVERS: "kafka-1:19092" TOPIC: "orders" diff --git a/fh-local-labs/lab-01/consumer.py b/fh-local-labs/lab-01/consumer.py index 7a61e3f..1b168e7 100644 --- a/fh-local-labs/lab-01/consumer.py +++ b/fh-local-labs/lab-01/consumer.py @@ -35,7 +35,7 @@ def on_assign(consumer: Consumer, partitions: List[TopicPartition]): client_conf = { "bootstrap.servers": BOOTSTRAP_SERVERS, - "group.id": f"{TOPIC_NAME}-group-{(MODEL_VERSION or 'generic')}", + "group.id": f"{TOPIC_NAME}-lab-01-group-{(MODEL_VERSION or 'generic')}", "auto.offset.reset": "earliest", "enable.auto.commit": False, } diff --git a/getting-started/README.md b/getting-started/README.md new file mode 100644 index 0000000..61aa481 --- /dev/null +++ b/getting-started/README.md @@ -0,0 +1,50 @@ +# Getting Started + +The fastest way to try **Kpow** (Kafka) and/or **Flex** (Flink) locally. + +Each folder is a self-contained `docker-compose.yml`. You'll need to get a free license, but once you've included it in its docker file, that's all the setup you'll need. When you've got the docker file running, you'll be able to see it on localhost. It is worth noting that will take a couple of minutes for it to warm up and for data to appear. + +If you're looking for the full local development environment (multi-broker Kafka, Flink + Spark + Iceberg + Pinot, etc.) instead, see [Factor House Local](https://github.com/factorhouse/factorhouse-local). And if you want to explore more, try [the Labs](https://github.com/factorhouse//fh-local-labs/). + +## Prerequisites + +- Docker (Desktop or Engine) installed and running. +- A Factor House license. Get one free at [account.factorhouse.io/](https://account.factorhouse.io/) — a single **Community License** (free, non-commercial) covers a limited set of features for Kpow and Flex. A 30-day trial license gives full Enterprise features for Kpow or Flex if you want to evaluate those instead. + +## Pick a path + +| Folder | What it starts | Where | +|---|---|---| +| [`kpow/`](./kpow/) | Kpow (Enterprise) + a single Kafka broker (KRaft) + a producer that seeds a topic for you | `cd getting-started/kpow` | +| [`flex/`](./flex/) | Flex (Enterprise) + a standalone Flink cluster running the built-in `TopSpeedWindowing` example job | `cd getting-started/flex` | +| [`community/`](./community/) | Kpow and/or Flex **Community Edition**, same idea, selected via Compose profiles | `cd getting-started/community` | + +### Add your license + +Each `docker-compose.yml` here has a `LICENSE_ID` / `LICENSE_CODE` / `LICENSEE` / `LICENSE_EXPIRY` / `LICENSE_SIGNATURE` block with either `` values or nothing filled in. Open the license file you generated from the portal above and copy the matching values into that block before running `docker compose up`. + +These are meant to stay **local, uncommitted changes** — don't commit a compose file with real license values filled in. + +### Run the docker command + +```bash +# in the kpow or flex folder +docker compose up -d +``` +Kpow or Flex: `http://localhost:3000` + +```bash +# in the community folder (add --profile kpow, --profile flex or both) +docker compose --profile kpow --profile flex up -d +``` +Kpow: `http://localhost:3000` · Flex: `http://localhost:3001` + +## Stop everything + +From inside whichever folder you started: + +```bash +docker compose down +``` + +(for `community/`, add the same `--profile` flags you used to start it)