diff --git a/.dockerignore b/.dockerignore index 94378b0..001686d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,17 +1,23 @@ .git +.github .nx node_modules **/node_modules +.next +coverage dist **/dist .pnpm-store +*.log +logs.txt npm-debug.log* yarn-debug.log* yarn-error.log* .env .env.* -Dockerfile -**/*.tsbuildinfo +!.env.example +*.tsbuildinfo tmp temp pnpm-lock.yaml.prev +ops/deploy diff --git a/.env.example b/.env.example index b9f4061..1f5691f 100644 --- a/.env.example +++ b/.env.example @@ -1,32 +1,43 @@ +# ── DevImpact Global Environment Configuration ────────────────────────────── +# Copy this file to .env before starting the web application or background worker: +# cp .env.example .env + +# ── GitHub API Configuration ──────────────────────────────────────────────── +# GitHub Personal Access Token (PAT) for GraphQL / REST API requests. +# Required scope: public_repo (or read-only fine-grained token). GITHUB_TOKEN=your_github_token_here -# Public GitHub repository URL shown by the app. -# If omitted, the app falls back to https://github.com/O2sa/DevImpact -NEXT_PUBLIC_GITHUB_REPO_URL=your_github_repo_url_here +# Public GitHub repository URL shown by the web application. +NEXT_PUBLIC_GITHUB_REPO_URL=https://github.com/O2sa/DevImpact -# GitHub query limits +# GitHub query fetch limits per user GITHUB_REPO_COUNT=30 GITHUB_PR_COUNT=80 GITHUB_ISSUE_COUNT=20 GITHUB_DISCUSSION_COUNT=10 +GITHUB_USER_STALE_DAYS=14 -# Leaderboard source data -LEADERBOARD_SOURCE_URL_TEMPLATE=https://raw.githubusercontent.com/ashkulz/committers.top/gh-pages/_data/locations/{country}.yml -DISABLE_CALCULATE_LEADERBOARD_ENDPOINT=false +# ── Database (PostgreSQL) ─────────────────────────────────────────────────── +DATABASE_URL=postgresql://devimpact:devimpact@localhost:5432/devimpact?sslmode=disable +POSTGRES_PASSWORD=CHANGE_THIS_TO_A_LONG_RANDOM_PASSWORD -# Redis caching (optional — strongly recommended for leaderboard performance) -# Use either redis://localhost:6379 or include password if enabled: redis://:password@localhost:6379 -REDIS_URL= +# ── Caching (Redis) ───────────────────────────────────────────────────────── REDIS_ENABLED=false +REDIS_URL=redis://localhost:6379 REDIS_PASSWORD= -# CACHE_NAMESPACE is also accepted as an alias. -# CACHE_NAMESPACE=devimpact:v1 REDIS_CACHE_NAMESPACE=devimpact:v1 -# CACHE_TTL_SECONDS is also accepted as an alias. Valid range: 1-31536000. -# CACHE_TTL_SECONDS=604800 REDIS_CACHE_TTL_SECONDS=604800 REDIS_CONNECT_TIMEOUT_MS=1500 -# ── PostgreSQL (local development) ───────────────── -DATABASE_URL=postgresql://devimpact:devimpact@localhost:5432/devimpact?sslmode=disable -POSTGRES_PASSWORD=CHANGE_THIS_TO_A_LONG_RANDOM_PASSWORD +# ── Leaderboard Data & Worker Options ─────────────────────────────────────── +# Cron schedule for background calculation worker (standard 5-field cron syntax) +# Default: "0 0 * * *" (runs once per day at midnight) +LEADERBOARD_CRON_SCHEDULE=0 0 * * * + +# Data source template for seed country profiles +LEADERBOARD_SOURCE_URL_TEMPLATE=https://raw.githubusercontent.com/ashkulz/committers.top/gh-pages/_data/locations/{country}.yml + +LEADERBOARD_SEED_LIMIT=256 +LEADERBOARD_REFRESH_LIMIT=500 +LEADERBOARD_USER_STALE_DAYS=30 +DISABLE_CALCULATE_LEADERBOARD_ENDPOINT=false diff --git a/.github/workflows/deploy-vps.yml b/.github/workflows/deploy-vps.yml new file mode 100644 index 0000000..3b095dd --- /dev/null +++ b/.github/workflows/deploy-vps.yml @@ -0,0 +1,38 @@ +name: Deploy Leaderboard Worker to VPS + +on: + workflow_run: + workflows: ["Leaderboard Worker GHCR Image"] + types: + - completed + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +jobs: + deploy: + name: Deploy to VPS + runs-on: ubuntu-latest + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + timeout-minutes: 10 + env: + VPS_HOST: ${{ secrets.VPS_HOST }} + VPS_USER: ${{ secrets.VPS_USER }} + VPS_SSH_KEY: ${{ secrets.VPS_SSH_KEY }} + VPS_SSH_PORT: ${{ secrets.VPS_SSH_PORT || '22' }} + + steps: + - name: Deploy via SSH + if: ${{ env.VPS_HOST != '' && env.VPS_USER != '' && env.VPS_SSH_KEY != '' }} + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ env.VPS_HOST }} + username: ${{ env.VPS_USER }} + key: ${{ env.VPS_SSH_KEY }} + port: ${{ env.VPS_SSH_PORT }} + script: | + cd ~/projects/DevImpact || cd /app || exit 1 + bash ops/deploy/deploy-leaderboard.sh diff --git a/.github/workflows/leaderboard-image.yml b/.github/workflows/leaderboard-image.yml new file mode 100644 index 0000000..18d1241 --- /dev/null +++ b/.github/workflows/leaderboard-image.yml @@ -0,0 +1,71 @@ +name: Leaderboard Worker GHCR Image + +on: + push: + branches: + - main + paths: + - "ops/docker/**" + - "ops/cron/**" + - "package.json" + - "pnpm-lock.yaml" + - "scripts/**" + - "lib/**" + - "app/**" + - "types/**" + - ".github/workflows/leaderboard-image.yml" + workflow_dispatch: + +permissions: + contents: read + packages: write + +concurrency: + group: leaderboard-image-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-and-push: + name: Build & Push GHCR Image + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/o2sa/devimpact-leaderboard + tags: | + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + type=sha,prefix=,format=long + type=ref,event=branch + + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: . + file: ops/docker/Dockerfile.worker + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + GIT_COMMIT_SHA=${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/README.md b/README.md index cb0ee95..7f8932a 100644 --- a/README.md +++ b/README.md @@ -161,57 +161,59 @@ Final Score = ## 🚀 Getting Started -### 1. Clone the repo +### ⚡ Option A: Run Full Platform via Docker (Fastest) -```bash -git clone https://github.com/O2sa/DevImpact.git -cd DevImpact -``` - ---- - -### 2. Install dependencies +Run the entire platform (Web App UI + PostgreSQL + Redis + Worker) in 2 simple steps: ```bash -pnpm install +# 1. Copy environment template and set GITHUB_TOKEN +cp .env.example .env + +# 2. Start full platform +docker compose -f ops/docker/docker-compose.yml up -d --build ``` +Then open `http://localhost:3000` in your browser! --- -### 3. Set up environment variables +### 📦 Option B: Run Locally with Node.js & pnpm -Create a `.env` file: +1. **Install dependencies**: + ```bash + pnpm install + ``` -``` -GITHUB_TOKEN=your_github_token -NEXT_PUBLIC_GITHUB_REPO_URL=your_github_repo_url -GITHUB_REPO_COUNT=30 -GITHUB_PR_COUNT=80 -GITHUB_ISSUE_COUNT=20 -GITHUB_DISCUSSION_COUNT=10 -REDIS_URL=redis://localhost:6379 -REDIS_ENABLED=false -REDIS_CACHE_NAMESPACE=devimpact:v1 -REDIS_CACHE_TTL_SECONDS=604800 -``` +2. **Configure environment**: + ```bash + cp .env.example .env + ``` -`CACHE_NAMESPACE` and `CACHE_TTL_SECONDS` are accepted as aliases for the -Redis-prefixed cache settings. The namespace must be non-empty. Cache TTL must -be a positive integer no greater than `31536000` seconds (one year); invalid or -missing values fall back to `devimpact:v1` and `604800` seconds (seven days). +3. **Start local database & Redis**: + ```bash + pnpm db:up && pnpm redis:up + ``` ---- +4. **Run development server**: + ```bash + pnpm run dev + ``` -### 4. Run the app +5. **Calculate leaderboard scores manually**: + ```bash + pnpm leaderboard:calculate + ``` -```bash -pnpm run dev -``` +### Leaderboard Worker & Infrastructure + +The leaderboard score updates run via a dedicated background worker container using Docker & Supercronic. -To calculate the next leaderboard country from the script, run: +For complete local setup, Docker Compose instructions, GHCR publishing, and VPS deployment documentation, see **[ops/README.md](file:///c:/Users/msii/Documents/DevImpact/ops/README.md)**. ```bash -pnpm run calculate-next-country +# Quick worker setup (pulls & runs published image) +cp .env.example .env +docker compose -f ops/docker/leaderboard-compose.yml pull +docker compose -f ops/docker/leaderboard-compose.yml up -d ``` --- diff --git a/app/api/calculate-leaderboard/route.ts b/app/api/calculate-leaderboard/route.ts deleted file mode 100644 index 5a9c83b..0000000 --- a/app/api/calculate-leaderboard/route.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { NextResponse } from "next/server"; -import { calculateLeaderboard } from "@/lib/calculate-leaderboard"; - -export const runtime = "nodejs"; - -function isCalculateLeaderboardDisabled(): boolean { - const raw = - process.env.DISABLE_CALCULATE_LEADERBOARD_ENDPOINT?.trim().toLowerCase(); - return raw === "true" || raw === "1" || raw === "yes"; -} - -export async function POST(request: Request) { - if (isCalculateLeaderboardDisabled()) { - return NextResponse.json( - { success: false, error: "Not found" }, - { status: 404 }, - ); - } - - const { searchParams } = new URL(request.url); - const country = searchParams.get("country")?.trim(); - - if (!country) { - return NextResponse.json( - { success: false, error: "Provide a country parameter" }, - { status: 400 }, - ); - } - - try { - const result = await calculateLeaderboard(country); - return NextResponse.json({ success: true, ...result }); - } catch (err) { - return NextResponse.json( - { - success: false, - error: err instanceof Error ? err.message : "Failed to calculate leaderboard", - }, - { status: 502 }, - ); - } -} diff --git a/lib/db-store.ts b/lib/db-store.ts index 90e8ce1..a9882a7 100644 --- a/lib/db-store.ts +++ b/lib/db-store.ts @@ -147,6 +147,13 @@ export class DatabaseStore { async getNextCountryToCalculate(): Promise<{ slug: string; title: string } | null> { const client = getPool(); + // Auto-recover calculations that were interrupted (e.g. by container deploy/restart) + await client.query( + `UPDATE leaderboard_calculation + SET status = 'failed', error_message = 'Interrupted by process restart or timeout', updated_at = NOW() + WHERE status = 'running' AND updated_at < NOW() - INTERVAL '6 hours'`, + ); + const result = await client.query( `SELECT country_slug, country_title FROM leaderboard_calculation WHERE status != 'running' diff --git a/ops/README.md b/ops/README.md new file mode 100644 index 0000000..bc03de8 --- /dev/null +++ b/ops/README.md @@ -0,0 +1,143 @@ +# DevImpact Infrastructure & Deployment (ops/) + +This directory contains the infrastructure, Docker configuration, cron job definitions, and deployment scripts for the **DevImpact Leaderboard Worker**. + +--- + +## Overview + +### What is the Leaderboard Worker? +The Leaderboard Worker is a standalone service that periodically fetches contributor metadata from the GitHub GraphQL/REST APIs, recalculates scores, and updates the shared PostgreSQL database and Redis cache. + +### Why is it separate from the web application? +Calculating leaderboard scores involves heavy API querying, rate limit tracking, and database bulk operations. Running this work asynchronously via a background worker ensures that the Next.js web application remains fast, responsive, and unaffected by calculation spikes. + +--- + +## Directory Structure + +``` +ops/ +├── docker/ +│ ├── Dockerfile.web # Dockerfile for Next.js Web App UI +│ ├── Dockerfile.worker # Single Dockerfile for Leaderboard Worker +│ ├── .dockerignore # Docker build context exclusions +│ ├── entrypoint.sh # Worker cron startup entrypoint +│ ├── docker-compose.yml # Full platform Compose (PostgreSQL, Redis, Web App, Worker) +│ └── leaderboard-compose.yml # Leaderboard Worker Compose +├── cron/ +│ └── leaderboard.cron # Supercronic job schedule +├── deploy/ +│ └── deploy-leaderboard.sh # VPS deployment automation script +└── README.md # Infrastructure documentation +``` + +--- + +## Environment Configuration + +Copy `.env.example` to `.env` in the root directory before running the worker: + +```bash +cp .env.example .env +``` + +--- + +## Developer Workflows + +### 1. Run Published GHCR Image (Locally or on VPS) + +To run the worker using the published container image: + +```bash +# 1. Copy environment template +cp .env.example .env + +# 2. Pull and start container +docker compose -f ops/docker/leaderboard-compose.yml pull +docker compose -f ops/docker/leaderboard-compose.yml up -d +``` + +### 2. Build Docker Image Directly (Optional Local Testing) + +If you want to build the Docker image locally from source: + +```bash +docker build \ + -f ops/docker/Dockerfile.worker \ + --build-arg GIT_COMMIT_SHA=$(git rev-parse HEAD) \ + -t devimpact-leaderboard:local \ + . +``` + +### 3. Manually Run a Single Calculation + +To trigger a calculation manually inside a worker container: + +```bash +docker compose -f ops/docker/leaderboard-compose.yml run --rm \ + leaderboard-cron \ + pnpm leaderboard:calculate +``` + +### 4. Inspect Worker Logs + +The container logs Supercronic output and script execution directly to `stdout`/`stderr`: + +```bash +docker logs -f devimpact-leaderboard-cron +``` + +--- + +## CI/CD & GHCR Publishing Workflow + +The GitHub Actions workflow at [.github/workflows/leaderboard-image.yml](file:///c:/Users/msii/Documents/DevImpact/.github/workflows/leaderboard-image.yml) triggers automatically on pushes to `main` when worker or scoring code changes. + +### Image Naming & Tagging Architecture + +- **Registry**: `ghcr.io/o2sa/devimpact-leaderboard` +- **Tags**: + - `latest`: Latest build from `main` branch. + - `` (e.g., `ghcr.io/o2sa/devimpact-leaderboard:a1b2c3d...`): Immutable commit tag for reproducibility and pin/rollback capability. + +--- + +## VPS Deployment + +Production deployments on the VPS consume the prebuilt GHCR image. + +### Deployment Script + +To deploy or update the worker on the VPS: + +```bash +bash ops/deploy/deploy-leaderboard.sh +``` + +This script safely executes: +1. `docker compose -f ops/docker/leaderboard-compose.yml pull` +2. **Waits for any active calculation job to finish**: Checks if `devimpact-leaderboard-cron` is currently running a calculation job (`calculate-next-country`) and polls until the job completes naturally. +3. `docker compose -f ops/docker/leaderboard-compose.yml up -d --remove-orphans` once no calculation is running. + +### Rolling Back to a Specific Version + +To rollback to a previous version on the VPS, set the `LEADERBOARD_IMAGE` variable to an explicit commit SHA tag before executing: + +```bash +LEADERBOARD_IMAGE=ghcr.io/o2sa/devimpact-leaderboard: bash ops/deploy/deploy-leaderboard.sh +``` + +--- + +## Concurrent Job Handling & Locking Note + +The leaderboard script uses database-level tracking (`leaderboard_calculation` table with `status = 'running'`). The query selects the next country where `status != 'running'`, preventing the worker from picking a country currently being processed. + +--- + +## Security Best Practices + +1. **Non-Root & Unprivileged**: The container runs under standard user permissions without `--privileged` or Docker socket access. +2. **Runtime Injection**: All credentials (`GITHUB_TOKEN`, `DATABASE_URL`) are passed at container startup via environment variables. diff --git a/ops/cron/leaderboard.cron b/ops/cron/leaderboard.cron new file mode 100644 index 0000000..475068a --- /dev/null +++ b/ops/cron/leaderboard.cron @@ -0,0 +1 @@ +0 0 * * * cd /app && pnpm leaderboard:calculate >> /proc/1/fd/1 2>> /proc/1/fd/2 diff --git a/ops/deploy/deploy-leaderboard.sh b/ops/deploy/deploy-leaderboard.sh new file mode 100644 index 0000000..1fcf9bb --- /dev/null +++ b/ops/deploy/deploy-leaderboard.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Determine script directory and path to compose file +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPOSE_FILE="${SCRIPT_DIR}/../docker/leaderboard-compose.yml" + +echo "==================================================" +echo " DevImpact Leaderboard Worker Deployment" +echo "==================================================" +echo "Compose File: ${COMPOSE_FILE}" + +if [ ! -f "${COMPOSE_FILE}" ]; then + echo "Error: Compose file not found at ${COMPOSE_FILE}" >&2 + exit 1 +fi + +echo "[1/3] Pulling latest GHCR image..." +docker compose -f "${COMPOSE_FILE}" pull + +# Check if worker container is running and an active calculation is in progress +CONTAINER_NAME="devimpact-leaderboard-cron" +if docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then + echo "[2/3] Checking for active leaderboard calculation..." + while docker exec "${CONTAINER_NAME}" pgrep -f "calculate-next-country" > /dev/null 2>&1 || \ + docker exec "${CONTAINER_NAME}" sh -c 'ps aux | grep -v grep | grep -q "calculate-next-country"'; do + echo " >> A leaderboard calculation job is currently running. Waiting for it to finish..." + sleep 10 + done + echo " >> No active calculation running (or active calculation completed)." +else + echo "[2/3] Worker container is not running yet." +fi + +echo "[3/3] Recreating leaderboard worker container with new image..." +docker compose -f "${COMPOSE_FILE}" up -d --remove-orphans + +echo "==================================================" +echo " Leaderboard Worker Deployed Successfully!" +echo "==================================================" diff --git a/ops/docker/.dockerignore b/ops/docker/.dockerignore new file mode 100644 index 0000000..001686d --- /dev/null +++ b/ops/docker/.dockerignore @@ -0,0 +1,23 @@ +.git +.github +.nx +node_modules +**/node_modules +.next +coverage +dist +**/dist +.pnpm-store +*.log +logs.txt +npm-debug.log* +yarn-debug.log* +yarn-error.log* +.env +.env.* +!.env.example +*.tsbuildinfo +tmp +temp +pnpm-lock.yaml.prev +ops/deploy diff --git a/ops/docker/Dockerfile.web b/ops/docker/Dockerfile.web new file mode 100644 index 0000000..72fe08d --- /dev/null +++ b/ops/docker/Dockerfile.web @@ -0,0 +1,28 @@ +FROM node:24-alpine + +WORKDIR /app + +# Enable pnpm via corepack +RUN corepack enable + +# Copy dependency manifests first for layer caching +COPY package.json pnpm-lock.yaml ./ + +# Configure workspace policy for native dependencies if required +RUN printf "packages:\n - '.'\n\nallowBuilds:\n esbuild: true\n sharp: true\n unrs-resolver: true\n" > pnpm-workspace.yaml + +# Install dependencies using frozen lockfile +RUN pnpm install --frozen-lockfile + +# Copy application source code +COPY . . + +# Build Next.js application +RUN pnpm build + +EXPOSE 3000 + +ENV PORT=3000 +ENV NODE_ENV=production + +CMD ["pnpm", "start"] diff --git a/ops/docker/Dockerfile.worker b/ops/docker/Dockerfile.worker new file mode 100644 index 0000000..cc957c0 --- /dev/null +++ b/ops/docker/Dockerfile.worker @@ -0,0 +1,37 @@ +FROM node:24-alpine + +WORKDIR /app + +# Enable pnpm via corepack +RUN corepack enable + +# Copy dependency manifests first for layer caching +COPY package.json pnpm-lock.yaml ./ + +# Configure workspace policy for native dependencies if required +RUN printf "packages:\n - '.'\n\nallowBuilds:\n esbuild: true\n sharp: true\n unrs-resolver: true\n" > pnpm-workspace.yaml + +# Install dependencies using frozen lockfile +RUN pnpm install --frozen-lockfile + +# Copy application source code +COPY . . + +# Install Supercronic cron runner +RUN wget -O /usr/local/bin/supercronic \ + https://github.com/aptible/supercronic/releases/download/v0.2.29/supercronic-linux-amd64 \ + && chmod +x /usr/local/bin/supercronic + +# Copy entrypoint script and initial leaderboard cron job definition +COPY ops/docker/entrypoint.sh /usr/local/bin/entrypoint.sh +COPY ops/cron/leaderboard.cron /etc/leaderboard.cron +RUN chmod +x /usr/local/bin/entrypoint.sh + +# Build-time argument for version identification +ARG GIT_COMMIT_SHA=development +ENV DEVIMPACT_VERSION=${GIT_COMMIT_SHA} + +# Default cron schedule environment variable (daily execution at midnight) +ENV LEADERBOARD_CRON_SCHEDULE="0 0 * * *" + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/docker-compose.yml b/ops/docker/docker-compose.yml similarity index 68% rename from docker-compose.yml rename to ops/docker/docker-compose.yml index 45588fb..b35b5b4 100644 --- a/docker-compose.yml +++ b/ops/docker/docker-compose.yml @@ -3,25 +3,20 @@ services: image: postgres:16-alpine container_name: devimpact-postgres restart: unless-stopped - environment: POSTGRES_DB: devimpact POSTGRES_USER: devimpact POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-devimpact} - ports: - "5432:5432" - volumes: - postgres-data:/var/lib/postgresql/data - healthcheck: test: ["CMD-SHELL", "pg_isready -U devimpact -d devimpact"] interval: 10s timeout: 5s retries: 5 start_period: 30s - logging: driver: json-file options: @@ -32,7 +27,6 @@ services: image: redis:7-alpine container_name: devimpact-redis restart: unless-stopped - command: [ "redis-server", @@ -45,13 +39,10 @@ services: "--maxmemory-policy", "allkeys-lru" ] - ports: - "6379:6379" - volumes: - redis-data:/data - healthcheck: test: [ @@ -65,13 +56,33 @@ services: timeout: 3s retries: 5 start_period: 10s - logging: driver: json-file options: max-size: "10m" max-file: "5" + web: + build: + context: ../.. + dockerfile: ops/docker/Dockerfile.web + container_name: devimpact-web + restart: unless-stopped + ports: + - "3000:3000" + env_file: + - path: ${ENV_FILE:-../../.env} + required: false + environment: + DATABASE_URL: ${DATABASE_URL:-postgresql://devimpact:devimpact@postgres:5432/devimpact?sslmode=disable} + REDIS_URL: ${REDIS_URL:-redis://redis:6379} + REDIS_ENABLED: ${REDIS_ENABLED:-false} + REDIS_PASSWORD: ${REDIS_PASSWORD:-} + GITHUB_TOKEN: ${GITHUB_TOKEN:-} + depends_on: + postgres: + condition: service_healthy + volumes: postgres-data: - redis-data: \ No newline at end of file + redis-data: diff --git a/ops/docker/entrypoint.sh b/ops/docker/entrypoint.sh new file mode 100644 index 0000000..fe01ff4 --- /dev/null +++ b/ops/docker/entrypoint.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -e + +# Default to daily execution at midnight if LEADERBOARD_CRON_SCHEDULE is not set +CRON_EXPR="${LEADERBOARD_CRON_SCHEDULE:-0 0 * * *}" + +echo "Configuring leaderboard cron schedule: ${CRON_EXPR}" +echo "${CRON_EXPR} cd /app && pnpm leaderboard:calculate >> /proc/1/fd/1 2>> /proc/1/fd/2" > /etc/leaderboard.cron + +# Execute supercronic +exec supercronic /etc/leaderboard.cron diff --git a/ops/docker/leaderboard-compose.yml b/ops/docker/leaderboard-compose.yml new file mode 100644 index 0000000..5d005bf --- /dev/null +++ b/ops/docker/leaderboard-compose.yml @@ -0,0 +1,28 @@ +services: + leaderboard-cron: + image: ${LEADERBOARD_IMAGE:-ghcr.io/o2sa/devimpact-leaderboard:latest} + container_name: devimpact-leaderboard-cron + restart: unless-stopped + stop_grace_period: 10m + env_file: + - path: ${ENV_FILE:-../../.env} + required: false + environment: + GITHUB_TOKEN: ${GITHUB_TOKEN} + DATABASE_URL: ${DATABASE_URL} + REDIS_URL: ${REDIS_URL} + REDIS_ENABLED: ${REDIS_ENABLED:-false} + REDIS_PASSWORD: ${REDIS_PASSWORD} + REDIS_CACHE_NAMESPACE: ${REDIS_CACHE_NAMESPACE:-devimpact:v1} + REDIS_CACHE_TTL_SECONDS: ${REDIS_CACHE_TTL_SECONDS:-604800} + REDIS_CONNECT_TIMEOUT_MS: ${REDIS_CONNECT_TIMEOUT_MS:-1500} + LEADERBOARD_CRON_SCHEDULE: ${LEADERBOARD_CRON_SCHEDULE:-0 0 * * *} + LEADERBOARD_SOURCE_URL_TEMPLATE: ${LEADERBOARD_SOURCE_URL_TEMPLATE:-https://raw.githubusercontent.com/ashkulz/committers.top/gh-pages/_data/locations/{country}.yml} + LEADERBOARD_SEED_LIMIT: ${LEADERBOARD_SEED_LIMIT:-256} + LEADERBOARD_REFRESH_LIMIT: ${LEADERBOARD_REFRESH_LIMIT:-500} + LEADERBOARD_USER_STALE_DAYS: ${LEADERBOARD_USER_STALE_DAYS:-30} + GITHUB_USER_STALE_DAYS: ${GITHUB_USER_STALE_DAYS:-14} + GITHUB_REPO_COUNT: ${GITHUB_REPO_COUNT:-30} + GITHUB_PR_COUNT: ${GITHUB_PR_COUNT:-80} + GITHUB_ISSUE_COUNT: ${GITHUB_ISSUE_COUNT:-20} + GITHUB_DISCUSSION_COUNT: ${GITHUB_DISCUSSION_COUNT:-10} diff --git a/package.json b/package.json index e34c93a..e0a21d4 100644 --- a/package.json +++ b/package.json @@ -9,9 +9,9 @@ "lint": "eslint .", "test": "vitest", "test:watch": "vitest --watch", - "redis:up": "docker compose up -d redis", - "redis:down": "docker compose stop redis", - "db:up": "docker compose up -d postgres", + "redis:up": "docker compose -f ops/docker/docker-compose.yml up -d redis", + "redis:down": "docker compose -f ops/docker/docker-compose.yml stop redis", + "db:up": "docker compose -f ops/docker/docker-compose.yml up -d postgres", "db:init": "tsx scripts/init-db.ts", "db:migrate": "tsx scripts/init-db.ts", "leaderboard:calculate": "tsx scripts/calculate-next-country.ts", diff --git a/scripts/calculate-next-country.ts b/scripts/calculate-next-country.ts index d97a399..b328fd7 100644 --- a/scripts/calculate-next-country.ts +++ b/scripts/calculate-next-country.ts @@ -23,10 +23,22 @@ import { getDatabaseStore } from "@/lib/db-store"; import { calculateLeaderboard } from "@/lib/calculate-leaderboard"; import { logger } from "@/lib/logger"; +let activeCountrySlug: string | null = null; +let isCalculating = false; + +const handleShutdownSignal = (signal: string) => { + logger.warn(`Received ${signal}. Graceful shutdown initiated. Waiting for active calculation (${activeCountrySlug ?? "none"}) to finish...`); +}; + +process.on("SIGTERM", () => handleShutdownSignal("SIGTERM")); +process.on("SIGINT", () => handleShutdownSignal("SIGINT")); + async function main() { const overallStartTime = performance.now(); + const workerVersion = process.env.DEVIMPACT_VERSION || process.env.GIT_COMMIT_SHA || "development"; logger.info("=== DevImpact Leaderboard Calculator Start ==="); - logger.info(`DB: ${(process.env.DATABASE_URL ?? "").slice(0, 40)}...`); + logger.info(`Version: ${workerVersion}`); + logger.info(`DB: ${(process.env.DATABASE_URL ?? "").slice(0, 40)}...`); const db = getDatabaseStore(); await db.initializeSchema(); @@ -40,6 +52,9 @@ async function main() { process.exit(0); } + activeCountrySlug = next.slug; + isCalculating = true; + logger.info(`Selected: ${next.title} (${next.slug})`); // 2. Mark as running