From cfb92868e4c3fa677c87e0fb23ffb822c78b3505 Mon Sep 17 00:00:00 2001 From: Alina Banerjee Date: Fri, 12 Jun 2026 10:49:52 +0530 Subject: [PATCH 1/3] Add Dockerfile --- Dockerfile | 236 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 236 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..2e5dfe17 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,236 @@ +# ============================================================ +# CoRE Stack Backend — Base Image +# Base: Ubuntu 24.04 LTS (Noble Numbat) +# +# Provides the OS-level dependencies that install.sh cannot +# easily handle itself: +# - PostgreSQL +# - Erlang 27 (compiled from source — Ubuntu 24.04 ships +# Erlang 24 which is too old for RabbitMQ 4.x) +# - RabbitMQ 4.3.1 +# - Miniconda + corestack-backend conda environment +# +# After running the container, complete setup with: +# bash installation/install.sh \ +# --skip unzip_install,miniconda,rabbitmq,conda_env,geoserver +# ============================================================ + +FROM --platform=linux/arm64 ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=UTC + +# ── 1. Base system tools ────────────────────────────────────────────────────── +RUN apt-get update && apt-get install -y \ + git wget curl build-essential libpq-dev unzip \ + sudo ca-certificates gnupg lsb-release \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# ── 2. PostgreSQL ───────────────────────────────────────────────────────────── +RUN apt-get update && apt-get install -y \ + postgresql postgresql-contrib \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# ── 3. Erlang build dependencies ───────────────────────────────────────────── +RUN apt-get update && apt-get install -y \ + autoconf m4 libssl-dev libncurses-dev socat logrotate \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# ── 4. Erlang OTP 27.3.4.8 — compiled from source ──────────────────────────── +# Ubuntu 24.04 ships Erlang 24 which is too old for RabbitMQ 4.x. +ARG OTP_VERSION=27.3.4.8 +RUN curl -fsSL \ + https://github.com/erlang/otp/releases/download/OTP-${OTP_VERSION}/otp_src_${OTP_VERSION}.tar.gz \ + -o /tmp/otp_src.tar.gz + +RUN tar -zxf /tmp/otp_src.tar.gz -C /tmp && rm /tmp/otp_src.tar.gz + +WORKDIR /tmp/otp_src_${OTP_VERSION} + +RUN export ERL_TOP=$(pwd) && ./otp_build autoconf + +RUN export ERL_TOP=$(pwd) && ./configure \ + --without-javac \ + --without-wx \ + --without-odbc + +RUN make -j$(nproc) + +RUN make install && rm -rf /tmp/otp_src_${OTP_VERSION} + +# ── 5. RabbitMQ 4.3.1 ──────────────────────────────────────────────────────── +WORKDIR / + +RUN curl -fsSL \ + https://github.com/rabbitmq/rabbitmq-server/releases/download/v4.3.1/rabbitmq-server_4.3.1-1_all.deb \ + -o /tmp/rabbitmq.deb \ + && dpkg -i --force-depends /tmp/rabbitmq.deb \ + && rm /tmp/rabbitmq.deb + +# Tell RabbitMQ where the source-built Erlang lives +RUN mkdir -p /etc/rabbitmq \ + && echo 'ERLANG_HOME=/usr/local' > /etc/rabbitmq/rabbitmq-env.conf \ + && echo 'PATH=/usr/local/bin:/usr/bin:/bin' >> /etc/rabbitmq/rabbitmq-env.conf \ + && echo 'deprecated_features.permit.transient_nonexcl_queues = true' > /etc/rabbitmq/rabbitmq.conf + +# ── 6. Miniconda ────────────────────────────────────────────────────────────── +ENV CONDA_DIR=/opt/conda +ENV PATH="${CONDA_DIR}/bin:${PATH}" + +RUN ARCH=$(uname -m) \ + && wget -q \ + "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-${ARCH}.sh" \ + -O /tmp/miniconda.sh \ + && bash /tmp/miniconda.sh -b -p ${CONDA_DIR} \ + && rm /tmp/miniconda.sh + +RUN conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main || true \ + && conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r || true \ + && conda config --set channel_priority flexible \ + && conda clean -afy + +# ── 7. Clone repo and create conda environment ──────────────────────────────── +WORKDIR /opt +RUN git clone https://github.com/core-stack-org/core-stack-backend.git corestack +WORKDIR /opt/corestack + +RUN conda env create -f installation/environment.yml \ + && conda clean -afy + +# ── 8. Fix missing/broken packages ─────────────────────────────────────────── +SHELL ["/opt/conda/bin/conda", "run", "-n", "corestack-backend", "/bin/bash", "-c"] + +RUN pip install pyogrio "setuptools<81" psycopg2-binary + +# ── 9. Create required runtime data directories ─────────────────────────────── +RUN mkdir -p \ + /opt/corestack/data/fc_to_shape \ + /opt/corestack/data/admin-boundary/input \ + /opt/corestack/data/admin-boundary/output \ + /opt/corestack/data/excel_files \ + /opt/corestack/data/tmp \ + /opt/corestack/bot_interface/whatsapp_media \ + /opt/corestack/data/activated_locations && \ + echo '[]' > /opt/corestack/data/activated_locations/active_locations.json +RUN chmod -R 755 /opt/conda/envs/corestack-backend/share/proj + +# ── 10. Patch api.py — guard os.makedirs against empty WHATSAPP_MEDIA_PATH ─── +RUN sed -i \ + 's|os.makedirs(WHATSAPP_MEDIA_PATH, exist_ok=True)|if WHATSAPP_MEDIA_PATH: os.makedirs(WHATSAPP_MEDIA_PATH, exist_ok=True)|' \ + /opt/corestack/bot_interface/api.py + +# Disable automatic GeoServer style assignment — styles can be applied +# manually via the GeoServer UI. The auto-assignment fails if the named +# style doesn't exist, causing a 500 error that aborts the whole task. +RUN sed -i \ + 's| if style_name:| if False: # style disabled — apply via GeoServer UI|' \ + /opt/corestack/utilities/gee_utils.py + +# ── 11. Patch computing/tasks.py — register all Celery tasks ───────────────── +RUN cat > /opt/corestack/computing/tasks.py << 'TASKS' +from computing.STAC_specs.stac_collection import generate_stac_collection_task +import computing.change_detection.change_detection +import computing.change_detection.change_detection_vector +import computing.clart.clart +import computing.clart.drainage_density +import computing.clart.fes_clart_to_geoserver +import computing.clart.lithology +import computing.crop_grid.crop_grid +import computing.cropping_intensity.cropping_intensity +import computing.drought.drought +import computing.drought.drought_causality +import computing.layer_dependency.layer_generation_in_order +import computing.lulc.lulc_v3 +import computing.lulc.lulc_vector +import computing.lulc.river_basin_lulc.lulc_v2_river_basin +import computing.lulc.river_basin_lulc.lulc_v3_river_basin_using_v2 +import computing.lulc.tehsil_level.lulc_v2 +import computing.lulc.tehsil_level.lulc_v3 +import computing.lulc.v4.lulc_v4 +import computing.lulc_X_terrain.lulc_on_plain_cluster +import computing.lulc_X_terrain.lulc_on_slope_cluster +import computing.misc.admin_boundary +import computing.misc.agroecological_space +import computing.misc.antyodaya +import computing.misc.aquifer_vector +import computing.misc.canal_layer +import computing.misc.catchment_area +import computing.misc.digital_elevation_model +import computing.misc.distancetonearestdrainage +import computing.misc.drainage_lines +import computing.misc.facilities_proximity +import computing.misc.factory_csr +import computing.misc.green_credit +import computing.misc.lcw_conflict +import computing.misc.mining_data +import computing.misc.naturaldepression +import computing.misc.ndvi_time_series +import computing.misc.nrega +import computing.misc.restoration_opportunity +import computing.misc.slope_percentage +import computing.misc.soge_vector +import computing.misc.stream_order +import computing.mws.generate_hydrology +import computing.mws.mws +import computing.mws.mws_centroid +import computing.mws.mws_connectivity +import computing.plantation.site_suitability +import computing.surface_water_bodies.merge_swb_ponds +import computing.surface_water_bodies.swb +import computing.terrain_descriptor.terrain_clusters +import computing.terrain_descriptor.terrain_raster +import computing.terrain_descriptor.terrain_raster_fabdem +import computing.tree_health.canopy_height +import computing.tree_health.canopy_height_vector +import computing.tree_health.ccd +import computing.tree_health.ccd_vector +import computing.tree_health.overall_change +import computing.tree_health.overall_change_vector +import computing.zoi_layers.zoi +__all__ = ['generate_stac_collection_task'] +TASKS + +# ── 12. Copy patched install.sh ─────────────────────────────────────────────── +# Copies the patched install-docker.sh from installation/ in the repo root. +# Place install-docker.sh at installation/install-docker.sh in your local clone. +# This is a patched version of install.sh with Docker-specific fixes — +# the original install.sh in the repo is left untouched. +ARG INSTALL_CACHE_BUST=1 +COPY installation/install-docker.sh /opt/corestack/installation/install-docker.sh +RUN chmod +x /opt/corestack/installation/install-docker.sh + +# ── 13. Startup script ──────────────────────────────────────────────────────── +RUN cat > /start.sh << 'STARTEOF' +#!/bin/bash +service postgresql start +service rabbitmq-server start + +until su -c "pg_isready -q" postgres; do + sleep 1 +done + +# Create DB user and database matching install.sh defaults +su -c "psql -c \"CREATE USER corestack_admin WITH PASSWORD 'corestack@123' SUPERUSER;\" " postgres 2>/dev/null || true +su -c "psql -c \"CREATE DATABASE corestack_db OWNER corestack_admin;\" " postgres 2>/dev/null || true + +echo "" +echo "============================================================" +echo " CoRE Stack container ready." +echo "" +echo " Complete setup by running:" +echo " bash installation/install-docker.sh \\" +echo " --skip unzip_install,miniconda,rabbitmq,conda_env,geoserver" +echo "============================================================" +echo "" +exec "$@" +STARTEOF +RUN chmod +x /start.sh + +# ── 14. Environment ─────────────────────────────────────────────────────────── +ENV PROJ_DATA=/opt/conda/envs/corestack-backend/share/proj +ENV PROJ_LIB=/opt/conda/envs/corestack-backend/share/proj + +EXPOSE 8000 +WORKDIR /opt/corestack +ENTRYPOINT ["/start.sh"] +CMD ["bash"] From 2b780a58db1a37ddbaa033c0a762469058bb8d5d Mon Sep 17 00:00:00 2001 From: Alina Banerjee Date: Fri, 12 Jun 2026 10:50:09 +0530 Subject: [PATCH 2/3] Add a complete set of instructions for running an LULC pipeline --- run-lulc-docker-instructions.md | 408 ++++++++++++++++++++++++++++++++ 1 file changed, 408 insertions(+) create mode 100644 run-lulc-docker-instructions.md diff --git a/run-lulc-docker-instructions.md b/run-lulc-docker-instructions.md new file mode 100644 index 00000000..5edb484a --- /dev/null +++ b/run-lulc-docker-instructions.md @@ -0,0 +1,408 @@ +# CoRE Stack — LULC Pipeline Runbook + +Complete steps to build, set up and run a LULC pipeline from scratch. + +--- + +## Prerequisites + +- Docker installed and running +- `service-account.json` GEE key file on your host machine +- GCS bucket created (e.g. `fpl-core-stack-dev`) in GCP project `arcane-mason-493503-a6` +- Local clone of `core-stack-backend` repo with: + - `Dockerfile` placed at the repo root + - `install-docker.sh` placed at `installation/install-docker.sh` + +``` +core-stack-backend/ ← your local clone / build context +├── Dockerfile ← add this +├── installation/ +│ └── install-docker.sh ← replace with patched version +└── ...rest of repo +``` + +--- + +## Part 1 — Build and Start the Container + +```bash +# 0. Enter your local repo clone +cd core-stack-backend + +# 1. Build the image (only needed once or after Dockerfile changes) +# add `--no-cache` to create one without any cached layers +# add `--platform linux/arm64` to build for ARM64 architecture +docker build -t corestack . + +# 2. Create a shared network for core-stack and geoserver +docker network create corestack-network 2>/dev/null || true + +# 3. Start GeoServer +# add `--platform linux/arm64` to run on ARM64 architecture +docker run -dit \ + --name geoserver \ + --network corestack-network \ + -p 8080:8080 \ + docker.osgeo.org/geoserver:2.28.0 + +# 4. Start the CoRE Stack container +docker run -it \ + --name core-stack \ + --network corestack-network \ + -p 9001:8000 \ + -v /path/to/service-account.json:/opt/gee-keys/service-account.json \ + corestack +``` + +> After step 4 you will be inside the container at `/opt/corestack`. + +--- + +## Part 2 — Run install-docker.sh (inside the container) + +Run each step in order. If a step was already completed in a previous run, +`install-docker.sh` will skip it automatically. + +```bash +# Step 1 — PostgreSQL setup +CONDA_ENV_NAME=corestack-backend bash installation/install-docker.sh --only postgres + +# Step 2 — Generate .env file +CONDA_ENV_NAME=corestack-backend bash installation/install-docker.sh --only env_file + +# Step 3 — Fix $BACKEND_DIR literal in .env +sed -i 's|\$BACKEND_DIR|/opt/corestack|g' /opt/corestack/nrm_app/.env + +# Step 4 — Set credentials in .env +sed -i 's|GCS_BUCKET_NAME=""|GCS_BUCKET_NAME="fpl-core-stack-dev"|g' nrm_app/.env +sed -i 's|S3_BUCKET=""|S3_BUCKET="fpl-core-stack-dev"|g' nrm_app/.env +sed -i 's|DPR_S3_BUCKET=""|DPR_S3_BUCKET="fpl-core-stack-dev"|g' nrm_app/.env +sed -i 's|GEE_STORAGE_PROJECT=""|GEE_STORAGE_PROJECT="arcane-mason-493503-a6"|g' nrm_app/.env +sed -i 's|GEE_STORAGE_PROJECT_HELPER=""|GEE_STORAGE_PROJECT_HELPER="arcane-mason-493503-a6"|g' nrm_app/.env +sed -i 's|GEOSERVER_URL=""|GEOSERVER_URL="http://geoserver:8080/geoserver"|g' nrm_app/.env +sed -i 's|GEOSERVER_USERNAME=""|GEOSERVER_USERNAME="admin"|g' nrm_app/.env +sed -i 's|GEOSERVER_PASSWORD=""|GEOSERVER_PASSWORD="geoserver"|g' nrm_app/.env + +# Step 5 — Django migrations + seed data + superuser +CONDA_ENV_NAME=corestack-backend bash installation/install-docker.sh \ + --only django_migrations,seed_data,superuser + +# Step 6 — GEE configuration +CONDA_ENV_NAME=corestack-backend bash installation/install-docker.sh \ + --only gee_configuration \ + --gee-json /opt/gee-keys/service-account.json + +# Step 7 — Admin boundary data (~8GB, takes a while) +CONDA_ENV_NAME=corestack-backend bash installation/install-docker.sh \ + --only admin_boundary_data +``` + +> After Step 5 note your superuser credentials — install-docker.sh prints: +> `Installer test superuser: username=test_user_XXXX password=test_change_me` + +--- + +## Part 3 — Set Up GeoServer Workspaces + +Run these from your **host machine** (not inside the container): + +```bash +# Create all required workspaces +for workspace in mws panchayat_boundaries customkml ndvi_timeseries \ + nrega_assets plantation swb crop_grid_layers ne test_workspace; do + STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -X POST http://localhost:8080/geoserver/rest/workspaces \ + -H "Content-Type: application/json" \ + -u admin:geoserver \ + -d "{\"workspace\": {\"name\": \"$workspace\"}}") + if [ "$STATUS" = "201" ]; then + echo "Created: $workspace" + elif [ "$STATUS" = "409" ]; then + echo "Already exists: $workspace" + else + echo "Warning ($STATUS): $workspace" + fi +done +``` + +--- + +## Part 4 — Start Django and Celery + +Open **two terminal windows**. In each, exec into the container: + +**Terminal 1 — Django:** +```bash +docker exec -it core-stack bash -c " + source /opt/conda/etc/profile.d/conda.sh && \ + conda activate corestack-backend && \ + cd /opt/corestack && \ + python manage.py runserver 0.0.0.0:8000 +" +``` + +**Terminal 2 — Celery:** +```bash +docker exec -it core-stack bash -c " + source /opt/conda/etc/profile.d/conda.sh && \ + conda activate corestack-backend && \ + cd /opt/corestack && \ + celery -A nrm_app worker -l info -Q nrm --pool solo +" +``` + +--- + +## Part 5 — Run the LULC Pipeline + +Run these from your **host machine**. + +**Get a token:** +```bash +TOKEN=$(curl -s -X POST http://localhost:9001/api/v1/auth/login/ \ + -H "Content-Type: application/json" \ + -d '{"username":"test_user_0944","password":"test_change_me"}' \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['access'])") +echo $TOKEN +``` + +** COPY echoed value into $TOKEN in the following commands** +eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNzg4OTUyNzk1LCJpYXQiOjE3ODExNzY3OTUsImp0aSI6ImE1YTJlMWYxNGIyYzQ3YjJiNjBkMTQ5MDk0YWYwZjgxIiwidXNlcl9pZCI6Mn0.GICG9iyoDd2D5xXlfJLld_wT4ech7QPoMS13zCmvehg + +**Step 1 — Admin boundary** (wait for `succeeded` in Celery before next step): +```bash +curl -s -X POST http://localhost:9001/api/v1/generate_block_layer/ \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $TOKEN" \ + -d '{ + "state": "karnataka", + "district": "hassan", + "block": "hassan", + "gee_account_id": 1 + }' +``` +Output should be: ```{"Success":"Successfully initiated"}``` +Wait until the terminal window with Celery shows a status such as: + +``` +[2026-06-11 16:52:44,979: INFO/MainProcess] Task computing.misc.admin_boundary.generate_tehsil_shape_file_data[9fe9f019-a775-491f-8f8a-9ef728a6fa79] succeeded in 31.10820138899726s: True +``` + +**Step 2 — MWS layer** (wait for `succeeded` before next step): +```bash +curl -s -X POST http://localhost:9001/api/v1/generate_mws_layer/ \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $TOKEN" \ + -d '{ + "state": "karnataka", + "district": "hassan", + "block": "hassan", + "gee_account_id": 1 + }' +``` +Output should be: ```{"Success":"Successfully initiated"}%``` +Wait until the terminal window with Celery show a status such as: +``` +[2026-06-11 16:57:47,249: INFO/MainProcess] Task computing.mws.mws.mws_layer[796d4f16-74c6-4885-beba-3d84919f51bf] succeeded in 7.108237585998722s: True +``` + +**Step 3 — LULC** (takes 10–30 minutes): +```bash +curl -s -X POST http://localhost:9001/api/v1/lulc_v3/ \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $TOKEN" \ + -d '{ + "state": "karnataka", + "district": "hassan", + "block": "hassan", + "start_year": 2017, + "end_year": 2024, + "gee_account_id": 1 + }' +``` +Output should be: ```{"Success":"LULC v3 task initiated"}%``` + +**Watch Celery progress:** +```bash +docker exec core-stack tail -f /var/log/corestack/celery.log +``` + +--- +## Part 6 — View the LULC Layer + +**Apply colormap style to GeoServer:** (or see alternate way): +```bash +curl -s -X POST http://localhost:8080/geoserver/rest/styles \ + -H "Content-Type: application/vnd.ogc.sld+xml" \ + -u admin:geoserver \ + -d ' + + + lulc_style + + LULC Style + + + + + + + + + + + + + + + + + + + + + + +' + +# Apply style to the LULC layer +curl -s -X PUT \ + "http://localhost:8080/geoserver/rest/layers/ne:LULC_17_18_hassan_hassan_level_1" \ + -H "Content-Type: application/json" \ + -u admin:geoserver \ + -d '{"layer": {"defaultStyle": {"name": "lulc_style"}}}' +``` +====== +The styling can also be added at:http://localhost:8080/geoserver/web/wicket/bookmarkable/org.geoserver.wms.web.data.StyleNewPage?12 with any name (e.g. lulc_style): + +``` + + + + lulc_style + + LULC Style + + + + + + + + + + + + + + + + + + + + + + + +``` + + + +**View in browser:** +``` +http://localhost:8080/geoserver/wms?service=WMS&version=1.1.1&request=GetMap&layers=ne:LULC_17_18_hassan_hassan_level_1&bbox=75.8,12.8,76.5,13.5&width=800&height=600&srs=EPSG:4326&format=image/png +``` + +Or via GeoServer Layer Preview: +``` +http://localhost:8080/geoserver/web/ +``` +→ Layer Preview → `ne:LULC_17_18_hassan_hassan_level_1` → OpenLayers + +--- + +## Quick Reference + +| Service | URL | Credentials | +|--------------|------------------------------------------|---------------------| +| Django admin | http://localhost:9001/admin/ | test_user_0944 / test_change_me | +| Swagger API | http://localhost:9001/swagger/ | (log in via admin first) | +| GeoServer | http://localhost:8080/geoserver/web/ | admin / geoserver | + +--- + +## Troubleshooting + +**Container already exists:** +```bash +docker stop core-stack && docker rm core-stack +docker stop geoserver && docker rm geoserver +``` + +**RabbitMQ not running:** +```bash +docker exec core-stack service rabbitmq-server start +``` + +**PostgreSQL not running:** +```bash +docker exec core-stack service postgresql start +``` + +**GeoServer namespace null error:** +```bash +# Re-create all workspaces +for workspace in mws panchayat_boundaries customkml ndvi_timeseries \ + nrega_assets plantation swb crop_grid_layers ne test_workspace; do + curl -s -X POST http://localhost:8080/geoserver/rest/workspaces \ + -H "Content-Type: application/json" \ + -u admin:geoserver \ + -d "{\"workspace\": {\"name\": \"$workspace\"}}" +done +``` + +**Token expired:** +```bash +TOKEN=$(curl -s -X POST http://localhost:9001/api/v1/auth/login/ \ + -H "Content-Type: application/json" \ + -d '{"username":"test_user_XXXX","password":"test_change_me"}' \ + | python3 -c "import sys,json; print(json.load(sys.stdin)['access'])") +``` + +**Check Celery logs:** +```bash +docker exec core-stack tail -50 /var/log/corestack/celery.log +``` + +** If old GEE assets remain ** +```bash +docker exec -it core-stack bash -c " + source /opt/conda/etc/profile.d/conda.sh && \ + conda activate corestack-backend && \ + cd /opt/corestack && \ + python manage.py shell -c \" +import ee +from utilities.gee_utils import ee_initialize +ee_initialize(account_id=1) + +base = 'projects/arcane-mason-493503-a6/assets/apps/mws/// +assets = ee.data.listAssets({'parent': base}) +for a in assets.get('assets', []): + asset_name = a['name'] + if '_old' in asset_name: + print('Deleting:', asset_name) + ee.data.deleteAsset(asset_name) + print('Deleted.') +\" +" \ No newline at end of file From 5b480657bf10d56d4d7644db7cfe6c6b3b5277f8 Mon Sep 17 00:00:00 2001 From: Alina Banerjee Date: Fri, 12 Jun 2026 10:50:38 +0530 Subject: [PATCH 3/3] Add install script used only by Docker (install.sh remains for local installs) --- installation/install-docker.sh | 1815 ++++++++++++++++++++++++++++++++ 1 file changed, 1815 insertions(+) create mode 100644 installation/install-docker.sh diff --git a/installation/install-docker.sh b/installation/install-docker.sh new file mode 100644 index 00000000..f08d7772 --- /dev/null +++ b/installation/install-docker.sh @@ -0,0 +1,1815 @@ +#!/bin/bash +# CoRE Stack backend installer. + +set -euo pipefail + +# === CONFIGURATION === +MINICONDA_DIR="${MINICONDA_DIR:-$HOME/miniconda3}" +CONDA_ENV_NAME="${CONDA_ENV_NAME:-corestackenv}" +INSTALL_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CONDA_ENV_YAML="$INSTALL_SCRIPT_DIR/environment.yml" +BACKEND_DIR="$(cd "$INSTALL_SCRIPT_DIR/.." && pwd)" +INSTALL_INVOCATION_DIR="$PWD" +POSTGRES_USER="corestack_admin" +POSTGRES_DB="corestack_db" +POSTGRES_PASSWORD="corestack@123" +SHELL_RC="$HOME/.bashrc" +INSTALL_STATE_DIR="$BACKEND_DIR/.installation_state" +APP_ENV_FILE="$BACKEND_DIR/nrm_app/.env" +LEGACY_ROOT_ENV_FILE="$BACKEND_DIR/.env" +DEFAULT_GEE_ACCOUNT_NAME="local-gee-account" +POST_INSTALL_REQUIRE_GEE=0 +POST_INSTALL_INITIALISATION_FAILED=0 +GEE_JSON_PATH_ARG="" +PUBLIC_API_X_API_KEY_ARG="" +PUBLIC_API_BASE_URL_ARG="" +GEOSERVER_URL_ARG="" +GEOSERVER_USERNAME_ARG="" +GEOSERVER_PASSWORD_ARG="" +STEP_START_FROM="" +LIST_STEPS_ONLY=0 +DEFAULT_PUBLIC_API_BASE_URL="https://geoserver.core-stack.org/api/v1" +DEFAULT_PUBLIC_API_SAMPLE_STATE="assam" +DEFAULT_PUBLIC_API_SAMPLE_DISTRICT="cachar" +DEFAULT_PUBLIC_API_SAMPLE_TEHSIL="lakhipur" +declare -a ONLY_STEPS=() +declare -a SKIP_STEPS=() +declare -a OPTIONAL_INPUT_KEYS=( + "gee_json" + "public_api_key" + "public_api_base_url" + "geoserver_url" + "geoserver_username" + "geoserver_password" +) +declare -A OPTIONAL_INPUT_VALUES=() + +# === ENV FILE CONFIGURATION === +ENV_DB_NAME="$POSTGRES_DB" +ENV_DB_USER="$POSTGRES_USER" +ENV_DB_PASSWORD="$POSTGRES_PASSWORD" +ENV_DEPLOYMENT_DIR="$BACKEND_DIR" +ENV_TMP_LOCATION="$BACKEND_DIR/tmp" + +STEP_ORDER=( + "unzip_install" + "miniconda" + "postgres" + "rabbitmq" + "conda_env" + "env_file" + "geoserver" + "collectstatic" + "django_migrations" + "seed_data" + "superuser" + "gee_configuration" + "admin_boundary_data" + "initialisation_check" + "public_api_check" +) + +function step_label() { + case "$1" in + unzip_install) echo "Install unzip" ;; + miniconda) echo "Install Miniconda" ;; + postgres) echo "Install PostgreSQL" ;; + rabbitmq) echo "Install RabbitMQ" ;; + conda_env) echo "Set up conda environment" ;; + env_file) echo "Generate/update .env" ;; + collectstatic) echo "Collect static files" ;; + django_migrations) echo "Run Django migrations" ;; + seed_data) echo "Load seed data" ;; + superuser) echo "Ensure test superuser" ;; + geoserver) echo "Install GeoServer" ;; + gee_configuration) echo "Configure Google Earth Engine" ;; + admin_boundary_data) echo "Download admin-boundary data" ;; + initialisation_check) echo "Run internal API initialisation check" ;; + public_api_check) echo "Run public API smoke test" ;; + *) echo "$1" ;; + esac +} + +function print_usage() { + cat < "$(step_marker_path "$step_name")" +} + +function ensure_conda() { + if command -v conda >/dev/null 2>&1; then + MINICONDA_DIR="$(conda info --base)" + return 0 + fi + + if [ -f "$MINICONDA_DIR/etc/profile.d/conda.sh" ]; then + # shellcheck disable=SC1091 + source "$MINICONDA_DIR/etc/profile.d/conda.sh" + fi + + if ! command -v conda >/dev/null 2>&1; then + echo "Conda was not found. Expected it at $MINICONDA_DIR." + exit 1 + fi + MINICONDA_DIR="$(conda info --base)" +} + +function activate_conda_env() { + ensure_conda + export OPENBLAS_NUM_THREADS="${OPENBLAS_NUM_THREADS:-1}" + export OMP_NUM_THREADS="${OMP_NUM_THREADS:-1}" + export MKL_NUM_THREADS="${MKL_NUM_THREADS:-1}" + export NUMEXPR_NUM_THREADS="${NUMEXPR_NUM_THREADS:-1}" + export VECLIB_MAXIMUM_THREADS="${VECLIB_MAXIMUM_THREADS:-1}" + export BLIS_NUM_THREADS="${BLIS_NUM_THREADS:-1}" + export GOTO_NUM_THREADS="${GOTO_NUM_THREADS:-1}" + # shellcheck disable=SC1091 + source "$MINICONDA_DIR/etc/profile.d/conda.sh" + conda activate "$CONDA_ENV_NAME" +} + +function conda_env_exists() { + ensure_conda + conda env list | sed 's/^[* ]*//' | awk '{print $1}' | grep -qx "$CONDA_ENV_NAME" +} + +function gee_configuration_present() { + [ -f "$APP_ENV_FILE" ] && grep -Eq '^GEE_DEFAULT_ACCOUNT_ID="?([0-9]+)' "$APP_ENV_FILE" +} + +function directory_has_contents() { + local directory_path="$1" + [ -d "$directory_path" ] && find "$directory_path" -mindepth 1 -print -quit 2>/dev/null | grep -q . +} + +function admin_boundary_data_present() { + local admin_boundary_dir="$BACKEND_DIR/data/admin-boundary" + [ -f "$admin_boundary_dir/input/soi_tehsil.geojson" ] && \ + find "$admin_boundary_dir/input" -mindepth 2 -name '*.geojson' -print -quit 2>/dev/null | grep -q . +} + +function nested_admin_boundary_data_present() { + local nested_admin_boundary_dir="$BACKEND_DIR/data/admin-boundary/admin-boundary" + [ -f "$nested_admin_boundary_dir/input/soi_tehsil.geojson" ] && \ + find "$nested_admin_boundary_dir/input" -mindepth 2 -name '*.geojson' -print -quit 2>/dev/null | grep -q . +} + +function move_directory_contents() { + local source_dir="$1" + local destination_dir="$2" + local items=() + + mkdir -p "$destination_dir" + shopt -s dotglob nullglob + items=("$source_dir"/*) + if [ "${#items[@]}" -gt 0 ]; then + mv "${items[@]}" "$destination_dir/" + fi + shopt -u dotglob nullglob +} + +function normalize_existing_admin_boundary_data() { + local admin_boundary_dir="$BACKEND_DIR/data/admin-boundary" + local nested_admin_boundary_dir="$admin_boundary_dir/admin-boundary" + + if admin_boundary_data_present; then + mark_step_complete "admin_boundary_data" + return 0 + fi + + if ! nested_admin_boundary_data_present; then + return 1 + fi + + echo "Found admin-boundary data in a nested extracted layout. Normalizing it to $admin_boundary_dir ..." + mkdir -p "$admin_boundary_dir/input" "$admin_boundary_dir/output" + + if [ -d "$nested_admin_boundary_dir/input" ]; then + move_directory_contents "$nested_admin_boundary_dir/input" "$admin_boundary_dir/input" + fi + + if [ -d "$nested_admin_boundary_dir/output" ]; then + move_directory_contents "$nested_admin_boundary_dir/output" "$admin_boundary_dir/output" + fi + + rmdir "$nested_admin_boundary_dir/output" 2>/dev/null || true + rmdir "$nested_admin_boundary_dir/input" 2>/dev/null || true + rmdir "$nested_admin_boundary_dir" 2>/dev/null || true + + if admin_boundary_data_present; then + echo "Admin-boundary data is ready at $admin_boundary_dir" + mark_step_complete "admin_boundary_data" + return 0 + fi + + echo "Admin-boundary data was detected, but automatic normalization did not finish cleanly." + return 1 +} + +function finalize_admin_boundary_extraction() { + local extracted_root="$1" + local admin_boundary_dir="$BACKEND_DIR/data/admin-boundary" + local candidate_dir="" + local first_child="" + + if [ -d "$extracted_root/admin-boundary" ]; then + candidate_dir="$extracted_root/admin-boundary" + elif [ -d "$extracted_root/input" ] || [ -d "$extracted_root/output" ]; then + candidate_dir="$extracted_root" + else + first_child="$(find "$extracted_root" -mindepth 1 -maxdepth 1 -type d | head -n 1)" + if [ -n "$first_child" ] && { [ -d "$first_child/input" ] || [ -d "$first_child/output" ]; }; then + candidate_dir="$first_child" + fi + fi + + if [ -z "$candidate_dir" ]; then + echo "Unable to detect the extracted admin-boundary layout under $extracted_root" + return 1 + fi + + rm -rf "$admin_boundary_dir" + mkdir -p "$(dirname "$admin_boundary_dir")" + + if [ "$candidate_dir" = "$extracted_root" ]; then + mkdir -p "$admin_boundary_dir" + move_directory_contents "$candidate_dir" "$admin_boundary_dir" + else + mv "$candidate_dir" "$admin_boundary_dir" + fi + + normalize_existing_admin_boundary_data +} + +function install_miniconda() { + if command -v conda >/dev/null 2>&1; then + MINICONDA_DIR="$(conda info --base)" + echo "Conda already available ($(conda --version)) at $MINICONDA_DIR." + mark_step_complete "miniconda" + return + fi + + if [ -d "$MINICONDA_DIR" ]; then + echo "Miniconda found at $MINICONDA_DIR. Sourcing it..." + # shellcheck disable=SC1091 + source "$MINICONDA_DIR/etc/profile.d/conda.sh" + MINICONDA_DIR="$(conda info --base)" + mark_step_complete "miniconda" + return + fi + + echo "Installing Miniconda..." + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O "$BACKEND_DIR/miniconda.sh" + bash "$BACKEND_DIR/miniconda.sh" -b -p "$MINICONDA_DIR" + rm -f "$BACKEND_DIR/miniconda.sh" + + if ! grep -qs 'miniconda3/etc/profile.d/conda.sh' "$SHELL_RC" 2>/dev/null; then + { + echo "" + echo "# >>> conda initialize >>>" + echo "source \"$MINICONDA_DIR/etc/profile.d/conda.sh\"" + echo "# <<< conda initialize <<<" + } >> "$SHELL_RC" + fi + + # shellcheck disable=SC1091 + source "$MINICONDA_DIR/etc/profile.d/conda.sh" + echo "Miniconda installed." + mark_step_complete "miniconda" +} + +function setup_conda_env() { + local force="${1:-0}" + + ensure_conda + if conda_env_exists && [ "$force" -ne 1 ]; then + echo "Conda environment '$CONDA_ENV_NAME' already exists. Keeping it." + mark_step_complete "conda_env" + return + fi + + echo "Setting up conda environment '$CONDA_ENV_NAME'..." + conda env remove -n "$CONDA_ENV_NAME" -y >/dev/null 2>&1 || true + conda env create -f "$CONDA_ENV_YAML" -n "$CONDA_ENV_NAME" + echo "Conda environment ready." + mark_step_complete "conda_env" +} + +function install_postgres() { + if command -v psql >/dev/null 2>&1; then + echo "PostgreSQL already installed ($(psql --version))." + else + echo "Installing PostgreSQL..." + apt-get update + apt-get install -y postgresql postgresql-contrib postgis libpq-dev + fi + + service postgresql start + + + echo "Setting up PostgreSQL user/database..." + + su -c "psql -c \"CREATE USER $POSTGRES_USER WITH PASSWORD '$POSTGRES_PASSWORD';\" " postgres 2>/dev/null || true + su -c "psql -c \"ALTER USER $POSTGRES_USER WITH PASSWORD '$POSTGRES_PASSWORD';\"" postgres + su -c "psql -c \"CREATE DATABASE $POSTGRES_DB OWNER $POSTGRES_USER;\" " postgres 2>/dev/null || true + su -c "psql -c \"ALTER USER $POSTGRES_USER WITH SUPERUSER;\"" postgres + echo "PostgreSQL ready." + mark_step_complete "postgres" +} + +function install_rabbitmq() { + if command -v rabbitmqctl >/dev/null 2>&1; then + echo "RabbitMQ already installed." + else + echo "Installing RabbitMQ..." + apt-get install -y rabbitmq-server + fi + + service rabbitmq-server start + + echo "RabbitMQ ready." + mark_step_complete "rabbitmq" +} + +GEOSERVER_DEFAULT_URL="http://localhost:8080/geoserver" +GEOSERVER_DEFAULT_USER="admin" +GEOSERVER_DEFAULT_PASS="geoserver" +GEOSERVER_WORKSPACE="corestack" + +function geoserver_deployed() { + [ -d "/opt/tomcat/webapps/geoserver" ] +} + +function tomcat_running() { + curl -sf http://localhost:8080 >/dev/null 2>&1 +} + +function wait_for_geoserver_rest() { + local url="${1:-$GEOSERVER_DEFAULT_URL}" + local user="${2:-$GEOSERVER_DEFAULT_USER}" + local pass="${3:-$GEOSERVER_DEFAULT_PASS}" + local elapsed=0 + local timeout=120 + + echo "Waiting for GeoServer REST API to be ready (up to ${timeout}s)..." + while [ "$elapsed" -lt "$timeout" ]; do + local http_code + http_code=$(curl -s -o /dev/null -w "%{http_code}" \ + -u "${user}:${pass}" \ + "${url}/rest/workspaces.json" 2>/dev/null) + if [ "$http_code" = "200" ]; then + echo "GeoServer REST API is ready." + return 0 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + + echo "WARNING: GeoServer REST API did not become ready after ${timeout}s." + return 1 +} + +function ensure_geoserver_workspace() { + local workspace="${1:-$GEOSERVER_WORKSPACE}" + local url="${2:-$GEOSERVER_DEFAULT_URL}" + local user="${3:-$GEOSERVER_DEFAULT_USER}" + local pass="${4:-$GEOSERVER_DEFAULT_PASS}" + local http_code + + http_code=$(curl -s -o /dev/null -w "%{http_code}" \ + -u "${user}:${pass}" \ + "${url}/rest/workspaces/${workspace}.json" 2>/dev/null) + + if [ "$http_code" = "200" ]; then + echo "GeoServer workspace '${workspace}' already exists." + return 0 + fi + + echo "Creating GeoServer workspace '${workspace}'..." + http_code=$(curl -s -o /dev/null -w "%{http_code}" \ + -u "${user}:${pass}" \ + -H "Content-Type: application/json" \ + -X POST \ + -d "{\"workspace\": {\"name\": \"${workspace}\"}}" \ + "${url}/rest/workspaces" 2>/dev/null) + + if [ "$http_code" = "201" ]; then + echo "GeoServer workspace '${workspace}' created." + else + echo "WARNING: Failed to create GeoServer workspace '${workspace}' (HTTP ${http_code})." + return 1 + fi +} + +function install_geoserver() { + local tomcat_version="9.0.98" + local geoserver_version="2.23.6" + local tomcat_dir="/opt/tomcat" + local tomcat_tar="/tmp/apache-tomcat-${tomcat_version}.tar.gz" + local geoserver_zip="/tmp/geoserver-${geoserver_version}-war.zip" + local geoserver_war_dir="/tmp/geoserver-war" + + if geoserver_deployed; then + echo "GeoServer already deployed at $tomcat_dir/webapps/geoserver." + if ! tomcat_running; then + echo "Starting Tomcat..." + "$tomcat_dir/bin/startup.sh" + fi + wait_for_geoserver_rest && ensure_geoserver_workspace || true + mark_step_complete "geoserver" + return + fi + + if ! command -v java >/dev/null 2>&1; then + echo "Installing Java..." + apt-get update + apt-get install -y default-jdk + else + echo "Java already installed ($(java -version 2>&1 | head -n1))." + fi + + if ! getent group tomcat >/dev/null 2>&1; then + groupadd tomcat + fi + if ! id -u tomcat >/dev/null 2>&1; then + useradd -s /bin/false -g tomcat -d "$tomcat_dir" tomcat + fi + + if [ ! -f "$tomcat_dir/bin/startup.sh" ]; then + echo "Downloading Tomcat $tomcat_version..." + wget -qL \ + "https://archive.apache.org/dist/tomcat/tomcat-9/v${tomcat_version}/bin/apache-tomcat-${tomcat_version}.tar.gz" \ + -O "$tomcat_tar" + mkdir -p "$tomcat_dir" + tar xzf "$tomcat_tar" -C "$tomcat_dir" --strip-components=1 + rm -f "$tomcat_tar" + echo "Tomcat $tomcat_version installed at $tomcat_dir." + else + echo "Tomcat already installed at $tomcat_dir." + fi + + chgrp -R tomcat "$tomcat_dir" + chmod -R g+r "$tomcat_dir/conf" + chmod g+x "$tomcat_dir/conf" + chown -R tomcat "$tomcat_dir/webapps" "$tomcat_dir/work" "$tomcat_dir/temp" "$tomcat_dir/logs" + + if ! tomcat_running; then + echo "Starting Tomcat..." + "$tomcat_dir/bin/startup.sh" + sleep 5 + fi + + echo "Downloading GeoServer $geoserver_version WAR (~108MB)..." + wget -L \ + "https://sourceforge.net/projects/geoserver/files/GeoServer/${geoserver_version}/geoserver-${geoserver_version}-war.zip/download" \ + -O "$geoserver_zip" + + local zip_size + zip_size=$(stat -c%s "$geoserver_zip" 2>/dev/null || echo 0) + if [ "$zip_size" -lt 10000000 ]; then + echo "ERROR: GeoServer download appears incomplete (${zip_size} bytes). Re-run this step." + rm -f "$geoserver_zip" + return 1 + fi + + apt-get install -y unzip + rm -rf "$geoserver_war_dir" + unzip -q "$geoserver_zip" -d "$geoserver_war_dir" + cp "$geoserver_war_dir/geoserver.war" "$tomcat_dir/webapps/" + rm -rf "$geoserver_zip" "$geoserver_war_dir" + + echo "Waiting for GeoServer to deploy (up to 60s)..." + local elapsed=0 + while [ "$elapsed" -lt 60 ]; do + if geoserver_deployed; then + break + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + + if ! geoserver_deployed; then + echo "WARNING: GeoServer webapps directory not found after ${elapsed}s." + echo "Check deployment logs: tail -20 $tomcat_dir/logs/catalina.out" + return 1 + fi + + if [ -f "$APP_ENV_FILE" ]; then + if [ -z "$(current_env_value "$APP_ENV_FILE" "GEOSERVER_URL")" ]; then + set_env_value "$APP_ENV_FILE" "GEOSERVER_URL" "http://localhost:8080/geoserver/" + fi + if [ -z "$(current_env_value "$APP_ENV_FILE" "GEOSERVER_USERNAME")" ]; then + set_env_value "$APP_ENV_FILE" "GEOSERVER_USERNAME" "admin" + fi + if [ -z "$(current_env_value "$APP_ENV_FILE" "GEOSERVER_PASSWORD")" ]; then + set_env_value "$APP_ENV_FILE" "GEOSERVER_PASSWORD" "geoserver" + fi + echo "GeoServer credentials written to .env." + fi + + wait_for_geoserver_rest && ensure_geoserver_workspace || true + + echo "GeoServer $geoserver_version running at http://localhost:8080/geoserver/web/" + echo " Username: admin | Password: geoserver" + echo " IMPORTANT: Change the default password after first login." + mark_step_complete "geoserver" +} + +function apply_env_overrides() { + local env_file="$1" + + if [ -n "$ENV_DB_NAME" ]; then + sed -i "s|^DB_NAME=\"\"|DB_NAME=\"$ENV_DB_NAME\"|" "$env_file" + fi + if [ -n "$ENV_DB_USER" ]; then + sed -i "s|^DB_USER=\"\"|DB_USER=\"$ENV_DB_USER\"|" "$env_file" + fi + if [ -n "$ENV_DB_PASSWORD" ]; then + sed -i "s|^DB_PASSWORD=\"\"|DB_PASSWORD=\"$ENV_DB_PASSWORD\"|" "$env_file" + fi + if [ -n "$ENV_DEPLOYMENT_DIR" ]; then + sed -i "s|^DEPLOYMENT_DIR=\"\"|DEPLOYMENT_DIR=\"$ENV_DEPLOYMENT_DIR\"|" "$env_file" + fi + if [ -n "$ENV_TMP_LOCATION" ]; then + sed -i "s|^TMP_LOCATION=\"\"|TMP_LOCATION=\"$ENV_TMP_LOCATION\"|" "$env_file" + fi +} + +function set_env_value() { + local env_file="$1" + local key="$2" + local value="$3" + + if grep -q "^${key}=" "$env_file"; then + sed -i "s|^${key}=.*|${key}=\"$value\"|" "$env_file" + else + echo "${key}=\"$value\"" >> "$env_file" + fi +} + +function normalize_public_api_base_url() { + local base_url="$1" + + base_url="$(trim "$base_url")" + base_url="$(strip_wrapping_quotes "$base_url")" + base_url="${base_url%/}" + + if [ -n "$base_url" ] && [[ "$base_url" != */api/v1 ]]; then + base_url="$base_url/api/v1" + fi + + printf '%s\n' "$base_url" +} + +function strip_wrapping_quotes() { + local value="$1" + if [[ "$value" == \"*\" && "$value" == *\" ]]; then + value="${value:1:${#value}-2}" + elif [[ "$value" == \'*\' && "$value" == *\' ]]; then + value="${value:1:${#value}-2}" + fi + printf '%s\n' "$value" +} + +function current_env_value() { + local env_file="$1" + local key="$2" + local value="" + + if [ ! -f "$env_file" ]; then + return 0 + fi + + value=$(grep -E "^${key}=" "$env_file" | tail -n 1 | cut -d'=' -f2- || true) + strip_wrapping_quotes "$value" +} + +function maybe_set_installer_managed_path_value() { + local env_file="$1" + local key="$2" + local legacy_absolute="$3" + local legacy_relative="$4" + local managed_value="$5" + local current_value="" + + current_value="$(current_env_value "$env_file" "$key")" + + if [ -z "$current_value" ] || [ "$current_value" = "$legacy_absolute" ] || [ "$current_value" = "$legacy_relative" ] || [ "$current_value" = "$managed_value" ]; then + set_env_value "$env_file" "$key" "$managed_value" + fi +} + +function generate_fernet_key() { + activate_conda_env + python - <<'PY' +from cryptography.fernet import Fernet +print(Fernet.generate_key().decode("utf-8")) +PY +} + +function fernet_key_is_valid() { + local candidate="$1" + + if [ -z "$candidate" ]; then + return 1 + fi + + activate_conda_env + python - "$candidate" <<'PY' +import sys +from cryptography.fernet import Fernet + +try: + Fernet(sys.argv[1].encode("utf-8")) +except Exception: + raise SystemExit(1) +raise SystemExit(0) +PY +} + +function generate_env_file() { + local settings_file="$BACKEND_DIR/nrm_app/settings.py" + local env_file="$APP_ENV_FILE" + local env_vars="" + local env_vars_simple="" + local all_vars="" + local existing_vars="" + local var_name="" + local current_fernet_key="" + local fernet_key="" + + echo "Generating .env file from settings.py..." + + if [ ! -f "$settings_file" ]; then + echo "ERROR: settings.py not found at $settings_file" + return 1 + fi + + env_vars=$(grep -oE 'env\.[a-z]*\s*\(\s*"[A-Za-z_][A-Za-z0-9_]*"' "$settings_file" 2>/dev/null | \ + sed -E 's/env\.[a-z]*\s*\(\s*"([^"]+)"/\1/' | sort -u) + env_vars_simple=$(grep -oE 'env\s*\(\s*"[A-Za-z_][A-Za-z0-9_]*"' "$settings_file" 2>/dev/null | \ + sed -E 's/env\s*\(\s*"([^"]+)"/\1/' | sort -u) + all_vars=$(printf '%s\n%s\n' "$env_vars" "$env_vars_simple" | sort -u | grep -v '^$' || true) + + if [ ! -f "$env_file" ] && [ -f "$LEGACY_ROOT_ENV_FILE" ]; then + echo "Migrating existing root .env to $APP_ENV_FILE ..." + mkdir -p "$(dirname "$env_file")" + cp "$LEGACY_ROOT_ENV_FILE" "$env_file" + fi + + if [ ! -f "$env_file" ]; then + echo "Creating new .env file..." + mkdir -p "$(dirname "$env_file")" + { + echo "# Auto-generated .env file" + echo "# Generated on $(date -u +"%Y-%m-%dT%H:%M:%SZ")" + echo "" + echo "SECRET_KEY=$(openssl rand -base64 32)" + echo "DEBUG=True" + echo "" + } > "$env_file" + else + echo "Existing .env file found. Updating missing variables..." + fi + + existing_vars=$(grep -E '^[A-Za-z_][A-Za-z0-9_]*=' "$env_file" | cut -d'=' -f1 | sort -u || true) + + while IFS= read -r var_name; do + if [ -z "$var_name" ] || [ "$var_name" = "SECRET_KEY" ] || [ "$var_name" = "DEBUG" ]; then + continue + fi + + if echo "$existing_vars" | grep -qx "$var_name"; then + continue + fi + + case "$var_name" in + WHATSAPP_MEDIA_PATH) + echo "WHATSAPP_MEDIA_PATH=$BACKEND_DIR/bot_interface/whatsapp_media" >> "$env_file" + mkdir -p "$BACKEND_DIR/bot_interface/whatsapp_media" + ;; + EXCEL_DIR) + echo "EXCEL_DIR=$BACKEND_DIR/data/excel_files" >> "$env_file" + mkdir -p "$BACKEND_DIR/data/excel_files" + ;; + EXCEL_PATH) + echo "EXCEL_PATH=$BACKEND_DIR" >> "$env_file" + ;; + *) + echo "${var_name}=\"\"" >> "$env_file" + ;; + esac + done <<< "$all_vars" + + if ! grep -q '^BACKEND_DIR=' "$env_file"; then + echo 'BACKEND_DIR=.' >> "$env_file" + fi + + if ! grep -q '^EXCEL_DIR=' "$env_file"; then + echo "EXCEL_DIR=$BACKEND_DIR/data/excel_files" >> "$env_file" + mkdir -p "$BACKEND_DIR/data/excel_files" + fi + + if ! grep -q '^WHATSAPP_MEDIA_PATH=' "$env_file"; then + echo "WHATSAPP_MEDIA_PATH=$BACKEND_DIR/bot_interface/whatsapp_media" >> "$env_file" + mkdir -p "$BACKEND_DIR/bot_interface/whatsapp_media" + fi + + if ! grep -q '^EXCEL_PATH=' "$env_file"; then + echo "EXCEL_PATH=$BACKEND_DIR" >> "$env_file" + fi + + apply_env_overrides "$env_file" + maybe_set_installer_managed_path_value "$env_file" "BACKEND_DIR" "$BACKEND_DIR" "." "." + maybe_set_installer_managed_path_value "$env_file" "DEPLOYMENT_DIR" "$BACKEND_DIR" "." "$BACKEND_DIR" + maybe_set_installer_managed_path_value "$env_file" "TMP_LOCATION" "$BACKEND_DIR/tmp" "tmp" "$BACKEND_DIR/tmp" + maybe_set_installer_managed_path_value "$env_file" "WHATSAPP_MEDIA_PATH" "$BACKEND_DIR/bot_interface/whatsapp_media" "bot_interface/whatsapp_media" "$BACKEND_DIR/bot_interface/whatsapp_media" + maybe_set_installer_managed_path_value "$env_file" "EXCEL_DIR" "$BACKEND_DIR/data/excel_files" "data/excel_files" "$BACKEND_DIR/data/excel_files" + maybe_set_installer_managed_path_value "$env_file" "EXCEL_PATH" "$BACKEND_DIR" "." "$BACKEND_DIR" + set_env_value "$env_file" "PUBLIC_API_BASE_URL" "$(normalize_public_api_base_url "${PUBLIC_API_BASE_URL_ARG:-$(current_env_value "$env_file" "PUBLIC_API_BASE_URL")}")" + if [ -z "$(current_env_value "$env_file" "PUBLIC_API_BASE_URL")" ]; then + set_env_value "$env_file" "PUBLIC_API_BASE_URL" "$DEFAULT_PUBLIC_API_BASE_URL" + fi + if [ -n "$PUBLIC_API_X_API_KEY_ARG" ]; then + set_env_value "$env_file" "PUBLIC_API_X_API_KEY" "$PUBLIC_API_X_API_KEY_ARG" + elif ! grep -q '^PUBLIC_API_X_API_KEY=' "$env_file"; then + echo 'PUBLIC_API_X_API_KEY=""' >> "$env_file" + fi + if [ -n "$GEOSERVER_URL_ARG" ]; then + set_env_value "$env_file" "GEOSERVER_URL" "$GEOSERVER_URL_ARG" + fi + if [ -n "$GEOSERVER_USERNAME_ARG" ]; then + set_env_value "$env_file" "GEOSERVER_USERNAME" "$GEOSERVER_USERNAME_ARG" + fi + if [ -n "$GEOSERVER_PASSWORD_ARG" ]; then + set_env_value "$env_file" "GEOSERVER_PASSWORD" "$GEOSERVER_PASSWORD_ARG" + fi + + current_fernet_key="$(current_env_value "$env_file" "FERNET_KEY")" + if fernet_key_is_valid "$current_fernet_key"; then + echo "FERNET_KEY already present and valid. Keeping the existing value." + else + fernet_key="$(generate_fernet_key)" + set_env_value "$env_file" "FERNET_KEY" "$fernet_key" + echo "FERNET_KEY generated and added to .env" + fi + + chown "${USER:-root}:${USER:-root}" "$env_file" 2>/dev/null || true + chmod 640 "$env_file" + + echo "Total variables in .env: $(grep -c '^[A-Za-z_]' "$env_file")" + echo ".env ready at $env_file" + mark_step_complete "env_file" +} + +function collect_static_files() { + echo "Collecting static files..." + activate_conda_env + cd "$BACKEND_DIR" + python manage.py collectstatic --noinput --clear --skip-checks + echo "Static files collected." + mark_step_complete "collectstatic" +} + +function reset_django_migrations() { + echo "Resetting Django migrations..." + cd "$BACKEND_DIR" + + find . -path "*/migrations/*.py" -not -name "__init__.py" -delete + find . -path "*/migrations/*.pyc" -delete + + find . -maxdepth 2 -name "apps.py" -type f | while IFS= read -r f; do + d=$(dirname "$f") + mkdir -p "$d/migrations" + touch "$d/migrations/__init__.py" + done + + echo "Migrations cleaned." +} + +function run_django_migrations() { + echo "Running Django migrations..." + activate_conda_env + + reset_django_migrations + + cd "$BACKEND_DIR" + python manage.py makemigrations --skip-checks + python manage.py migrate --plan --skip-checks + python manage.py migrate --fake-initial --skip-checks + + echo "Django migrations complete." + mark_step_complete "django_migrations" +} + +function seed_data_loaded() { + activate_conda_env + cd "$BACKEND_DIR" + local has_seed_data + has_seed_data=$(python manage.py shell -c "from geoadmin.models import StateSOI; print(1 if StateSOI.objects.exists() else 0)" 2>/dev/null | tail -n 1) + [ "${has_seed_data:-0}" = "1" ] +} + +function load_seed_data() { + local force="${1:-0}" + local seed_file="$BACKEND_DIR/installation/seed/seed_data.json" + + if [ ! -f "$seed_file" ]; then + echo "No seed data found at $seed_file. Skipping." + return + fi + + activate_conda_env + cd "$BACKEND_DIR" + + if [ "$force" -ne 1 ] && seed_data_loaded; then + echo "Seed data already looks loaded. Keeping the existing database contents." + python manage.py seed_default_plantation --skip-checks + mark_step_complete "seed_data" + return + fi + + echo "Loading seed data..." + python manage.py loaddata --skip-checks "$seed_file" + python manage.py seed_default_plantation --skip-checks + echo "Seed data loaded." + mark_step_complete "seed_data" +} + +function normalize_user_path() { + local raw_path="$1" + local normalized_path="" + local drive_letter="" + local remainder="" + + normalized_path="$(trim "$raw_path")" + normalized_path="$(strip_wrapping_quotes "$normalized_path")" + normalized_path="${normalized_path//\\//}" + + if [[ "$normalized_path" == "~"* ]]; then + normalized_path="${HOME}${normalized_path:1}" + fi + + if [[ "$normalized_path" =~ ^([A-Za-z]):/(.*)$ ]]; then + drive_letter="${BASH_REMATCH[1],,}" + remainder="${BASH_REMATCH[2]}" + normalized_path="/mnt/${drive_letter}/${remainder}" + elif [[ "$normalized_path" != /* ]]; then + normalized_path="$INSTALL_INVOCATION_DIR/$normalized_path" + fi + + if command -v realpath >/dev/null 2>&1; then + normalized_path="$(realpath -m "$normalized_path")" + fi + + printf '%s\n' "$normalized_path" +} + +function looks_like_user_path_input() { + local candidate="$1" + candidate="$(trim "$candidate")" + candidate="$(strip_wrapping_quotes "$candidate")" + + [ -n "$candidate" ] || return 1 + + [[ "$candidate" =~ ^[A-Za-z]:[\\/].* ]] && return 0 + [[ "$candidate" == *"/"* ]] && return 0 + [[ "$candidate" == *"\\"* ]] && return 0 + [[ "$candidate" == ./* ]] && return 0 + [[ "$candidate" == ../* ]] && return 0 + [[ "$candidate" == "~"* ]] && return 0 + [[ "$candidate" == *.json ]] && return 0 + + return 1 +} + +function auto_configure_gee_account_ids() { + local env_file="$APP_ENV_FILE" + local first_account_id="" + local helper_account_id="" + + [ -f "$env_file" ] || return 0 + + activate_conda_env + cd "$BACKEND_DIR" + + first_account_id=$(python manage.py shell -c "from gee_computing.models import GEEAccount; account = GEEAccount.objects.order_by('id').first(); print(account.id if account else '')" 2>/dev/null | tail -n 1) + helper_account_id=$(python manage.py shell -c "from gee_computing.models import GEEAccount; account = GEEAccount.objects.exclude(helper_account=None).order_by('id').first(); print(account.helper_account_id if account and account.helper_account_id else '')" 2>/dev/null | tail -n 1) + + if [ -n "$first_account_id" ] && grep -q '^GEE_DEFAULT_ACCOUNT_ID=""' "$env_file"; then + sed -i "s|^GEE_DEFAULT_ACCOUNT_ID=\"\"|GEE_DEFAULT_ACCOUNT_ID=\"$first_account_id\"|" "$env_file" + echo "Auto-configured GEE_DEFAULT_ACCOUNT_ID=$first_account_id" + fi + + if [ -n "$helper_account_id" ] && grep -q '^GEE_HELPER_ACCOUNT_ID=""' "$env_file"; then + sed -i "s|^GEE_HELPER_ACCOUNT_ID=\"\"|GEE_HELPER_ACCOUNT_ID=\"$helper_account_id\"|" "$env_file" + echo "Auto-configured GEE_HELPER_ACCOUNT_ID=$helper_account_id" + fi +} + +function configure_paths() { + local gee_json_path_input="$1" + local env_file="$APP_ENV_FILE" + local account_name="${2:-$DEFAULT_GEE_ACCOUNT_NAME}" + local normalized_gee_json_path="" + local import_result="" + local account_id="" + local staged_relative_path="" + + normalized_gee_json_path="$(normalize_user_path "$gee_json_path_input")" + + if [ "$normalized_gee_json_path" != "$gee_json_path_input" ]; then + echo "Resolved GEE credentials path to: $normalized_gee_json_path" + fi + + if [ ! -f "$normalized_gee_json_path" ]; then + echo "GEE credentials file not found: $normalized_gee_json_path" + return 1 + fi + + activate_conda_env + cd "$BACKEND_DIR" + + import_result=$(GEE_JSON_PATH="$normalized_gee_json_path" GEE_ACCOUNT_NAME="$account_name" PYTHONPATH="$BACKEND_DIR" python - <<'PY' +import json +import os + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "nrm_app.settings") + +import django + +django.setup() + +from utilities.gee_utils import copy_gee_credentials_into_repo, upsert_gee_account_from_json + +staged_credentials = copy_gee_credentials_into_repo( + credentials_path=os.environ["GEE_JSON_PATH"], +) + +account = upsert_gee_account_from_json( + credentials_path=staged_credentials["absolute_path"], + account_name=os.environ["GEE_ACCOUNT_NAME"], +) + +print( + json.dumps( + { + "account_id": account.id, + "relative_path": staged_credentials["relative_path"], + } + ) +) +PY +) + + account_id=$(echo "$import_result" | tail -n 1 | python -c 'import json,sys; print(json.loads(sys.stdin.read()).get("account_id",""))' 2>/dev/null | tr -d '[:space:]') + staged_relative_path=$(echo "$import_result" | tail -n 1 | python -c 'import json,sys; print(json.loads(sys.stdin.read()).get("relative_path",""))' 2>/dev/null | tr -d '\r') + + if [ -z "$account_id" ] || [ -z "$staged_relative_path" ]; then + echo "Unable to create or update the GEE account from the provided JSON." + return 1 + fi + + set_env_value "$env_file" "GEE_DEFAULT_ACCOUNT_ID" "$account_id" + set_env_value "$env_file" "GEE_HELPER_ACCOUNT_ID" "$account_id" + set_env_value "$env_file" "GEE_SERVICE_ACCOUNT_KEY_PATH" "$staged_relative_path" + set_env_value "$env_file" "GEE_HELPER_SERVICE_ACCOUNT_KEY_PATH" "$staged_relative_path" + POST_INSTALL_REQUIRE_GEE=1 + echo "Configured GEE account id=$account_id using staged credentials at $staged_relative_path" + mark_step_complete "gee_configuration" +} + +function optional_configure_gee_account() { + local force="${1:-0}" + local gee_json_path_input="" + local had_existing_gee_configuration=0 + + auto_configure_gee_account_ids + if gee_configuration_present; then + had_existing_gee_configuration=1 + POST_INSTALL_REQUIRE_GEE=1 + fi + + if [ -n "$GEE_JSON_PATH_ARG" ]; then + gee_json_path_input="$GEE_JSON_PATH_ARG" + elif [ "$force" -ne 1 ] && [ "$had_existing_gee_configuration" -eq 1 ]; then + echo "Existing Google Earth Engine configuration detected. Keeping it." + mark_step_complete "gee_configuration" + return + elif [ ! -t 0 ]; then + if [ "$had_existing_gee_configuration" -eq 1 ]; then + echo "No interactive terminal detected. Keeping the existing GEE configuration." + mark_step_complete "gee_configuration" + return + fi + echo "No interactive terminal detected. Skipping optional GEE setup." + POST_INSTALL_REQUIRE_GEE=0 + return + else + echo "" + echo "Optional: configure Google Earth Engine now." + echo "If your organization shared a service-account JSON, enter its full path below." + echo "Windows paths like C:\\Users\\name\\Downloads\\file.json and relative paths like .\\file.json are accepted." + echo "Press Enter to skip." + read -r -p "GEE JSON path: " gee_json_path_input + fi + + if [ -z "$gee_json_path_input" ]; then + if [ "$had_existing_gee_configuration" -eq 1 ]; then + echo "Keeping the existing GEE configuration." + POST_INSTALL_REQUIRE_GEE=1 + mark_step_complete "gee_configuration" + return + fi + + echo "Skipping optional GEE setup for now." + POST_INSTALL_REQUIRE_GEE=0 + return + fi + + if configure_paths "$gee_json_path_input"; then + echo "GEE credentials imported. The final initialisation test will validate the live GEE path too." + else + POST_INSTALL_REQUIRE_GEE="$had_existing_gee_configuration" + echo "GEE setup did not complete. Continuing with the core initialisation test only." + fi +} + +function ensure_django_superuser() { + local result="" + local username="" + local action="" + + echo "Ensuring installer test superuser exists..." + activate_conda_env + cd "$BACKEND_DIR" + + result=$(python manage.py shell <<'PY' +import random +from django.contrib.auth import get_user_model + +User = get_user_model() +installer_user = User.objects.filter(username__startswith="test_user_", is_superuser=True).order_by("id").first() + +if installer_user is None: + while True: + username = f"test_user_{random.randint(0, 9999):04d}" + if not User.objects.filter(username=username).exists(): + break + installer_user = User.objects.create_superuser( + username=username, + email="", + password="test_change_me", + ) + installer_user.is_active = True + installer_user.is_staff = True + installer_user.save(update_fields=["is_active", "is_staff"]) + print(f"{installer_user.username}|created") +else: + installer_user.is_active = True + installer_user.is_staff = True + installer_user.is_superuser = True + installer_user.set_password("test_change_me") + installer_user.save(update_fields=["is_active", "is_staff", "is_superuser", "password"]) + print(f"{installer_user.username}|updated") +PY +) + + username="${result%%|*}" + action="${result##*|}" + echo "Installer test superuser $action: username=$username password=test_change_me" + mark_step_complete "superuser" +} + +function public_api_configuration_present() { + local api_key="" + local base_url="" + + api_key="$(current_env_value "$APP_ENV_FILE" "PUBLIC_API_X_API_KEY")" + base_url="$(current_env_value "$APP_ENV_FILE" "PUBLIC_API_BASE_URL")" + + [ -n "$api_key" ] && [ -n "$base_url" ] +} + +function persist_optional_inputs_to_env() { + local base_url="" + + [ -f "$APP_ENV_FILE" ] || return 0 + + if [ -n "$PUBLIC_API_X_API_KEY_ARG" ]; then + set_env_value "$APP_ENV_FILE" "PUBLIC_API_X_API_KEY" "$PUBLIC_API_X_API_KEY_ARG" + fi + + base_url="$PUBLIC_API_BASE_URL_ARG" + if [ -z "$base_url" ]; then + base_url="$(current_env_value "$APP_ENV_FILE" "PUBLIC_API_BASE_URL")" + fi + if [ -n "$base_url" ]; then + set_env_value "$APP_ENV_FILE" "PUBLIC_API_BASE_URL" "$(normalize_public_api_base_url "$base_url")" + fi + + if [ -n "$GEOSERVER_URL_ARG" ]; then + set_env_value "$APP_ENV_FILE" "GEOSERVER_URL" "$GEOSERVER_URL_ARG" + fi + if [ -n "$GEOSERVER_USERNAME_ARG" ]; then + set_env_value "$APP_ENV_FILE" "GEOSERVER_USERNAME" "$GEOSERVER_USERNAME_ARG" + fi + if [ -n "$GEOSERVER_PASSWORD_ARG" ]; then + set_env_value "$APP_ENV_FILE" "GEOSERVER_PASSWORD" "$GEOSERVER_PASSWORD_ARG" + fi +} + +function ensure_dirs() { + mkdir -p "$BACKEND_DIR/logs" + touch "$BACKEND_DIR/logs/app.log" "$BACKEND_DIR/logs/nrm_app.log" + mkdir -p "$BACKEND_DIR/data/activated_locations" + mkdir -p "$BACKEND_DIR/data/excel_files" + mkdir -p "$BACKEND_DIR/tmp" + mkdir -p "$INSTALL_STATE_DIR" + echo "Required directories ready." +} + +function install_gdown_if_missing() { + activate_conda_env + if python -m pip show gdown >/dev/null 2>&1; then + return + fi + echo "Installing gdown into $CONDA_ENV_NAME ..." + python -m pip install gdown +} + +function download_admin_boundary_data() { + local force="${1:-0}" + local admin_boundary_dir="$BACKEND_DIR/data/admin-boundary" + local archive_path="$BACKEND_DIR/dataset.7z" + local extraction_root="$BACKEND_DIR/data/.admin-boundary-extract" + local fileid="1VqIhB6HrKFDkDnlk1vedcEHhh5fk4f1d" + + if [ "$force" -ne 1 ] && admin_boundary_data_present; then + echo "Admin-boundary data already present. Skipping download." + mark_step_complete "admin_boundary_data" + return + fi + + if [ "$force" -ne 1 ] && normalize_existing_admin_boundary_data; then + echo "Existing admin-boundary data detected. Keeping it." + return + fi + + echo "Downloading admin-boundary data (~8GB, this may take a while)..." + rm -rf "$admin_boundary_dir" "$extraction_root" + rm -f "$archive_path" + mkdir -p "$BACKEND_DIR/data" "$extraction_root" + + install_gdown_if_missing + # Use py7zr (Python) instead of p7zip-full — not available on all systems + python -m pip install --quiet py7zr + + activate_conda_env + cd "$BACKEND_DIR" + gdown "$fileid" -O "$archive_path" + python -c " +import py7zr, sys +print('Extracting archive...') +with py7zr.SevenZipFile('$archive_path', 'r') as z: + z.extractall('$extraction_root') +print('Extraction complete.') +" + rm -f "$archive_path" + + finalize_admin_boundary_extraction "$extraction_root" + rm -rf "$extraction_root" + + echo "Admin-boundary data extracted to $admin_boundary_dir" + mark_step_complete "admin_boundary_data" +} + +function run_post_install_initialisation_check() { + local initialisation_args=() + + normalize_existing_admin_boundary_data >/dev/null 2>&1 || true + persist_optional_inputs_to_env + + activate_conda_env + cd "$BACKEND_DIR" + + echo "" + echo "Running internal API initialisation test..." + echo "This validation runs Django in-process, creates a JWT Bearer token automatically," + echo "and forces Celery eager mode for the checked task. You do not need runserver" + echo "or a separate Celery worker for this installer-time verification." + + if [ "${POST_INSTALL_REQUIRE_GEE:-0}" -eq 1 ]; then + initialisation_args=(--require-gee) + fi + + if python computing/misc/internal_api_initialisation_test.py "${initialisation_args[@]}"; then + POST_INSTALL_INITIALISATION_FAILED=0 + echo "Internal API initialisation test passed." + else + POST_INSTALL_INITIALISATION_FAILED=1 + echo "Internal API initialisation test found issues. Review the output above before using the APIs." + fi + mark_step_complete "initialisation_check" +} + +function run_public_api_smoke_test() { + local api_key="" + local base_url="" + local smoke_test_args=() + + if [ ! -f "$APP_ENV_FILE" ]; then + if [ -z "$PUBLIC_API_X_API_KEY_ARG" ]; then + echo "Skipping public API smoke test because $APP_ENV_FILE does not exist yet." + mark_step_complete "public_api_check" + return + fi + else + persist_optional_inputs_to_env + fi + + api_key="$PUBLIC_API_X_API_KEY_ARG" + if [ -z "$api_key" ] && [ -f "$APP_ENV_FILE" ]; then + api_key="$(current_env_value "$APP_ENV_FILE" "PUBLIC_API_X_API_KEY")" + fi + base_url="$PUBLIC_API_BASE_URL_ARG" + if [ -z "$base_url" ] && [ -f "$APP_ENV_FILE" ]; then + base_url="$(current_env_value "$APP_ENV_FILE" "PUBLIC_API_BASE_URL")" + fi + + if [ -z "$api_key" ]; then + echo "Skipping public API smoke test because PUBLIC_API_X_API_KEY is not configured." + echo "Provide it up front with --input public_api_key=... or update $APP_ENV_FILE later." + mark_step_complete "public_api_check" + return + fi + + if [ -z "$base_url" ]; then + base_url="$DEFAULT_PUBLIC_API_BASE_URL" + if [ -f "$APP_ENV_FILE" ]; then + set_env_value "$APP_ENV_FILE" "PUBLIC_API_BASE_URL" "$base_url" + fi + fi + + echo "" + echo "Running public API smoke test..." + echo "Sample location: $DEFAULT_PUBLIC_API_SAMPLE_STATE / $DEFAULT_PUBLIC_API_SAMPLE_DISTRICT / $DEFAULT_PUBLIC_API_SAMPLE_TEHSIL" + + activate_conda_env + cd "$BACKEND_DIR" + + if [ -f "$APP_ENV_FILE" ]; then + smoke_test_args+=(--env-file "$APP_ENV_FILE") + fi + smoke_test_args+=(--api-key "$api_key" --base-url "$base_url") + + if python installation/public_api_client.py \ + "${smoke_test_args[@]}" \ + smoke-test \ + --state "$DEFAULT_PUBLIC_API_SAMPLE_STATE" \ + --district "$DEFAULT_PUBLIC_API_SAMPLE_DISTRICT" \ + --tehsil "$DEFAULT_PUBLIC_API_SAMPLE_TEHSIL"; then + echo "Public API smoke test passed." + else + POST_INSTALL_INITIALISATION_FAILED=1 + echo "Public API smoke test found issues. Review the output above before sharing public API instructions." + fi + + mark_step_complete "public_api_check" +} + +function run_step() { + local step="$1" + local force="$2" + + echo "" + echo "==============================================" + echo " $(step_label "$step")" + echo "==============================================" + + case "$step" in + unzip_install) + if command -v unzip >/dev/null 2>&1; then + echo "unzip already installed." + else + apt-get install -y unzip + fi + mark_step_complete "unzip_install" + ;; + miniconda) + install_miniconda + ;; + postgres) + install_postgres + ;; + rabbitmq) + install_rabbitmq + ;; + conda_env) + setup_conda_env "$force" + ;; + env_file) + generate_env_file + ;; + geoserver) + install_geoserver + ;; + collectstatic) + collect_static_files + ;; + django_migrations) + run_django_migrations + ;; + seed_data) + load_seed_data "$force" + ;; + superuser) + ensure_django_superuser + ;; + gee_configuration) + optional_configure_gee_account "$force" + ;; + admin_boundary_data) + download_admin_boundary_data "$force" + ;; + initialisation_check) + run_post_install_initialisation_check + ;; + public_api_check) + run_public_api_smoke_test + ;; + *) + echo "Unknown step: $step" + exit 1 + ;; + esac +} + +function print_selection_summary() { + local selected_steps=() + local step="" + + for step in "${STEP_ORDER[@]}"; do + if should_execute_step "$step"; then + selected_steps+=("$step") + fi + done + + echo "Selected steps:" + for step in "${selected_steps[@]}"; do + echo " - $step" + done +} + +function main() { + local step="" + local force=0 + + parse_args "$@" + + if [ "$LIST_STEPS_ONLY" -eq 1 ]; then + print_available_steps + return 0 + fi + + prompt_for_optional_inputs + ensure_dirs + print_selection_summary + print_optional_input_summary + + for step in "${STEP_ORDER[@]}"; do + if ! should_execute_step "$step"; then + continue + fi + + force=0 + if step_is_forced "$step"; then + force=1 + fi + + run_step "$step" "$force" + done + + echo "" + echo "==============================================" + echo " Core installation complete!" + echo "==============================================" + echo "" + echo "Activate env: conda activate $CONDA_ENV_NAME" + echo "" + echo "IMPORTANT: Review and update the .env file at $BACKEND_DIR/nrm_app/.env" + echo " with your actual credentials before running in production." + echo "" + + if [ "${POST_INSTALL_INITIALISATION_FAILED:-0}" -eq 1 ]; then + echo "All done, but post-install validation found issues that still need attention." + else + echo "All done! Setup is fully complete." + fi +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi