Bulk fetch sms parts+cost - platform admin service list page

[whabanks]

Summary | Résumé

This PR hooks up the platform admin services list page to the new bulk sms parts + cost API endpoint. Previously we fetched these stats one service at a time by id, which resulted in massive slow downs on the platform admin service list page load times and unnecessary load on the DB.

Page load performance comparison

Staging

Compares staging as is without the fragment + sms cost counts in the table to this PR running locally hooked up to staging.

1 month of data

Staging: 2s
This PR: 2.1s

4 months of data

Staging: 2.65s
This PR: 4s

1 Year of data

Staging: 5.1s
This PR: 6s

Prod

1 month of data

Prod: 3.4s
This PR: 3.8s

4 months of data

Prod: 12.3s
This PR: 16s

6 months of data

Prod: 23s
This PR: 23.7s

1 Year of data

Prod: 504 Gateway timeout
This PR: 504 Gateway timeout

Test instructions | Instructions pour tester la modification

1. Check out this branch and the associated api branch
2. [Optional, can hook up to staging too] Use the following SQL script to populate your local DB with a lot of data to simulate load times closer to what we'd see in prod:
   • The script inserts services, templates, users, and ft_billing / notification_status data used to calculate the SMS cost and parts
   • After execution a marker date is output, keep track of this as you will use it later to clean this data up from your local DB

Example:

GENERATOR MARKER_TS: 2026-04-23 16:10:44.919749-04

--- Data gen script ---

-- Generator: Inserts test data for ft_billing and ft_notification_status. The script creates
-- X services
-- 1 Template per service
-- 1 User per service
-- 9 ft_notification_status rows per service
-- 10 ft_billing rows per service
-- Usage:
-- The script prints a marker timestamp (NOTICE) which you should use with the cleanup script.

DO $$
DECLARE
  no_job uuid := '00000000-0000-0000-0000-000000000000';
  marker_ts timestamptz := now();
  v_user uuid;
  v_service uuid;
  v_template uuid;
  i int;
BEGIN
  RAISE NOTICE 'GENERATOR MARKER_TS: %', marker_ts;

  FOR i IN 1..200 LOOP -- Adjust the number of services to create as needed
    v_user := gen_random_uuid();
    v_service := gen_random_uuid();
    v_template := gen_random_uuid();

    -- create a user for the service
    INSERT INTO users (id, name, email_address, _password, mobile_number, password_changed_at, state, auth_type, created_at, blocked, platform_admin, failed_login_count, password_expired)
    VALUES (v_user, 'test-user-' || i, 'test-user-' || i || '@example.local', 'simulated-no-login', '+16135550199', marker_ts, 'active', 'email_auth', marker_ts, false, false, 0, false);

    -- create the service (minimal required columns, adjust as necessary)
    INSERT INTO services (id, name, created_by_id, created_at, active, restricted, message_limit, sms_daily_limit, sms_annual_limit, email_annual_limit, email_from, organisation_type, prefix_sms, rate_limit, count_as_live, research_mode, organisation_id, version, default_branding_is_french)
    VALUES (v_service, 'test-service-' || i, v_user, marker_ts, true, false, 1000000, 1000, 1000000, 1000000, 'no-reply@example.local', 'central', true, 5000, true, false, NULL, 1, false);

    -- create a simple template for the service
    INSERT INTO templates (id, name, template_type, created_at, content, subject, archived, hidden, service_id, created_by_id, version, process_type)
    VALUES (v_template, 'test-template-' || i, 'sms', marker_ts, 'Test message for service ' || i, NULL, false, false, v_service, v_user, 1, 'normal');

    -- Insert representative ft_notification_status rows (2025 sample dates)
    INSERT INTO ft_notification_status (bst_date, template_id, service_id, job_id, notification_type, key_type, notification_status, notification_count, billable_units, created_at)
    VALUES
      ('2025-01-05', v_template, v_service, no_job, 'sms', 'normal', 'delivered', 80,  120, marker_ts),
      ('2025-01-15', v_template, v_service, no_job, 'sms', 'normal', 'delivered', 130, 200, marker_ts),
      ('2025-01-28', v_template, v_service, no_job, 'sms', 'normal', 'delivered', 100, 150, marker_ts),
      ('2025-03-01', v_template, v_service, no_job, 'sms', 'normal', 'delivered', 200, 300, marker_ts),
      ('2025-03-15', v_template, v_service, no_job, 'sms', 'normal', 'delivered', 280, 450, marker_ts),
      ('2025-03-31', v_template, v_service, no_job, 'sms', 'normal', 'delivered', 120, 180, marker_ts),
      ('2025-06-10', v_template, v_service, no_job, 'sms', 'normal', 'delivered', 350, 500, marker_ts),
      ('2025-06-20', v_template, v_service, no_job, 'sms', 'normal', 'delivered', 140, 220, marker_ts),
      ('2025-09-05', v_template, v_service, no_job, 'sms', 'normal', 'delivered', 240, 380, marker_ts);

    -- Insert representative ft_billing rows (matching dates)
    INSERT INTO ft_billing (bst_date, template_id, service_id, notification_type, provider, rate_multiplier, international, rate, postage, sms_sending_vehicle, billable_units, notifications_sent, billing_total, created_at)
    VALUES
      ('2025-01-05', v_template, v_service, 'sms', 'pinpoint', 1, false, 0.0162, 'none', 'long_code', 120, 80, 120 * 0.0162, marker_ts),
      ('2025-01-15', v_template, v_service, 'sms', 'pinpoint', 1, false, 0.0162, 'none', 'long_code', 200, 130, 200 * 0.0162, marker_ts),
      ('2025-01-28', v_template, v_service, 'sms', 'pinpoint', 1, false, 0.0162, 'none', 'long_code', 150, 100, 150 * 0.0162, marker_ts),
      ('2025-03-01', v_template, v_service, 'sms', 'pinpoint', 1, false, 0.0162, 'none', 'long_code', 300, 200, 300 * 0.0162, marker_ts),
      ('2025-03-15', v_template, v_service, 'sms', 'pinpoint', 1, false, 0.0162, 'none', 'long_code', 450, 280, 450 * 0.0162, marker_ts),
      ('2025-03-31', v_template, v_service, 'sms', 'pinpoint', 1, false, 0.0162, 'none', 'long_code', 180, 120, 180 * 0.0162, marker_ts),
      ('2025-06-10', v_template, v_service, 'sms', 'pinpoint', 1, false, 0.02683, 'none', 'short_code', 500, 350, 500 * 0.02683, marker_ts),
      ('2025-06-20', v_template, v_service, 'sms', 'pinpoint', 1, false, 0.0162, 'none', 'long_code', 220, 140, 220 * 0.0162, marker_ts),
      ('2025-09-05', v_template, v_service, 'sms', 'pinpoint', 1, false, 0.0162, 'none', 'long_code', 350, 230, 350 * 0.0162, marker_ts),
      ('2025-09-05', v_template, v_service, 'sms', 'pinpoint', 3, true, 0.0847, 'none', 'long_code', 30, 10, 30 * 3 * 0.0847, marker_ts);

  END LOOP;
END $$;

3. Navigate to /platform-admin/live-services > expand the Apply filters section
   • Ensure the load time is reasonable (load time for chrome is a nice extension for looking at a glance)
4. Filter anywhere between these date ranges: 2025-01-05 > 2025-09-05

• Note that your service appears in the list
• Note that as you adjust the date filter ranges that data in the SMS parts and SMS cost columns are updated
• Ensure the load times are reasonable

To clean up your DB, add the GENERATOR MARKER_TS date output from the generation script into the markers array in the clean up script then execute it.

--- Clean up script ---

DO $$
DECLARE
  markers timestamptz[] := ARRAY[
    -- Replace or paste your marker timestamps here (examples):
    '2026-04-23 15:53:12.123456+00'::timestamptz
  ];
  m timestamptz;
  del_count int;
BEGIN
  RAISE NOTICE 'Cleanup run: % markers', array_length(markers, 1);

  FOREACH m IN ARRAY markers LOOP
    RAISE NOTICE 'Cleaning marker % (1-hour window starting at marker)', m;

    DELETE FROM ft_notification_status
    WHERE created_at >= m
      AND created_at < (m + interval '1 hour');
    GET DIAGNOSTICS del_count = ROW_COUNT;
    RAISE NOTICE 'Deleted % rows from ft_notification_status for marker %', del_count, m;

    DELETE FROM ft_billing
    WHERE created_at >= m
      AND created_at < (m + interval '1 hour');
    GET DIAGNOSTICS del_count = ROW_COUNT;
    RAISE NOTICE 'Deleted % rows from ft_billing for marker %', del_count, m;
    
    -- Remove templates created by the generator in this window
    DELETE FROM template_folder_map
    WHERE template_id IN (
      SELECT id FROM templates WHERE created_at >= m AND created_at < (m + interval '1 hour') AND name LIKE 'test-template-%'
    );
    GET DIAGNOSTICS del_count = ROW_COUNT;
    RAISE NOTICE 'Deleted % rows from template_folder_map for marker %', del_count, m;

    DELETE FROM templates_history
    WHERE service_id IN (
      SELECT id FROM services WHERE created_at >= m AND created_at < (m + interval '1 hour') AND name LIKE 'test-service-%'
    );
    GET DIAGNOSTICS del_count = ROW_COUNT;
    RAISE NOTICE 'Deleted % rows from templates_history for marker %', del_count, m;

    DELETE FROM templates
    WHERE created_at >= m AND created_at < (m + interval '1 hour') AND name LIKE 'test-template-%';
    GET DIAGNOSTICS del_count = ROW_COUNT;
    RAISE NOTICE 'Deleted % templates for marker %', del_count, m;

    -- Remove service-related rows (permissions, senders, billing, reply-to, etc.)
    DELETE FROM user_folder_permissions
    WHERE service_id IN (
      SELECT id FROM services WHERE created_at >= m AND created_at < (m + interval '1 hour') AND name LIKE 'test-service-%'
    );
    GET DIAGNOSTICS del_count = ROW_COUNT;
    RAISE NOTICE 'Deleted % rows from user_folder_permissions for marker %', del_count, m;

    DELETE FROM user_to_service
    WHERE service_id IN (
      SELECT id FROM services WHERE created_at >= m AND created_at < (m + interval '1 hour') AND name LIKE 'test-service-%'
    );
    GET DIAGNOSTICS del_count = ROW_COUNT;
    RAISE NOTICE 'Deleted % rows from user_to_service for marker %', del_count, m;

    DELETE FROM service_permissions
    WHERE service_id IN (
      SELECT id FROM services WHERE created_at >= m AND created_at < (m + interval '1 hour') AND name LIKE 'test-service-%'
    );
    GET DIAGNOSTICS del_count = ROW_COUNT;
    RAISE NOTICE 'Deleted % rows from service_permissions for marker %', del_count, m;

    DELETE FROM service_sms_senders
    WHERE service_id IN (
      SELECT id FROM services WHERE created_at >= m AND created_at < (m + interval '1 hour') AND name LIKE 'test-service-%'
    );
    GET DIAGNOSTICS del_count = ROW_COUNT;
    RAISE NOTICE 'Deleted % rows from service_sms_senders for marker %', del_count, m;

    DELETE FROM annual_billing
    WHERE service_id IN (
      SELECT id FROM services WHERE created_at >= m AND created_at < (m + interval '1 hour') AND name LIKE 'test-service-%'
    );
    GET DIAGNOSTICS del_count = ROW_COUNT;
    RAISE NOTICE 'Deleted % rows from annual_billing for marker %', del_count, m;

    DELETE FROM service_email_reply_to
    WHERE service_id IN (
      SELECT id FROM services WHERE created_at >= m AND created_at < (m + interval '1 hour') AND name LIKE 'test-service-%'
    );
    GET DIAGNOSTICS del_count = ROW_COUNT;
    RAISE NOTICE 'Deleted % rows from service_email_reply_to for marker %', del_count, m;

    DELETE FROM services_history
    WHERE id IN (
      SELECT id FROM services WHERE created_at >= m AND created_at < (m + interval '1 hour') AND name LIKE 'test-service-%'
    );
    GET DIAGNOSTICS del_count = ROW_COUNT;
    RAISE NOTICE 'Deleted % rows from services_history for marker %', del_count, m;

    -- Finally delete services
    DELETE FROM services
    WHERE created_at >= m AND created_at < (m + interval '1 hour') AND name LIKE 'test-service-%';
    GET DIAGNOSTICS del_count = ROW_COUNT;
    RAISE NOTICE 'Deleted % services for marker %', del_count, m;

    -- Delete users created for the services (by email pattern and created_at window)
    DELETE FROM users
    WHERE created_at >= m AND created_at < (m + interval '1 hour') AND email_address LIKE 'test-user-%@example.local';
    GET DIAGNOSTICS del_count = ROW_COUNT;
    RAISE NOTICE 'Deleted % users for marker %', del_count, m;
  END LOOP;
END $$;

[github-actions]

🧪 Review environment

https://cs6x7xq2entvytq2ma3scad2r40mukuf.lambda-url.ca-central-1.on.aws/

[jzbahrai]

left one comment