diff --git a/database/bootstrap.sh b/database/bootstrap.sh index d082dd0..8c4de40 100755 --- a/database/bootstrap.sh +++ b/database/bootstrap.sh @@ -5,10 +5,10 @@ FILES=("public" "postgis" "awips" "vtec" "warnings" "mcd") # Load tables for sql_file in ${FILES[@]}; do echo "Running $sql_file.sql" - psql -h localhost -U postgres -d mds -f "./schemas/$sql_file.sql" || exit 2 + psql -v "ON_ERROR_STOP=1" -h localhost -U postgres -d mds -f "./schemas/$sql_file.sql" || exit 2 done for sql_file in states offices vtec; do echo "Running $sql_file" - psql -h localhost -U postgres -d mds -f "./data/$sql_file.sql" || exit 2 + psql -v "ON_ERROR_STOP=1" -h localhost -U postgres -d mds -f "./data/$sql_file.sql" || exit 2 done \ No newline at end of file diff --git a/database/init.sql b/database/init.sql index de7cb49..79f97b6 100644 --- a/database/init.sql +++ b/database/init.sql @@ -1,3 +1,16 @@ +-- Owner +CREATE USER mds; + +-- Service users +CREATE USER awips_service; +CREATE USER api_service; + +-- Script users +CREATE USER postgis; + +-- Nobody +CREATE USER nobody; + CREATE DATABASE mds; \connect mds \ No newline at end of file diff --git a/database/schemas/awips.sql b/database/schemas/awips.sql index c40e3f9..436637f 100644 --- a/database/schemas/awips.sql +++ b/database/schemas/awips.sql @@ -1,4 +1,5 @@ CREATE SCHEMA IF NOT EXISTS awips; +ALTER SCHEMA awips OWNER TO mds; -- Product -- CREATE TABLE IF NOT EXISTS awips.products ( @@ -15,14 +16,28 @@ CREATE TABLE IF NOT EXISTS awips.products ( PRIMARY KEY (id, issued), UNIQUE ( issued, wmo, awips, bbb, id) ) PARTITION BY RANGE (issued); +ALTER TABLE awips.products OWNER TO mds; +GRANT ALL ON TABLE awips.products TO awips_service; +GRANT SELECT ON TABLE awips.products TO nobody, api_service; -- Create all yearly table -CREATE OR REPLACE FUNCTION awips.CREATE_YEARLY_PARTITIONS (YEAR INTEGER) RETURNS VOID AS $$ +CREATE OR REPLACE FUNCTION awips.CREATE_YEARLY_PARTITIONS (start INTEGER, end INTEGER) RETURNS VOID AS $$ BEGIN - -- Products - PERFORM create_yearly_range_partition('awips.products', year); - EXECUTE format(' - CREATE INDEX product_%s_product_id ON awips.products_%s(product_id);', - year, year); + FOR year IN start..end + LOOP + -- Products + PERFORM create_yearly_range_partition('awips.products', year); + EXECUTE format(' + CREATE INDEX product_%s_product_id ON awips.products_%s(product_id);', + year, year); + EXECUTE format('GRANT ALL ON TABLE awips.products_%s TO mds;', year); + EXECUTE format('GRANT SELECT ON TABLE awips.products_%s TO nobody, api_service;', year); + END LOOP; END -$$ LANGUAGE PLPGSQL; \ No newline at end of file +$$ LANGUAGE PLPGSQL; + +DO $$ +BEGIN + PERFORM awips.CREATE_YEARLY_PARTITIONS(2020, 2030); +END +$$; \ No newline at end of file diff --git a/database/schemas/mcd.sql b/database/schemas/mcd.sql index 2d17f21..9f4e195 100644 --- a/database/schemas/mcd.sql +++ b/database/schemas/mcd.sql @@ -1,4 +1,5 @@ CREATE SCHEMA IF NOT EXISTS mcd; +ALTER SCHEMA mcd OWNER TO mds; -- MCD -- CREATE TABLE IF NOT EXISTS mcd.mcd ( @@ -17,13 +18,29 @@ CREATE TABLE IF NOT EXISTS mcd.mcd ( most_prob_hail text, PRIMARY KEY (id, year) ) PARTITION BY LIST (year); +ALTER TABLE mcd.mcd OWNER TO mds; +GRANT ALL ON TABLE mcd.mcd TO awips_service; +GRANT SELECT ON TABLE mcd.mcd TO nobody, api_service; -CREATE OR REPLACE FUNCTION mcd.CREATE_YEARLY_PARTITIONS (YEAR INTEGER) RETURNS VOID AS $$ +CREATE OR REPLACE FUNCTION mcd.CREATE_YEARLY_PARTITIONS (starts INTEGER, ends INTEGER) RETURNS VOID AS $$ BEGIN - -- MCD - PERFORM create_yearly_list_partition('mcd.mcd', year); - EXECUTE format(' - CREATE INDEX mcd_%s_geom ON mcd.mcd_%s USING GIST (geom);', - year, year); + FOR year IN starts..ends + LOOP + -- MCD + PERFORM create_yearly_list_partition('mcd.mcd', year); + EXECUTE format(' + CREATE INDEX mcd_%s_geom ON mcd.mcd_%s USING GIST (geom);', + year, year); + EXECUTE format('ALTER TABLE mcd.mcd_%s OWNER TO mds;', year); + EXECUTE format('GRANT ALL ON TABLE mcd.mcd_%s TO awips_service;', year); + EXECUTE format('GRANT SELECT ON TABLE mcd.mcd_%s TO nobody, api_service;', year); + END LOOP; END -$$ LANGUAGE PLPGSQL; \ No newline at end of file +$$ LANGUAGE PLPGSQL; + +-- Create the current decade partitions +DO $$ +BEGIN + PERFORM mcd.CREATE_YEARLY_PARTITIONS(2020, 2030); +END +$$; \ No newline at end of file diff --git a/database/schemas/postgis.sql b/database/schemas/postgis.sql index 6d3d5dd..e055ad8 100644 --- a/database/schemas/postgis.sql +++ b/database/schemas/postgis.sql @@ -1,6 +1,7 @@ CREATE EXTENSION postgis; CREATE SCHEMA IF NOT EXISTS postgis; +ALTER SCHEMA postgis OWNER TO mds; -- States -- CREATE TABLE IF NOT EXISTS postgis.states ( @@ -9,6 +10,9 @@ CREATE TABLE IF NOT EXISTS postgis.states ( fips varchar(2) UNIQUE NOT NULL, is_offshore boolean NOT NULL ); +ALTER TABLE postgis.states OWNER TO mds; +GRANT ALL ON TABLE postgis.states TO postgis; +GRANT SELECT ON TABLE postgis.states TO nobody, api_service; -- Office -- CREATE TABLE IF NOT EXISTS postgis.offices ( @@ -18,6 +22,9 @@ CREATE TABLE IF NOT EXISTS postgis.offices ( state char(2) NOT NULL REFERENCES postgis.states(id), location geometry(Point, 4326) ); +ALTER TABLE postgis.offices OWNER TO mds; +GRANT ALL ON TABLE postgis.offices TO postgis; +GRANT SELECT ON TABLE postgis.offices TO nobody, api_service; -- County Warning Area -- CREATE TABLE IF NOT EXISTS postgis.cwas ( @@ -29,6 +36,9 @@ CREATE TABLE IF NOT EXISTS postgis.cwas ( region char(2) NOT NULL, valid_from timestamptz NOT NULL ); +ALTER TABLE postgis.cwas OWNER TO mds; +GRANT ALL ON TABLE postgis.cwas TO postgis; +GRANT SELECT ON TABLE postgis.cwas TO nobody, api_service; -- UGC (Universal Geographic Code) -- CREATE TABLE IF NOT EXISTS postgis.ugcs ( @@ -47,4 +57,7 @@ CREATE TABLE IF NOT EXISTS postgis.ugcs ( valid_to timestamptz ); CREATE INDEX IF NOT EXISTS ugc_ugc ON postgis.ugcs(ugc); -CREATE INDEX IF NOT EXISTS ugc_geom ON postgis.ugcs USING GIST(geom); \ No newline at end of file +CREATE INDEX IF NOT EXISTS ugc_geom ON postgis.ugcs USING GIST(geom); +ALTER TABLE postgis.ugcs OWNER TO mds; +GRANT ALL ON TABLE postgis.ugcs TO postgis; +GRANT SELECT ON TABLE postgis.ugcs TO nobody, api_service; \ No newline at end of file diff --git a/database/schemas/vtec.sql b/database/schemas/vtec.sql index fa00734..63a93e3 100644 --- a/database/schemas/vtec.sql +++ b/database/schemas/vtec.sql @@ -1,4 +1,5 @@ CREATE SCHEMA IF NOT EXISTS vtec; +ALTER SCHEMA vtec OWNER TO mds; -- Phenomena types CREATE TABLE IF NOT EXISTS vtec.phenomena ( @@ -6,6 +7,9 @@ CREATE TABLE IF NOT EXISTS vtec.phenomena ( name varchar(64) NOT NULL, description varchar(64) ); +ALTER TABLE vtec.phenomena OWNER TO mds; +GRANT SELECT ON TABLE vtec.phenomena TO awips_service; +GRANT SELECT ON TABLE vtec.phenomena TO nobody, api_service; -- Significance levels CREATE TABLE IF NOT EXISTS vtec.significance ( @@ -13,6 +17,9 @@ CREATE TABLE IF NOT EXISTS vtec.significance ( name varchar(64) NOT NULL, description varchar(64) ); +ALTER TABLE vtec.significance OWNER TO mds; +GRANT SELECT ON TABLE vtec.significance TO awips_service; +GRANT SELECT ON TABLE vtec.significance TO nobody, api_service; -- Action types CREATE TABLE IF NOT EXISTS vtec.action ( @@ -20,6 +27,9 @@ CREATE TABLE IF NOT EXISTS vtec.action ( name varchar(64) NOT NULL, description varchar(64) ); +ALTER TABLE vtec.action OWNER TO mds; +GRANT SELECT ON TABLE vtec.action TO awips_service; +GRANT SELECT ON TABLE vtec.action TO nobody, api_service; -- VTEC Event -- CREATE TABLE IF NOT EXISTS vtec.events ( @@ -42,6 +52,9 @@ CREATE TABLE IF NOT EXISTS vtec.events ( is_pds boolean DEFAULT false, PRIMARY KEY (wfo, phenomena, significance, event_number, year) ) PARTITION BY LIST (year); +ALTER TABLE vtec.events OWNER TO mds; +GRANT ALL ON TABLE vtec.events TO awips_service; +GRANT SELECT ON TABLE vtec.events TO nobody, api_service; -- VTEC UGC -- CREATE TABLE IF NOT EXISTS vtec.ugcs ( @@ -64,6 +77,9 @@ CREATE TABLE IF NOT EXISTS vtec.ugcs ( REFERENCES vtec.events(wfo, phenomena, significance, event_number, year) ON DELETE CASCADE, PRIMARY KEY (wfo, phenomena, significance, event_number, year, ugc) ) PARTITION BY LIST (year); +ALTER TABLE vtec.ugcs OWNER TO mds; +GRANT ALL ON TABLE vtec.ugcs TO awips_service; +GRANT SELECT ON TABLE vtec.ugcs TO nobody, api_service; -- VTEC Event Updates -- CREATE TABLE IF NOT EXISTS vtec.updates ( @@ -109,45 +125,80 @@ CREATE TABLE IF NOT EXISTS vtec.updates ( FOREIGN KEY (wfo, phenomena, significance, event_number, year) REFERENCES vtec.events(wfo, phenomena, significance, event_number, year) ON DELETE CASCADE ) PARTITION BY LIST (year); +ALTER TABLE vtec.updates OWNER TO mds; +GRANT ALL ON TABLE vtec.updates TO awips_service; +GRANT SELECT ON TABLE vtec.updates TO nobody, api_service; -CREATE OR REPLACE FUNCTION vtec.CREATE_YEARLY_PARTITIONS (YEAR INTEGER) RETURNS VOID AS $$ +CREATE OR REPLACE FUNCTION vtec.CREATE_YEARLY_PARTITIONS (starts INTEGER, ends INTEGER) RETURNS VOID AS $$ BEGIN - -- VTEC Tables - PERFORM create_yearly_list_partition('vtec.events', year); - EXECUTE format(' - CREATE INDEX vtec_event_%s_issued ON vtec.events_%s(issued);', - year, year); - EXECUTE format(' - CREATE INDEX vtec_event_%s_starts ON vtec.events_%s(starts);', - year, year); - EXECUTE format(' - CREATE INDEX vtec_event_%s_expires ON vtec.events_%s(expires);', - year, year); - EXECUTE format(' - CREATE INDEX vtec_event_%s_ends ON vtec.events_%s(ends);', - year, year); - EXECUTE format(' - CREATE INDEX vtec_event_%s_phenomena_significance ON vtec.events_%s(phenomena, significance);', - year, year); - EXECUTE format(' - CREATE INDEX vtec_event_%s_is_emergency ON vtec.events_%s(is_emergency) WHERE is_emergency = true;', - year, year); - EXECUTE format(' - CREATE INDEX vtec_event_%s_is_pds ON vtec.events_%s(is_pds) WHERE is_pds = true;', - year, year); + FOR year IN starts..ends + LOOP + -- VTEC Tables + PERFORM create_yearly_list_partition('vtec.events', year); + EXECUTE format(' + CREATE INDEX vtec_event_%s_issued ON vtec.events_%s(issued);', + year, year); + EXECUTE format(' + CREATE INDEX vtec_event_%s_starts ON vtec.events_%s(starts);', + year, year); + EXECUTE format(' + CREATE INDEX vtec_event_%s_expires ON vtec.events_%s(expires);', + year, year); + EXECUTE format(' + CREATE INDEX vtec_event_%s_ends ON vtec.events_%s(ends);', + year, year); + EXECUTE format(' + CREATE INDEX vtec_event_%s_phenomena_significance ON vtec.events_%s(phenomena, significance);', + year, year); + EXECUTE format(' + CREATE INDEX vtec_event_%s_is_emergency ON vtec.events_%s(is_emergency) WHERE is_emergency = true;', + year, year); + EXECUTE format(' + CREATE INDEX vtec_event_%s_is_pds ON vtec.events_%s(is_pds) WHERE is_pds = true;', + year, year); + EXECUTE format(' + ALTER TABLE vtec.events_%s OWNER TO mds;', + year); + EXECUTE format(' + GRANT ALL ON TABLE vtec.events_%s TO awips_service;', + year); + EXECUTE format(' + GRANT SELECT ON TABLE vtec.events_%s TO nobody, api_service;', + year); - PERFORM create_yearly_list_partition('vtec.ugcs', year); - EXECUTE format(' - CREATE INDEX vtec_ugc_%s_ugc ON vtec.ugcs_%s(ugc);', - year, year); - EXECUTE format(' - CREATE INDEX vtec_ugc_%s_action ON vtec.ugcs_%s(action);', - year, year); + PERFORM create_yearly_list_partition('vtec.ugcs', year); + EXECUTE format(' + CREATE INDEX vtec_ugc_%s_ugc ON vtec.ugcs_%s(ugc);', + year, year); + EXECUTE format(' + CREATE INDEX vtec_ugc_%s_action ON vtec.ugcs_%s(action);', + year, year); + EXECUTE format(' + ALTER TABLE vtec.ugcs_%s OWNER TO mds;', + year); + EXECUTE format(' + GRANT ALL ON TABLE vtec.ugcs_%s TO awips_service;', + year); + EXECUTE format(' + GRANT SELECT ON TABLE vtec.ugcs_%s TO nobody, api_service;', + year); - PERFORM create_yearly_list_partition('vtec.updates', year); - EXECUTE format(' - CREATE INDEX vtec_update_%s_geom ON vtec.updates_%s USING GIST (geom);', - year, year); + PERFORM create_yearly_list_partition('vtec.updates', year); + EXECUTE format(' + CREATE INDEX vtec_update_%s_geom ON vtec.updates_%s USING GIST (geom);', + year, year); + EXECUTE format('ALTER TABLE vtec.updates_%s OWNER TO mds;', + year); + EXECUTE format('GRANT ALL ON TABLE vtec.updates_%s TO awips_service;', + year); + EXECUTE format('GRANT SELECT ON TABLE vtec.updates_%s TO nobody, api_service;', + year); + END LOOP; +END +$$ LANGUAGE PLPGSQL; +DO $$ +BEGIN + PERFORM vtec.CREATE_YEARLY_PARTITIONS(2020, 2030); END -$$ LANGUAGE PLPGSQL; \ No newline at end of file +$$; \ No newline at end of file diff --git a/database/schemas/warnings.sql b/database/schemas/warnings.sql index 583ac44..93638f9 100644 --- a/database/schemas/warnings.sql +++ b/database/schemas/warnings.sql @@ -1,4 +1,5 @@ CREATE SCHEMA IF NOT EXISTS warnings; +ALTER SCHEMA warnings OWNER TO mds; CREATE TABLE IF NOT EXISTS warnings.warnings ( id serial, @@ -45,4 +46,7 @@ CREATE INDEX IF NOT EXISTS warnings_issued ON warnings.warnings(issued); CREATE INDEX IF NOT EXISTS warnings_starts ON warnings.warnings(starts); CREATE INDEX IF NOT EXISTS warnings_expires ON warnings.warnings(expires); CREATE INDEX IF NOT EXISTS warnings_ends ON warnings.warnings(ends); -CREATE INDEX IF NOT EXISTS warnings_phenomena_significance ON warnings.warnings(phenomena, significance); \ No newline at end of file +CREATE INDEX IF NOT EXISTS warnings_phenomena_significance ON warnings.warnings(phenomena, significance); +ALTER TABLE warnings.warnings OWNER TO mds; +GRANT ALL ON TABLE warnings.warnings TO awips_service; +GRANT SELECT ON TABLE warnings.warnings TO nobody, api_service; \ No newline at end of file diff --git a/services/parse/awips/internal/server.go b/services/parse/awips/internal/server.go index ee46997..94eba81 100644 --- a/services/parse/awips/internal/server.go +++ b/services/parse/awips/internal/server.go @@ -142,7 +142,7 @@ func Server(logLevel zerolog.Level) { select { case <-secondTimer.C: monitor.Ready.Set(1) - + secondTimer.Reset(15 * time.Second) case <-minuteTimer.C: ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) @@ -161,6 +161,7 @@ func Server(logLevel zerolog.Level) { } cancel() + minuteTimer.Reset(time.Minute) } } }()