diff --git a/common-app-lib/src/cmr/common_app/services/kms_lookup.clj b/common-app-lib/src/cmr/common_app/services/kms_lookup.clj index 69df1a41b3..ed813505c9 100644 --- a/common-app-lib/src/cmr/common_app/services/kms_lookup.clj +++ b/common-app-lib/src/cmr/common_app/services/kms_lookup.clj @@ -31,6 +31,10 @@ (:import #_{:clj-kondo/ignore [:unused-import]} (clojure.lang ExceptionInfo))) +;; --------------------------------------------------------------------- +;; initializing kms cache keys +;; --------------------------------------------------------------------- + (def kms-short-name-cache-key "The key used to store the data generated from KMS into a short name index cache in the system hash cache map for fast lookups." @@ -116,6 +120,11 @@ in the system hash cache map for fast lookups." :kms-temporal-keywords-index) + +;; --------------------------------------------------------------------- +;; Initializing cache instances for kms keywords +;; --------------------------------------------------------------------- + (def kms-cache-ttl "Time To Live value for KMS caches. nil means never expire." nil) @@ -225,6 +234,10 @@ [] (create-kms-uuid-cache kms-temporal-keywords-cache-key)) +;; --------------------------------------------------------------------- +;; Define rules for scheme lookups +;; --------------------------------------------------------------------- + (def kms-scheme->fields-for-umm-c-lookup "Maps the KMS keyword scheme to the list of fields that should be matched when comparing fields between KMS and UMM-C, UMM-G, UMM-S, UMM-T, or UMM-Var." @@ -445,119 +458,53 @@ [(normalize-for-lookup keyword-map fields) keyword-map]) keyword-maps)))]]))) +;; --------------------------------------------------------------------- +;; Create KMS indexes for cache +;; --------------------------------------------------------------------- + +(def ^:private kms-index-definitions + "Pairs of [cache-key generate-fn] describing every KMS index. The generate-fn takes the + kms-keywords-map and returns the lookup map to store under that cache key. Adding a new + KMS index means adding one line here." + [[kms-short-name-cache-key generate-lookup-by-short-name-map] + [kms-projects-cache-key generate-lookup-by-project-name-map] + [kms-processing-level-cache-key generate-lookup-by-processing-level-map] + [kms-umm-c-cache-key generate-lookup-by-umm-c-map] + [kms-location-cache-key generate-lookup-by-location-map] + [kms-measurement-cache-key generate-lookup-by-measurement-name] + [kms-science-keywords-cache-key generate-lookup-by-science-keywords-map] + [kms-platforms-cache-key generate-lookup-by-platforms-name-map] + [kms-instruments-cache-key generate-lookup-by-instruments-name-map] + [kms-providers-cache-key generate-lookup-by-providers-name-map] + [kms-spatial-keywords-cache-key generate-lookup-by-spatial-keywords-map] + [kms-concepts-cache-key generate-lookup-by-concepts-name-map] + [kms-iso-topic-categories-cache-key generate-lookup-by-iso-topic-categories-name-map] + [kms-granule-data-format-cache-key generate-lookup-by-granule-data-format-name-map] + [kms-mime-type-cache-key generate-lookup-by-mime-type-name-map] + [kms-related-urls-cache-key generate-lookup-by-related-urls-map] + [kms-temporal-keywords-cache-key generate-lookup-by-temporal-keywords-name-map]]) + +(defn- store-kms-index! + "Writes a single KMS lookup map to redis under cache-key. No-ops on an empty map so we + never clobber a populated cache with nothing." + [context cache-key lookup-map] + (when (seq lookup-map) + (let [cache (hash-cache/context->cache context cache-key) + [tm _] (util/time-execution (hash-cache/set-values cache cache-key lookup-map))] + (rl-util/log-redis-write-complete "create-kms-index" cache-key tm)))) -;; Candidate to make private as this function is currently used very little outside of the -;; common_app package (kms_lookup.clj) and several tests. Try to not call this function in any -;; actual code to limit how many apps directly manage KMS cache. (defn create-kms-index "Creates the KMS index structure to be used for fast lookups and stores these values in redis. Calling this function will CHANGE an external resource." [context kms-keywords-map] - (let [short-name-lookup-map (generate-lookup-by-short-name-map kms-keywords-map) - project-uuid-lookup-map (generate-lookup-by-project-name-map kms-keywords-map) - processing-level-uuid-lookup-map (generate-lookup-by-processing-level-map kms-keywords-map) - umm-c-lookup-map (generate-lookup-by-umm-c-map kms-keywords-map) - location-lookup-map (generate-lookup-by-location-map kms-keywords-map) - measurement-lookup-map (generate-lookup-by-measurement-name kms-keywords-map) - science-keywords-uuid-lookup-map (generate-lookup-by-science-keywords-map kms-keywords-map) - platforms-uuid-lookup-map (generate-lookup-by-platforms-name-map kms-keywords-map) - instruments-uuid-lookup-map (generate-lookup-by-instruments-name-map kms-keywords-map) - providers-uuid-lookup-map (generate-lookup-by-providers-name-map kms-keywords-map) - spatial-keywords-uuid-lookup-map (generate-lookup-by-spatial-keywords-map kms-keywords-map) - concepts-uuid-lookup-map (generate-lookup-by-concepts-name-map kms-keywords-map) - iso-topic-categories-uuid-lookup-map (generate-lookup-by-iso-topic-categories-name-map kms-keywords-map) - granule-data-format-uuid-lookup-map (generate-lookup-by-granule-data-format-name-map kms-keywords-map) - mime-type-uuid-lookup-map (generate-lookup-by-mime-type-name-map kms-keywords-map) - related-urls-uuid-lookup-map (generate-lookup-by-related-urls-map kms-keywords-map) - temporal-keywords-uuid-lookup-map (generate-lookup-by-temporal-keywords-name-map kms-keywords-map) - project-cache (hash-cache/context->cache context kms-projects-cache-key) - processing-level-cache (hash-cache/context->cache context kms-processing-level-cache-key) - short-name-cache (hash-cache/context->cache context kms-short-name-cache-key) - umm-c-cache (hash-cache/context->cache context kms-umm-c-cache-key) - location-cache (hash-cache/context->cache context kms-location-cache-key) - measurement-cache (hash-cache/context->cache context kms-measurement-cache-key) - science-keywords-cache (hash-cache/context->cache context kms-science-keywords-cache-key) - platforms-cache (hash-cache/context->cache context kms-platforms-cache-key) - instruments-cache (hash-cache/context->cache context kms-instruments-cache-key) - providers-cache (hash-cache/context->cache context kms-providers-cache-key) - spatial-keywords-cache (hash-cache/context->cache context kms-spatial-keywords-cache-key) - concepts-cache (hash-cache/context->cache context kms-concepts-cache-key) - iso-topic-categories-cache (hash-cache/context->cache context kms-iso-topic-categories-cache-key) - granule-data-format-cache (hash-cache/context->cache context kms-granule-data-format-cache-key) - mime-type-cache (hash-cache/context->cache context kms-mime-type-cache-key) - related-urls-cache (hash-cache/context->cache context kms-related-urls-cache-key) - temporal-keywords-cache (hash-cache/context->cache context kms-temporal-keywords-cache-key) - _ (rl-util/log-refresh-start (format "%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s" - kms-short-name-cache-key - kms-umm-c-cache-key - kms-location-cache-key - kms-measurement-cache-key - kms-processing-level-cache-key - kms-science-keywords-cache-key - kms-platforms-cache-key - kms-instruments-cache-key - kms-providers-cache-key - kms-spatial-keywords-cache-key - kms-concepts-cache-key - kms-iso-topic-categories-cache-key - kms-granule-data-format-cache-key - kms-mime-type-cache-key - kms-related-urls-cache-key - kms-temporal-keywords-cache-key))] - ;; Only update caches that exist - (when-not (empty? short-name-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values short-name-cache kms-short-name-cache-key short-name-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-short-name-cache-key tm))) - (when-not (empty? project-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values project-cache kms-projects-cache-key project-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-projects-cache-key tm))) - (when-not (empty? processing-level-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values processing-level-cache kms-processing-level-cache-key processing-level-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-processing-level-cache-key tm))) - (when-not (empty? umm-c-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values umm-c-cache kms-umm-c-cache-key umm-c-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-umm-c-cache-key tm))) - (when-not (empty? location-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values location-cache kms-location-cache-key location-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-location-cache-key tm))) - (when-not (empty? measurement-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values measurement-cache kms-measurement-cache-key measurement-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-measurement-cache-key tm))) - (when-not (empty? science-keywords-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values science-keywords-cache kms-science-keywords-cache-key science-keywords-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-science-keywords-cache-key tm))) - (when-not (empty? platforms-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values platforms-cache kms-platforms-cache-key platforms-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-platforms-cache-key tm))) - (when-not (empty? instruments-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values instruments-cache kms-instruments-cache-key instruments-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-instruments-cache-key tm))) - (when-not (empty? providers-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values providers-cache kms-providers-cache-key providers-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-providers-cache-key tm))) - (when-not (empty? spatial-keywords-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values spatial-keywords-cache kms-spatial-keywords-cache-key spatial-keywords-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-spatial-keywords-cache-key tm))) - (when-not (empty? concepts-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values concepts-cache kms-concepts-cache-key concepts-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-concepts-cache-key tm))) - (when-not (empty? iso-topic-categories-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values iso-topic-categories-cache kms-iso-topic-categories-cache-key iso-topic-categories-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-iso-topic-categories-cache-key tm))) - (when-not (empty? granule-data-format-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values granule-data-format-cache kms-granule-data-format-cache-key granule-data-format-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-granule-data-format-cache-key tm))) - (when-not (empty? mime-type-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values mime-type-cache kms-mime-type-cache-key mime-type-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-mime-type-cache-key tm))) - (when-not (empty? related-urls-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values related-urls-cache kms-related-urls-cache-key related-urls-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-related-urls-cache-key tm))) - (when-not (empty? temporal-keywords-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values temporal-keywords-cache kms-temporal-keywords-cache-key temporal-keywords-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-temporal-keywords-cache-key tm))) - kms-keywords-map)) - + (rl-util/log-refresh-start (string/join " " (map first kms-index-definitions))) + (doseq [[cache-key generate-fn] kms-index-definitions] + (store-kms-index! context cache-key (generate-fn kms-keywords-map))) + kms-keywords-map) + +;; --------------------------------------------------------------------- +;; Define scheme lookups +;; --------------------------------------------------------------------- (defn- lookup-by-field "Generic lookup function for KMS caches that use a lowercased string as the key." diff --git a/ingest-app/docs/api.md b/ingest-app/docs/api.md index 927042ddff..1a519c96c3 100644 --- a/ingest-app/docs/api.md +++ b/ingest-app/docs/api.md @@ -173,6 +173,8 @@ The following fields are validated: **Note**: if cmr-validate-keywords header is not set explicitly, it will behave as if it was set to FALSE by default behind the scenes +**Note**: cmr-validate-keywords is being deprecated by provider to prevent ESDIS providers from using it and thus putting stale keywords onto the system + **Note**: that when multiple fields are present the combination of keywords are validated to match a known combination. **Note**: Among the validation fields above, [Platforms], [Instruments], [Projects], [Science Keywords], [Location Keywords] and [Data Centers] are also validated when the `Cmr-Validate-Keywords` header is not set to `true` except that validation errors will be returned to users as warnings. diff --git a/ingest-app/src/cmr/ingest/api/collections.clj b/ingest-app/src/cmr/ingest/api/collections.clj index 93417b8361..4d9e7e55a8 100644 --- a/ingest-app/src/cmr/ingest/api/collections.clj +++ b/ingest-app/src/cmr/ingest/api/collections.clj @@ -1,6 +1,7 @@ (ns cmr.ingest.api.collections "Collection ingest functions in support of the ingest API." (:require + [clojure.string :as string] [cmr.acl.core :as acl] [cmr.common-app.api.enabled :as common-enabled] [cmr.common-app.api.launchpad-token-validation :as lt-validation] @@ -21,12 +22,46 @@ "Checks to see if the feature toggle for validate-keywords-default-true is enabled." (ingest-config/validate-keywords-default-true-enabled)) +(defn- normalize-providers + "Normalizes the raw config value into a set of trimmed, non-blank provider id strings. + Accepts a collection whose elements may each be a single id or a comma-separated + string of ids: + - [\"PROV1\" \"PROV2\"] -> #{\"PROV1\" \"PROV2\"} + - [\"PROV1,PROV2\"] -> #{\"PROV1\" \"PROV2\"} + - [\"PROV1\"] -> #{\"PROV1\"} + - [] -> #{} + This makes the module resilient to upstream config plumbing that may or may + not apply the defconfig :parser." + [provider-list] + (->> (if (string? provider-list) [provider-list] provider-list) + (mapcat #(string/split (str %) #",")) + (map string/trim) + (remove string/blank?) + set)) + +;; Cache the enforced-providers set. NOTE: this uses `delay`, which evaluates +;; exactly once for the life of the JVM process. +(def enforced-providers-cache + "Cache the enforced-providers set. NOTE: this uses `delay`, which evaluates + exactly once for the life of the JVM process." + (delay (normalize-providers (ingest-config/keyword-enforced-providers)))) + +(defn provider-enforced? + [provider-id] + (contains? @enforced-providers-cache provider-id)) + (defn get-validation-options "Returns a map of validation options with boolean values" - [headers] - (let [validate-keywords-value (if validate-keywords-default-true-enabled? - (if (= "false" (get headers VALIDATE_KEYWORDS_HEADER)) false true) - (= "true" (get headers VALIDATE_KEYWORDS_HEADER)))] + [headers provider-id] + (let [validate-keywords-value + ;; There is a temporary carveout for the KMS API itself until we resolve the cache delay issue either by removing the CMR cache entirely + ;; or having KMS issue a cache refresh to CMR see CMR-11524. + (if (and (not (contains? headers SEND_KMS_METADATA_FIXER_HEADER)) + (provider-enforced? provider-id)) + true + (if validate-keywords-default-true-enabled? + (if (= "false" (get headers VALIDATE_KEYWORDS_HEADER)) false true) + (= "true" (get headers VALIDATE_KEYWORDS_HEADER))))] {:validate-keywords? validate-keywords-value :validate-umm? (= "true" (get headers ENABLE_UMM_C_VALIDATION_HEADER)) :test-existing-errors? (= "true" (get headers TESTING_EXISTING_ERRORS_HEADER)) @@ -36,7 +71,7 @@ [provider-id native-id request] (let [{:keys [body content-type _params headers request-context]} request concept (api-core/body->concept! :collection provider-id native-id body content-type headers) - validation-options (get-validation-options headers)] + validation-options (get-validation-options headers provider-id)] (api-core/verify-provider-exists request-context provider-id) (info (format "Validating Collection %s from client %s" (api-core/concept->loggable-string concept) (:client-id request-context))) @@ -57,7 +92,7 @@ (acl/verify-ingest-management-permission request-context :update :provider-object provider-id) (common-enabled/validate-write-enabled request-context "ingest") (let [concept (api-core/body->concept! :collection provider-id native-id body content-type headers) - validation-options (get-validation-options headers) + validation-options (get-validation-options headers provider-id) ;; Log the ingest attempt _ (info (format "Ingesting collection %s from client %s" (api-core/concept->loggable-string concept) diff --git a/ingest-app/src/cmr/ingest/config.clj b/ingest-app/src/cmr/ingest/config.clj index 7d2e6e25da..b4e5a2b599 100644 --- a/ingest-app/src/cmr/ingest/config.clj +++ b/ingest-app/src/cmr/ingest/config.clj @@ -1,6 +1,7 @@ (ns cmr.ingest.config "Contains functions to retrieve ingest specific configuration" (:require + [clojure.string :as string] [cmr.common-app.config :as common-config] [cmr.common.config :as cfg :refer [defconfig]] [cmr.message-queue.config :as queue-config] @@ -8,8 +9,8 @@ [cmr.oracle.connection :as conn])) (defconfig validate-keywords-default-true-enabled - "Flag for whether or not cmr-validate-keywords value is defaulted to true or false in backend when missing in ingest api headers" - {:default true :type Boolean}) + "Flag for whether or not cmr-validate-keywords value is defaulted to true or false in backend when missing in ingest api headers" + {:default true :type Boolean}) (defconfig progressive-update-enabled "Flag for whether or not collection progressive update is enabled." @@ -29,7 +30,7 @@ (defconfig granule-bulk-update-tasks-max-rows "The maximum number of rows from granule_bulk_update_tasks table that can be returned" - {:default 1000 + {:default 1000 :type Long}) (defconfig collection-bulk-update-enabled @@ -89,12 +90,12 @@ "Returns a db spec populated with config information that can be used to connect to oracle" [connection-pool-name] (conn/db-spec - connection-pool-name - (oracle-config/db-url) - (oracle-config/db-fcf-enabled) - (oracle-config/db-ons-config) - (ingest-username) - (ingest-password))) + connection-pool-name + (oracle-config/db-url) + (oracle-config/db-fcf-enabled) + (oracle-config/db-ons-config) + (ingest-username) + (ingest-password))) (defconfig ingest-queue-name "The queue containing provider events like 'index provider collections'." @@ -109,12 +110,12 @@ {:default "cmr_ingest.exchange"}) (defconfig provider-exchange-name - "The ingest exchange to which provider change and non-ingest messages are published." - {:default "cmr_ingest_provider.exchange"}) + "The ingest exchange to which provider change and non-ingest messages are published." + {:default "cmr_ingest_provider.exchange"}) (defconfig bulk-update-exchange-name - "The ingest exchange to which granule bulk update messages are published." - {:default "cmr_ingest_bulk_update.exchange"}) + "The ingest exchange to which granule bulk update messages are published." + {:default "cmr_ingest_bulk_update.exchange"}) (defconfig ingest-queue-listener-count "Number of worker threads to use for the queue listener for the provider queue" @@ -158,3 +159,13 @@ (defconfig ingest-subscription-enabled "This indicates whether or not ingest granule subscriptions are enabled." {:default true :type Boolean}) + +(defconfig keyword-enforced-providers + "A list of providers for which keyword validation is enforced on ingest. + Should be an array of provider ids for which keywords must be present/valid. + Example \"PROV1,PROV2\", would enforce keyword validation for PROV1 and PROV2. + If no providers should have keyword validation enforced, set to an empty array." + {:default [] + :parser #(->> (string/split % #",") + (remove string/blank?) + (map string/trim))}) diff --git a/ingest-app/src/cmr/ingest/validation/validation.clj b/ingest-app/src/cmr/ingest/validation/validation.clj index d9a61d0ef8..8721302349 100644 --- a/ingest-app/src/cmr/ingest/validation/validation.clj +++ b/ingest-app/src/cmr/ingest/validation/validation.clj @@ -1,9 +1,7 @@ (ns cmr.ingest.validation.validation "Provides functions to validate concept" (:require - [cheshire.core :as json] [clojure.data :as data] - [clojure.edn :as edn] [clojure.string :as string] [cmr.common-app.services.kms-lookup :as kms-lookup] [cmr.common.log :as log :refer (debug warn)] @@ -15,7 +13,6 @@ [cmr.ingest.services.messages :as msg] [cmr.ingest.validation.business-rule-validation :as bv] [cmr.transmit.config :as transmit-config] - [cmr.transmit.search :as transmit-search] [cmr.umm-spec.json-schema :as json-schema] [cmr.umm-spec.umm-json :as umm-json] [cmr.umm-spec.umm-spec-core :as umm-spec] diff --git a/ingest-app/test/cmr/ingest/api/collections_test.clj b/ingest-app/test/cmr/ingest/api/collections_test.clj index fe02cd3ddd..7cfe08811f 100644 --- a/ingest-app/test/cmr/ingest/api/collections_test.clj +++ b/ingest-app/test/cmr/ingest/api/collections_test.clj @@ -7,7 +7,8 @@ VALIDATE_KEYWORDS_HEADER ENABLE_UMM_C_VALIDATION_HEADER TESTING_EXISTING_ERRORS_HEADER - SEND_KMS_METADATA_FIXER_HEADER]])) + SEND_KMS_METADATA_FIXER_HEADER]] + [cmr.ingest.config :as ingest-config])) ;; --------------------------------------------------------------------- ;; :validate-keywords? — default-true-enabled? = true @@ -17,19 +18,19 @@ (with-redefs [v/validate-keywords-default-true-enabled? true] (testing "header explicitly \"false\" -> false" (is (= false (:validate-keywords? - (get-validation-options {VALIDATE_KEYWORDS_HEADER "false"}))))) + (get-validation-options {VALIDATE_KEYWORDS_HEADER "false"} "PROV1"))))) (testing "header explicitly \"true\" -> true" (is (= true (:validate-keywords? - (get-validation-options {VALIDATE_KEYWORDS_HEADER "true"}))))) + (get-validation-options {VALIDATE_KEYWORDS_HEADER "true"} "PROV1"))))) (testing "header missing -> true (defaults on)" (is (= true (:validate-keywords? - (get-validation-options {}))))) + (get-validation-options {} "PROV1"))))) (testing "header present but garbage value -> true (defaults on)" (is (= true (:validate-keywords? - (get-validation-options {VALIDATE_KEYWORDS_HEADER "nope"}))))))) + (get-validation-options {VALIDATE_KEYWORDS_HEADER "nope"} "PROV1"))))))) ;; --------------------------------------------------------------------- ;; :validate-keywords? — default-true-enabled? = false @@ -39,19 +40,89 @@ (with-redefs [v/validate-keywords-default-true-enabled? false] (testing "header explicitly \"true\" -> true" (is (= true (:validate-keywords? - (get-validation-options {VALIDATE_KEYWORDS_HEADER "true"}))))) + (get-validation-options {VALIDATE_KEYWORDS_HEADER "true"} "PROV1"))))) (testing "header explicitly \"false\" -> false" (is (= false (:validate-keywords? - (get-validation-options {VALIDATE_KEYWORDS_HEADER "false"}))))) + (get-validation-options {VALIDATE_KEYWORDS_HEADER "false"} "PROV1"))))) (testing "header missing -> false (defaults off)" (is (= false (:validate-keywords? - (get-validation-options {}))))) + (get-validation-options {} "PROV1"))))) (testing "header present but garbage value -> false (defaults off)" (is (= false (:validate-keywords? - (get-validation-options {VALIDATE_KEYWORDS_HEADER "nope"}))))))) + (get-validation-options {VALIDATE_KEYWORDS_HEADER "nope"} "PROV1"))))))) + +;; --------------------------------------------------------------------- +;; :validate-keywords? — provider is in keyword-enforced-providers, +;; SEND_KMS_METADATA_FIXER_HEADER absent -> enforced true +;; Since we cache this value we need to redef the whole cache on these tests +;; --------------------------------------------------------------------- +(deftest validate-keywords-enforced-provider-header-absent-test + (with-redefs [ingest-config/keyword-enforced-providers (constantly ["PROV1" "PROV2"])] + + (testing "enforced provider + fixer header absent + keywords header explicitly \"false\" -> still true" + (with-redefs [v/enforced-providers-cache (delay #{"PROV1" "PROV2"})] + (is (= true (:validate-keywords? + (get-validation-options {VALIDATE_KEYWORDS_HEADER "false"} "PROV1")))))) + + (testing "enforced provider + fixer header absent, default-true-enabled? false -> still true" + (with-redefs [v/enforced-providers-cache (delay #{"PROV1" "PROV2"})] + (is (= true (:validate-keywords? + (get-validation-options {VALIDATE_KEYWORDS_HEADER "false"} "PROV1")))))) + + (testing "enforced provider + no headers at all -> still true" + (with-redefs [v/enforced-providers-cache (delay #{"PROV1" "PROV2"})] + (is (= true (:validate-keywords? + (get-validation-options {} "PROV1")))))) + + (testing "non-enforced provider, fixer header absent -> falls back to normal header logic" + (with-redefs [v/validate-keywords-default-true-enabled? true] + (is (= false (:validate-keywords? + (get-validation-options {VALIDATE_KEYWORDS_HEADER "false"} "PROV3")))))))) + +;; --------------------------------------------------------------------- +;; :validate-keywords? — provider is in keyword-enforced-providers, +;; SEND_KMS_METADATA_FIXER_HEADER present -> enforced list is bypassed, +;; normal header logic applies regardless of its value +;; --------------------------------------------------------------------- +(deftest validate-keywords-enforced-provider-header-present-test + (with-redefs [ingest-config/keyword-enforced-providers (constantly ["PROV1" "PROV2"])] + + (testing "enforced provider + fixer header present (\"true\") + keywords header \"false\" -> false (enforcement bypassed)" + (with-redefs [v/validate-keywords-default-true-enabled? true] + (is (= false (:validate-keywords? + (get-validation-options {VALIDATE_KEYWORDS_HEADER "false" + SEND_KMS_METADATA_FIXER_HEADER "true"} + "PROV1")))))) + + (testing "enforced provider + fixer header present (\"false\") + keywords header \"false\" -> false (enforcement bypassed)" + (with-redefs [v/validate-keywords-default-true-enabled? true] + (is (= false (:validate-keywords? + (get-validation-options {VALIDATE_KEYWORDS_HEADER "false" + SEND_KMS_METADATA_FIXER_HEADER "false"} + "PROV1")))))) + + (testing "enforced provider + fixer header present + no keywords header, default-true-enabled? true -> true (normal default, not enforcement)" + (with-redefs [v/validate-keywords-default-true-enabled? true] + (is (= true (:validate-keywords? + (get-validation-options {SEND_KMS_METADATA_FIXER_HEADER "true"} "PROV1")))))) + + (testing "enforced provider + fixer header present + no keywords header, default-true-enabled? false -> false (normal default, not enforcement)" + (with-redefs [v/validate-keywords-default-true-enabled? false] + (is (= false (:validate-keywords? + (get-validation-options {SEND_KMS_METADATA_FIXER_HEADER "true"} "PROV1")))))))) + +;; --------------------------------------------------------------------- +;; :validate-keywords? — empty keyword-enforced-providers config +;; --------------------------------------------------------------------- +(deftest validate-keywords-no-enforced-providers-test + (with-redefs [ingest-config/keyword-enforced-providers (constantly []) + v/validate-keywords-default-true-enabled? true] + (testing "no enforced providers configured -> normal header logic applies" + (is (= false (:validate-keywords? + (get-validation-options {VALIDATE_KEYWORDS_HEADER "false"} "PROV1"))))))) ;; --------------------------------------------------------------------- ;; :validate-umm? — defaults to false, only "true" turns it on @@ -59,18 +130,18 @@ (deftest validate-umm-test (testing "header \"true\" -> true" (is (= true (:validate-umm? - (get-validation-options {ENABLE_UMM_C_VALIDATION_HEADER "true"}))))) + (get-validation-options {ENABLE_UMM_C_VALIDATION_HEADER "true"} "PROV1"))))) (testing "header missing -> false" - (is (= false (:validate-umm? (get-validation-options {}))))) + (is (= false (:validate-umm? (get-validation-options {} "PROV1"))))) (testing "header \"false\" -> false" (is (= false (:validate-umm? - (get-validation-options {ENABLE_UMM_C_VALIDATION_HEADER "false"}))))) + (get-validation-options {ENABLE_UMM_C_VALIDATION_HEADER "false"} "PROV1"))))) (testing "header garbage value -> false" (is (= false (:validate-umm? - (get-validation-options {ENABLE_UMM_C_VALIDATION_HEADER "yes"})))))) + (get-validation-options {ENABLE_UMM_C_VALIDATION_HEADER "yes"} "PROV1")))))) ;; --------------------------------------------------------------------- ;; :test-existing-errors? — defaults to false, only "true" turns it on @@ -78,37 +149,37 @@ (deftest test-existing-errors-test (testing "header \"true\" -> true" (is (= true (:test-existing-errors? - (get-validation-options {TESTING_EXISTING_ERRORS_HEADER "true"}))))) + (get-validation-options {TESTING_EXISTING_ERRORS_HEADER "true"} "PROV1"))))) (testing "header missing -> false" - (is (= false (:test-existing-errors? (get-validation-options {}))))) + (is (= false (:test-existing-errors? (get-validation-options {} "PROV1"))))) (testing "header \"false\" -> false" (is (= false (:test-existing-errors? - (get-validation-options {TESTING_EXISTING_ERRORS_HEADER "false"}))))) + (get-validation-options {TESTING_EXISTING_ERRORS_HEADER "false"} "PROV1"))))) (testing "header garbage value -> false" (is (= false (:test-existing-errors? - (get-validation-options {TESTING_EXISTING_ERRORS_HEADER "yes"})))))) + (get-validation-options {TESTING_EXISTING_ERRORS_HEADER "yes"} "PROV1")))))) ;; --------------------------------------------------------------------- ;; :send-metadata-fixer? — defaults to true, only explicit "false" turns it off ;; --------------------------------------------------------------------- (deftest send-metadata-fixer-test (testing "header missing -> true (defaults on)" - (is (= true (:send-metadata-fixer? (get-validation-options {}))))) + (is (= true (:send-metadata-fixer? (get-validation-options {} "PROV1"))))) (testing "header \"true\" -> true" (is (= true (:send-metadata-fixer? - (get-validation-options {SEND_KMS_METADATA_FIXER_HEADER "true"}))))) + (get-validation-options {SEND_KMS_METADATA_FIXER_HEADER "true"} "PROV1"))))) (testing "header \"false\" -> false" (is (= false (:send-metadata-fixer? - (get-validation-options {SEND_KMS_METADATA_FIXER_HEADER "false"}))))) + (get-validation-options {SEND_KMS_METADATA_FIXER_HEADER "false"} "PROV1"))))) (testing "header garbage value -> true (anything other than \"false\" is on)" (is (= true (:send-metadata-fixer? - (get-validation-options {SEND_KMS_METADATA_FIXER_HEADER "nope"})))))) + (get-validation-options {SEND_KMS_METADATA_FIXER_HEADER "nope"} "PROV1")))))) ;; --------------------------------------------------------------------- ;; Combination / full-map tests @@ -120,7 +191,7 @@ :validate-umm? false :test-existing-errors? false :send-metadata-fixer? true} - (get-validation-options {}))))) + (get-validation-options {} "PROV1"))))) (testing "empty headers map, default-true-enabled? false -> keywords off, rest off/on defaults" (with-redefs [v/validate-keywords-default-true-enabled? false] @@ -128,7 +199,7 @@ :validate-umm? false :test-existing-errors? false :send-metadata-fixer? true} - (get-validation-options {}))))) + (get-validation-options {} "PROV1"))))) (testing "all headers explicitly set" (with-redefs [v/validate-keywords-default-true-enabled? true] @@ -140,4 +211,44 @@ {VALIDATE_KEYWORDS_HEADER "false" ENABLE_UMM_C_VALIDATION_HEADER "true" TESTING_EXISTING_ERRORS_HEADER "true" - SEND_KMS_METADATA_FIXER_HEADER "false"})))))) + SEND_KMS_METADATA_FIXER_HEADER "false"} + "PROV1")))))) + +;; --------------------------------------------------------------------- +;; normalize-providers — directly exercise the shape-handling logic +;; that makes this module resilient to upstream parser variations. +;; --------------------------------------------------------------------- +(deftest normalize-providers-test + (testing "already-parsed collection of ids" + (is (= #{"PROV1" "PROV2"} (#'v/normalize-providers ["PROV1" "PROV2"])))) + + (testing "collection containing a single unsplit comma-string" + (is (= #{"PROV1" "PROV2"} (#'v/normalize-providers ["PROV1,PROV2"])))) + + (testing "bare comma-separated string" + (is (= #{"PROV1" "PROV2"} (#'v/normalize-providers "PROV1,PROV2")))) + + (testing "bare single-provider string" + (is (= #{"PROV1"} (#'v/normalize-providers "PROV1")))) + + (testing "single-element collection" + (is (= #{"PROV1"} (#'v/normalize-providers ["PROV1"])))) + + (testing "empty collection" + (is (= #{} (#'v/normalize-providers [])))) + + (testing "whitespace around ids is trimmed" + (is (= #{"PROV1" "PROV2"} (#'v/normalize-providers ["PROV1, PROV2"]))) + (is (= #{"PROV1" "PROV2"} (#'v/normalize-providers " PROV1 , PROV2 ")))) + + (testing "blank/empty entries from stray commas are dropped" + (is (= #{"PROV1" "PROV2"} (#'v/normalize-providers ["PROV1,,PROV2"]))) + (is (= #{"PROV1" "PROV2"} (#'v/normalize-providers "PROV1,PROV2,")))) + + (testing "duplicates collapse via set" + (is (= #{"PROV1"} (#'v/normalize-providers ["PROV1" "PROV1"]))) + (is (= #{"PROV1"} (#'v/normalize-providers "PROV1,PROV1")))) + + (testing "mixed collection (already-split ids and a comma-string) is flattened" + (is (= #{"PROV1" "PROV2" "PROV3"} + (#'v/normalize-providers ["PROV1" "PROV2,PROV3"])))))