From 15c0bfaf1a0b5d491786879784dd3a627d0398d6 Mon Sep 17 00:00:00 2001 From: Ed Olivares <34591886+eudoroolivares2016@users.noreply.github.com> Date: Thu, 27 Aug 2026 23:37:27 -0400 Subject: [PATCH 01/14] CMR-11422: Adding list of providers in the enforced keyword list --- ingest-app/src/cmr/ingest/api/collections.clj | 20 +++++++++++++------ ingest-app/src/cmr/ingest/config.clj | 9 +++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/ingest-app/src/cmr/ingest/api/collections.clj b/ingest-app/src/cmr/ingest/api/collections.clj index 93417b8361..044969b411 100644 --- a/ingest-app/src/cmr/ingest/api/collections.clj +++ b/ingest-app/src/cmr/ingest/api/collections.clj @@ -23,20 +23,28 @@ (defn get-validation-options "Returns a map of validation options with boolean values" - [headers] - (let [validate-keywords-value (if validate-keywords-default-true-enabled? - (if (= "false" (get headers VALIDATE_KEYWORDS_HEADER)) false true) - (= "true" (get headers VALIDATE_KEYWORDS_HEADER)))] + [headers provider-id] + (def p1 provider-id) + (let [validate-keywords-value (if (contains? (set (ingest-config/keyword-enforced-providers)) provider-id) + "true" + (if validate-keywords-default-true-enabled? + (if (= "false" (get headers VALIDATE_KEYWORDS_HEADER)) false true) + (= "true" (get headers VALIDATE_KEYWORDS_HEADER))))] {:validate-keywords? validate-keywords-value :validate-umm? (= "true" (get headers ENABLE_UMM_C_VALIDATION_HEADER)) :test-existing-errors? (= "true" (get headers TESTING_EXISTING_ERRORS_HEADER)) :send-metadata-fixer? (not= "false" (get headers SEND_KMS_METADATA_FIXER_HEADER))})) +(comment +(contains? (set (ingest-config/keyword-enforced-providers)) ["PROV1"]) +) + + (defn validate-collection [provider-id native-id request] (let [{:keys [body content-type _params headers request-context]} request concept (api-core/body->concept! :collection provider-id native-id body content-type headers) - validation-options (get-validation-options headers)] + validation-options (get-validation-options headers provider-id)] (api-core/verify-provider-exists request-context provider-id) (info (format "Validating Collection %s from client %s" (api-core/concept->loggable-string concept) (:client-id request-context))) @@ -57,7 +65,7 @@ (acl/verify-ingest-management-permission request-context :update :provider-object provider-id) (common-enabled/validate-write-enabled request-context "ingest") (let [concept (api-core/body->concept! :collection provider-id native-id body content-type headers) - validation-options (get-validation-options headers) + validation-options (get-validation-options headers provider-id) ;; Log the ingest attempt _ (info (format "Ingesting collection %s from client %s" (api-core/concept->loggable-string concept) diff --git a/ingest-app/src/cmr/ingest/config.clj b/ingest-app/src/cmr/ingest/config.clj index 7d2e6e25da..485fe3dab4 100644 --- a/ingest-app/src/cmr/ingest/config.clj +++ b/ingest-app/src/cmr/ingest/config.clj @@ -158,3 +158,12 @@ (defconfig ingest-subscription-enabled "This indicates whether or not ingest granule subscriptions are enabled." {:default true :type Boolean}) + +(defconfig keyword-enforced-providers + "A list of providers for which keyword validation is enforced on ingest. + Should be an array of provider ids for which keywords must be present/valid. + Example \"PROV1,PROV2\", would enforce keyword validation for PROV1 and PROV2. + If no providers should have keyword validation enforced, set to an empty array." + {:default [] + :parser #(map (comp keyword string/trim) (string/split % #","))}) + From 6be2d4246cee70ad9ad1a7db04d77b82bf4455d3 Mon Sep 17 00:00:00 2001 From: Ed Olivares <34591886+eudoroolivares2016@users.noreply.github.com> Date: Thu, 27 Aug 2026 23:54:17 -0400 Subject: [PATCH 02/14] CMR-11422: Fix import --- ingest-app/src/cmr/ingest/config.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/ingest-app/src/cmr/ingest/config.clj b/ingest-app/src/cmr/ingest/config.clj index 485fe3dab4..416f1ce1e9 100644 --- a/ingest-app/src/cmr/ingest/config.clj +++ b/ingest-app/src/cmr/ingest/config.clj @@ -1,6 +1,7 @@ (ns cmr.ingest.config "Contains functions to retrieve ingest specific configuration" (:require + [clojure.string :as string] [cmr.common-app.config :as common-config] [cmr.common.config :as cfg :refer [defconfig]] [cmr.message-queue.config :as queue-config] From e05c8bf9182786ea58be39714a552e331e132804 Mon Sep 17 00:00:00 2001 From: Ed Olivares <34591886+eudoroolivares2016@users.noreply.github.com> Date: Fri, 28 Aug 2026 00:06:09 -0400 Subject: [PATCH 03/14] CMR-11422: Add tests add escape hatch for kms-metadata-fixer --- ingest-app/src/cmr/ingest/api/collections.clj | 18 ++- .../test/cmr/ingest/api/collections_test.clj | 119 ++++++++++++++---- 2 files changed, 102 insertions(+), 35 deletions(-) diff --git a/ingest-app/src/cmr/ingest/api/collections.clj b/ingest-app/src/cmr/ingest/api/collections.clj index 044969b411..0c7082e7f7 100644 --- a/ingest-app/src/cmr/ingest/api/collections.clj +++ b/ingest-app/src/cmr/ingest/api/collections.clj @@ -24,22 +24,18 @@ (defn get-validation-options "Returns a map of validation options with boolean values" [headers provider-id] - (def p1 provider-id) - (let [validate-keywords-value (if (contains? (set (ingest-config/keyword-enforced-providers)) provider-id) - "true" - (if validate-keywords-default-true-enabled? - (if (= "false" (get headers VALIDATE_KEYWORDS_HEADER)) false true) - (= "true" (get headers VALIDATE_KEYWORDS_HEADER))))] + (let [validate-keywords-value + (if (and (not (contains? headers SEND_KMS_METADATA_FIXER_HEADER)) + (contains? (set (ingest-config/keyword-enforced-providers)) provider-id)) + true + (if validate-keywords-default-true-enabled? + (if (= "false" (get headers VALIDATE_KEYWORDS_HEADER)) false true) + (= "true" (get headers VALIDATE_KEYWORDS_HEADER))))] {:validate-keywords? validate-keywords-value :validate-umm? (= "true" (get headers ENABLE_UMM_C_VALIDATION_HEADER)) :test-existing-errors? (= "true" (get headers TESTING_EXISTING_ERRORS_HEADER)) :send-metadata-fixer? (not= "false" (get headers SEND_KMS_METADATA_FIXER_HEADER))})) -(comment -(contains? (set (ingest-config/keyword-enforced-providers)) ["PROV1"]) -) - - (defn validate-collection [provider-id native-id request] (let [{:keys [body content-type _params headers request-context]} request diff --git a/ingest-app/test/cmr/ingest/api/collections_test.clj b/ingest-app/test/cmr/ingest/api/collections_test.clj index fe02cd3ddd..2f09a926e4 100644 --- a/ingest-app/test/cmr/ingest/api/collections_test.clj +++ b/ingest-app/test/cmr/ingest/api/collections_test.clj @@ -7,7 +7,8 @@ VALIDATE_KEYWORDS_HEADER ENABLE_UMM_C_VALIDATION_HEADER TESTING_EXISTING_ERRORS_HEADER - SEND_KMS_METADATA_FIXER_HEADER]])) + SEND_KMS_METADATA_FIXER_HEADER]] + [cmr.ingest.config :as ingest-config])) ;; --------------------------------------------------------------------- ;; :validate-keywords? — default-true-enabled? = true @@ -17,19 +18,19 @@ (with-redefs [v/validate-keywords-default-true-enabled? true] (testing "header explicitly \"false\" -> false" (is (= false (:validate-keywords? - (get-validation-options {VALIDATE_KEYWORDS_HEADER "false"}))))) + (get-validation-options {VALIDATE_KEYWORDS_HEADER "false"} "PROV1"))))) (testing "header explicitly \"true\" -> true" (is (= true (:validate-keywords? - (get-validation-options {VALIDATE_KEYWORDS_HEADER "true"}))))) + (get-validation-options {VALIDATE_KEYWORDS_HEADER "true"} "PROV1"))))) (testing "header missing -> true (defaults on)" (is (= true (:validate-keywords? - (get-validation-options {}))))) + (get-validation-options {} "PROV1"))))) (testing "header present but garbage value -> true (defaults on)" (is (= true (:validate-keywords? - (get-validation-options {VALIDATE_KEYWORDS_HEADER "nope"}))))))) + (get-validation-options {VALIDATE_KEYWORDS_HEADER "nope"} "PROV1"))))))) ;; --------------------------------------------------------------------- ;; :validate-keywords? — default-true-enabled? = false @@ -39,19 +40,88 @@ (with-redefs [v/validate-keywords-default-true-enabled? false] (testing "header explicitly \"true\" -> true" (is (= true (:validate-keywords? - (get-validation-options {VALIDATE_KEYWORDS_HEADER "true"}))))) + (get-validation-options {VALIDATE_KEYWORDS_HEADER "true"} "PROV1"))))) (testing "header explicitly \"false\" -> false" (is (= false (:validate-keywords? - (get-validation-options {VALIDATE_KEYWORDS_HEADER "false"}))))) + (get-validation-options {VALIDATE_KEYWORDS_HEADER "false"} "PROV1"))))) (testing "header missing -> false (defaults off)" (is (= false (:validate-keywords? - (get-validation-options {}))))) + (get-validation-options {} "PROV1"))))) (testing "header present but garbage value -> false (defaults off)" (is (= false (:validate-keywords? - (get-validation-options {VALIDATE_KEYWORDS_HEADER "nope"}))))))) + (get-validation-options {VALIDATE_KEYWORDS_HEADER "nope"} "PROV1"))))))) + +;; --------------------------------------------------------------------- +;; :validate-keywords? — provider is in keyword-enforced-providers, +;; SEND_KMS_METADATA_FIXER_HEADER absent -> enforced true +;; --------------------------------------------------------------------- +(deftest validate-keywords-enforced-provider-header-absent-test + (with-redefs [ingest-config/keyword-enforced-providers (constantly ["PROV1" "PROV2"])] + + (testing "enforced provider + fixer header absent + keywords header explicitly \"false\" -> still true" + (with-redefs [v/validate-keywords-default-true-enabled? true] + (is (= true (:validate-keywords? + (get-validation-options {VALIDATE_KEYWORDS_HEADER "false"} "PROV1")))))) + + (testing "enforced provider + fixer header absent, default-true-enabled? false -> still true" + (with-redefs [v/validate-keywords-default-true-enabled? false] + (is (= true (:validate-keywords? + (get-validation-options {VALIDATE_KEYWORDS_HEADER "false"} "PROV1")))))) + + (testing "enforced provider + no headers at all -> still true" + (with-redefs [v/validate-keywords-default-true-enabled? false] + (is (= true (:validate-keywords? + (get-validation-options {} "PROV1")))))) + + (testing "non-enforced provider, fixer header absent -> falls back to normal header logic" + (with-redefs [v/validate-keywords-default-true-enabled? true] + (is (= false (:validate-keywords? + (get-validation-options {VALIDATE_KEYWORDS_HEADER "false"} "PROV3")))))))) + +;; --------------------------------------------------------------------- +;; :validate-keywords? — provider is in keyword-enforced-providers, +;; SEND_KMS_METADATA_FIXER_HEADER present -> enforced list is bypassed, +;; normal header logic applies regardless of its value +;; --------------------------------------------------------------------- +(deftest validate-keywords-enforced-provider-header-present-test + (with-redefs [ingest-config/keyword-enforced-providers (constantly ["PROV1" "PROV2"])] + + (testing "enforced provider + fixer header present (\"true\") + keywords header \"false\" -> false (enforcement bypassed)" + (with-redefs [v/validate-keywords-default-true-enabled? true] + (is (= false (:validate-keywords? + (get-validation-options {VALIDATE_KEYWORDS_HEADER "false" + SEND_KMS_METADATA_FIXER_HEADER "true"} + "PROV1")))))) + + (testing "enforced provider + fixer header present (\"false\") + keywords header \"false\" -> false (enforcement bypassed)" + (with-redefs [v/validate-keywords-default-true-enabled? true] + (is (= false (:validate-keywords? + (get-validation-options {VALIDATE_KEYWORDS_HEADER "false" + SEND_KMS_METADATA_FIXER_HEADER "false"} + "PROV1")))))) + + (testing "enforced provider + fixer header present + no keywords header, default-true-enabled? true -> true (normal default, not enforcement)" + (with-redefs [v/validate-keywords-default-true-enabled? true] + (is (= true (:validate-keywords? + (get-validation-options {SEND_KMS_METADATA_FIXER_HEADER "true"} "PROV1")))))) + + (testing "enforced provider + fixer header present + no keywords header, default-true-enabled? false -> false (normal default, not enforcement)" + (with-redefs [v/validate-keywords-default-true-enabled? false] + (is (= false (:validate-keywords? + (get-validation-options {SEND_KMS_METADATA_FIXER_HEADER "true"} "PROV1")))))))) + +;; --------------------------------------------------------------------- +;; :validate-keywords? — empty keyword-enforced-providers config +;; --------------------------------------------------------------------- +(deftest validate-keywords-no-enforced-providers-test + (with-redefs [ingest-config/keyword-enforced-providers (constantly []) + v/validate-keywords-default-true-enabled? true] + (testing "no enforced providers configured -> normal header logic applies" + (is (= false (:validate-keywords? + (get-validation-options {VALIDATE_KEYWORDS_HEADER "false"} "PROV1"))))))) ;; --------------------------------------------------------------------- ;; :validate-umm? — defaults to false, only "true" turns it on @@ -59,18 +129,18 @@ (deftest validate-umm-test (testing "header \"true\" -> true" (is (= true (:validate-umm? - (get-validation-options {ENABLE_UMM_C_VALIDATION_HEADER "true"}))))) + (get-validation-options {ENABLE_UMM_C_VALIDATION_HEADER "true"} "PROV1"))))) (testing "header missing -> false" - (is (= false (:validate-umm? (get-validation-options {}))))) + (is (= false (:validate-umm? (get-validation-options {} "PROV1"))))) (testing "header \"false\" -> false" (is (= false (:validate-umm? - (get-validation-options {ENABLE_UMM_C_VALIDATION_HEADER "false"}))))) + (get-validation-options {ENABLE_UMM_C_VALIDATION_HEADER "false"} "PROV1"))))) (testing "header garbage value -> false" (is (= false (:validate-umm? - (get-validation-options {ENABLE_UMM_C_VALIDATION_HEADER "yes"})))))) + (get-validation-options {ENABLE_UMM_C_VALIDATION_HEADER "yes"} "PROV1")))))) ;; --------------------------------------------------------------------- ;; :test-existing-errors? — defaults to false, only "true" turns it on @@ -78,37 +148,37 @@ (deftest test-existing-errors-test (testing "header \"true\" -> true" (is (= true (:test-existing-errors? - (get-validation-options {TESTING_EXISTING_ERRORS_HEADER "true"}))))) + (get-validation-options {TESTING_EXISTING_ERRORS_HEADER "true"} "PROV1"))))) (testing "header missing -> false" - (is (= false (:test-existing-errors? (get-validation-options {}))))) + (is (= false (:test-existing-errors? (get-validation-options {} "PROV1"))))) (testing "header \"false\" -> false" (is (= false (:test-existing-errors? - (get-validation-options {TESTING_EXISTING_ERRORS_HEADER "false"}))))) + (get-validation-options {TESTING_EXISTING_ERRORS_HEADER "false"} "PROV1"))))) (testing "header garbage value -> false" (is (= false (:test-existing-errors? - (get-validation-options {TESTING_EXISTING_ERRORS_HEADER "yes"})))))) + (get-validation-options {TESTING_EXISTING_ERRORS_HEADER "yes"} "PROV1")))))) ;; --------------------------------------------------------------------- ;; :send-metadata-fixer? — defaults to true, only explicit "false" turns it off ;; --------------------------------------------------------------------- (deftest send-metadata-fixer-test (testing "header missing -> true (defaults on)" - (is (= true (:send-metadata-fixer? (get-validation-options {}))))) + (is (= true (:send-metadata-fixer? (get-validation-options {} "PROV1"))))) (testing "header \"true\" -> true" (is (= true (:send-metadata-fixer? - (get-validation-options {SEND_KMS_METADATA_FIXER_HEADER "true"}))))) + (get-validation-options {SEND_KMS_METADATA_FIXER_HEADER "true"} "PROV1"))))) (testing "header \"false\" -> false" (is (= false (:send-metadata-fixer? - (get-validation-options {SEND_KMS_METADATA_FIXER_HEADER "false"}))))) + (get-validation-options {SEND_KMS_METADATA_FIXER_HEADER "false"} "PROV1"))))) (testing "header garbage value -> true (anything other than \"false\" is on)" (is (= true (:send-metadata-fixer? - (get-validation-options {SEND_KMS_METADATA_FIXER_HEADER "nope"})))))) + (get-validation-options {SEND_KMS_METADATA_FIXER_HEADER "nope"} "PROV1")))))) ;; --------------------------------------------------------------------- ;; Combination / full-map tests @@ -120,7 +190,7 @@ :validate-umm? false :test-existing-errors? false :send-metadata-fixer? true} - (get-validation-options {}))))) + (get-validation-options {} "PROV1"))))) (testing "empty headers map, default-true-enabled? false -> keywords off, rest off/on defaults" (with-redefs [v/validate-keywords-default-true-enabled? false] @@ -128,7 +198,7 @@ :validate-umm? false :test-existing-errors? false :send-metadata-fixer? true} - (get-validation-options {}))))) + (get-validation-options {} "PROV1"))))) (testing "all headers explicitly set" (with-redefs [v/validate-keywords-default-true-enabled? true] @@ -140,4 +210,5 @@ {VALIDATE_KEYWORDS_HEADER "false" ENABLE_UMM_C_VALIDATION_HEADER "true" TESTING_EXISTING_ERRORS_HEADER "true" - SEND_KMS_METADATA_FIXER_HEADER "false"})))))) + SEND_KMS_METADATA_FIXER_HEADER "false"} + "PROV1")))))) From 9da947af1b6ec6181289e48f9d389047f45b4fe7 Mon Sep 17 00:00:00 2001 From: Ed Olivares <34591886+eudoroolivares2016@users.noreply.github.com> Date: Mon, 31 Aug 2026 12:33:02 -0400 Subject: [PATCH 04/14] CMR-11422: Refactor too much hardcoding function --- .../cmr/common_app/services/kms_lookup.clj | 171 +++++++----------- 1 file changed, 63 insertions(+), 108 deletions(-) diff --git a/common-app-lib/src/cmr/common_app/services/kms_lookup.clj b/common-app-lib/src/cmr/common_app/services/kms_lookup.clj index 69df1a41b3..d954d07386 100644 --- a/common-app-lib/src/cmr/common_app/services/kms_lookup.clj +++ b/common-app-lib/src/cmr/common_app/services/kms_lookup.clj @@ -31,6 +31,10 @@ (:import #_{:clj-kondo/ignore [:unused-import]} (clojure.lang ExceptionInfo))) +;; --------------------------------------------------------------------- +;; initializing kms cache keys +;; --------------------------------------------------------------------- + (def kms-short-name-cache-key "The key used to store the data generated from KMS into a short name index cache in the system hash cache map for fast lookups." @@ -116,6 +120,11 @@ in the system hash cache map for fast lookups." :kms-temporal-keywords-index) + +;; --------------------------------------------------------------------- +;; Initializing cache instances for kms keywords +;; --------------------------------------------------------------------- + (def kms-cache-ttl "Time To Live value for KMS caches. nil means never expire." nil) @@ -225,6 +234,10 @@ [] (create-kms-uuid-cache kms-temporal-keywords-cache-key)) +;; --------------------------------------------------------------------- +;; Define rules for scheme lookups +;; --------------------------------------------------------------------- + (def kms-scheme->fields-for-umm-c-lookup "Maps the KMS keyword scheme to the list of fields that should be matched when comparing fields between KMS and UMM-C, UMM-G, UMM-S, UMM-T, or UMM-Var." @@ -301,6 +314,7 @@ [keyword-scheme maps-by-short-name])) (select-keys gcmd-keywords-map keywords-to-lookup-by-short-name)))) +;; TODO remove this via any means necessary (def duplicate-keywords "Lookup table to account for any duplicate keywords. Will choose the preferred value. Common key is :uuid which is a field in the location-keyword map. " @@ -430,8 +444,15 @@ (defn- generate-lookup-by-related-urls-map "Create a map with the related url comparison map as keys to the UUID for that related url." [gcmd-keywords-map] + (def gkm gcmd-keywords-map) (generate-lookup-by-comparison-map gcmd-keywords-map :related-urls)) +(comment + +(tap> (generate-lookup-by-comparison-map gkm :related-urls)) + +) + (defn- generate-lookup-by-measurement-name "Create a map with the measurement field values defined in UMM-Var map to the KMS keywords." [gcmd-keywords-map] @@ -445,119 +466,53 @@ [(normalize-for-lookup keyword-map fields) keyword-map]) keyword-maps)))]]))) +;; --------------------------------------------------------------------- +;; Create KMS indexes for cache +;; --------------------------------------------------------------------- + +(def ^:private kms-index-definitions + "Pairs of [cache-key generate-fn] describing every KMS index. The generate-fn takes the + kms-keywords-map and returns the lookup map to store under that cache key. Adding a new + KMS index means adding one line here." + [[kms-short-name-cache-key generate-lookup-by-short-name-map] + [kms-projects-cache-key generate-lookup-by-project-name-map] + [kms-processing-level-cache-key generate-lookup-by-processing-level-map] + [kms-umm-c-cache-key generate-lookup-by-umm-c-map] + [kms-location-cache-key generate-lookup-by-location-map] + [kms-measurement-cache-key generate-lookup-by-measurement-name] + [kms-science-keywords-cache-key generate-lookup-by-science-keywords-map] + [kms-platforms-cache-key generate-lookup-by-platforms-name-map] + [kms-instruments-cache-key generate-lookup-by-instruments-name-map] + [kms-providers-cache-key generate-lookup-by-providers-name-map] + [kms-spatial-keywords-cache-key generate-lookup-by-spatial-keywords-map] + [kms-concepts-cache-key generate-lookup-by-concepts-name-map] + [kms-iso-topic-categories-cache-key generate-lookup-by-iso-topic-categories-name-map] + [kms-granule-data-format-cache-key generate-lookup-by-granule-data-format-name-map] + [kms-mime-type-cache-key generate-lookup-by-mime-type-name-map] + [kms-related-urls-cache-key generate-lookup-by-related-urls-map] + [kms-temporal-keywords-cache-key generate-lookup-by-temporal-keywords-name-map]]) + +(defn- store-kms-index! + "Writes a single KMS lookup map to redis under cache-key. No-ops on an empty map so we + never clobber a populated cache with nothing." + [context cache-key lookup-map] + (when (seq lookup-map) + (let [cache (hash-cache/context->cache context cache-key) + [tm _] (util/time-execution (hash-cache/set-values cache cache-key lookup-map))] + (rl-util/log-redis-write-complete "create-kms-index" cache-key tm)))) -;; Candidate to make private as this function is currently used very little outside of the -;; common_app package (kms_lookup.clj) and several tests. Try to not call this function in any -;; actual code to limit how many apps directly manage KMS cache. (defn create-kms-index "Creates the KMS index structure to be used for fast lookups and stores these values in redis. Calling this function will CHANGE an external resource." [context kms-keywords-map] - (let [short-name-lookup-map (generate-lookup-by-short-name-map kms-keywords-map) - project-uuid-lookup-map (generate-lookup-by-project-name-map kms-keywords-map) - processing-level-uuid-lookup-map (generate-lookup-by-processing-level-map kms-keywords-map) - umm-c-lookup-map (generate-lookup-by-umm-c-map kms-keywords-map) - location-lookup-map (generate-lookup-by-location-map kms-keywords-map) - measurement-lookup-map (generate-lookup-by-measurement-name kms-keywords-map) - science-keywords-uuid-lookup-map (generate-lookup-by-science-keywords-map kms-keywords-map) - platforms-uuid-lookup-map (generate-lookup-by-platforms-name-map kms-keywords-map) - instruments-uuid-lookup-map (generate-lookup-by-instruments-name-map kms-keywords-map) - providers-uuid-lookup-map (generate-lookup-by-providers-name-map kms-keywords-map) - spatial-keywords-uuid-lookup-map (generate-lookup-by-spatial-keywords-map kms-keywords-map) - concepts-uuid-lookup-map (generate-lookup-by-concepts-name-map kms-keywords-map) - iso-topic-categories-uuid-lookup-map (generate-lookup-by-iso-topic-categories-name-map kms-keywords-map) - granule-data-format-uuid-lookup-map (generate-lookup-by-granule-data-format-name-map kms-keywords-map) - mime-type-uuid-lookup-map (generate-lookup-by-mime-type-name-map kms-keywords-map) - related-urls-uuid-lookup-map (generate-lookup-by-related-urls-map kms-keywords-map) - temporal-keywords-uuid-lookup-map (generate-lookup-by-temporal-keywords-name-map kms-keywords-map) - project-cache (hash-cache/context->cache context kms-projects-cache-key) - processing-level-cache (hash-cache/context->cache context kms-processing-level-cache-key) - short-name-cache (hash-cache/context->cache context kms-short-name-cache-key) - umm-c-cache (hash-cache/context->cache context kms-umm-c-cache-key) - location-cache (hash-cache/context->cache context kms-location-cache-key) - measurement-cache (hash-cache/context->cache context kms-measurement-cache-key) - science-keywords-cache (hash-cache/context->cache context kms-science-keywords-cache-key) - platforms-cache (hash-cache/context->cache context kms-platforms-cache-key) - instruments-cache (hash-cache/context->cache context kms-instruments-cache-key) - providers-cache (hash-cache/context->cache context kms-providers-cache-key) - spatial-keywords-cache (hash-cache/context->cache context kms-spatial-keywords-cache-key) - concepts-cache (hash-cache/context->cache context kms-concepts-cache-key) - iso-topic-categories-cache (hash-cache/context->cache context kms-iso-topic-categories-cache-key) - granule-data-format-cache (hash-cache/context->cache context kms-granule-data-format-cache-key) - mime-type-cache (hash-cache/context->cache context kms-mime-type-cache-key) - related-urls-cache (hash-cache/context->cache context kms-related-urls-cache-key) - temporal-keywords-cache (hash-cache/context->cache context kms-temporal-keywords-cache-key) - _ (rl-util/log-refresh-start (format "%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s" - kms-short-name-cache-key - kms-umm-c-cache-key - kms-location-cache-key - kms-measurement-cache-key - kms-processing-level-cache-key - kms-science-keywords-cache-key - kms-platforms-cache-key - kms-instruments-cache-key - kms-providers-cache-key - kms-spatial-keywords-cache-key - kms-concepts-cache-key - kms-iso-topic-categories-cache-key - kms-granule-data-format-cache-key - kms-mime-type-cache-key - kms-related-urls-cache-key - kms-temporal-keywords-cache-key))] - ;; Only update caches that exist - (when-not (empty? short-name-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values short-name-cache kms-short-name-cache-key short-name-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-short-name-cache-key tm))) - (when-not (empty? project-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values project-cache kms-projects-cache-key project-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-projects-cache-key tm))) - (when-not (empty? processing-level-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values processing-level-cache kms-processing-level-cache-key processing-level-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-processing-level-cache-key tm))) - (when-not (empty? umm-c-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values umm-c-cache kms-umm-c-cache-key umm-c-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-umm-c-cache-key tm))) - (when-not (empty? location-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values location-cache kms-location-cache-key location-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-location-cache-key tm))) - (when-not (empty? measurement-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values measurement-cache kms-measurement-cache-key measurement-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-measurement-cache-key tm))) - (when-not (empty? science-keywords-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values science-keywords-cache kms-science-keywords-cache-key science-keywords-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-science-keywords-cache-key tm))) - (when-not (empty? platforms-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values platforms-cache kms-platforms-cache-key platforms-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-platforms-cache-key tm))) - (when-not (empty? instruments-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values instruments-cache kms-instruments-cache-key instruments-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-instruments-cache-key tm))) - (when-not (empty? providers-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values providers-cache kms-providers-cache-key providers-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-providers-cache-key tm))) - (when-not (empty? spatial-keywords-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values spatial-keywords-cache kms-spatial-keywords-cache-key spatial-keywords-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-spatial-keywords-cache-key tm))) - (when-not (empty? concepts-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values concepts-cache kms-concepts-cache-key concepts-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-concepts-cache-key tm))) - (when-not (empty? iso-topic-categories-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values iso-topic-categories-cache kms-iso-topic-categories-cache-key iso-topic-categories-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-iso-topic-categories-cache-key tm))) - (when-not (empty? granule-data-format-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values granule-data-format-cache kms-granule-data-format-cache-key granule-data-format-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-granule-data-format-cache-key tm))) - (when-not (empty? mime-type-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values mime-type-cache kms-mime-type-cache-key mime-type-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-mime-type-cache-key tm))) - (when-not (empty? related-urls-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values related-urls-cache kms-related-urls-cache-key related-urls-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-related-urls-cache-key tm))) - (when-not (empty? temporal-keywords-uuid-lookup-map) - (let [[tm _] (util/time-execution (hash-cache/set-values temporal-keywords-cache kms-temporal-keywords-cache-key temporal-keywords-uuid-lookup-map))] - (rl-util/log-redis-write-complete "create-kms-index" kms-temporal-keywords-cache-key tm))) - kms-keywords-map)) - + (rl-util/log-refresh-start (string/join " " (map first kms-index-definitions))) + (doseq [[cache-key generate-fn] kms-index-definitions] + (store-kms-index! context cache-key (generate-fn kms-keywords-map))) + kms-keywords-map) + +;; --------------------------------------------------------------------- +;; Define scheme lookups +;; --------------------------------------------------------------------- (defn- lookup-by-field "Generic lookup function for KMS caches that use a lowercased string as the key." From 3d0df7cecebe5cb372b01ec14d216d2d5711dbdc Mon Sep 17 00:00:00 2001 From: Ed Olivares <34591886+eudoroolivares2016@users.noreply.github.com> Date: Mon, 31 Aug 2026 12:47:54 -0400 Subject: [PATCH 05/14] CMR-11422: Fix clj-kondo warnings --- ingest-app/src/cmr/ingest/validation/validation.clj | 3 --- 1 file changed, 3 deletions(-) diff --git a/ingest-app/src/cmr/ingest/validation/validation.clj b/ingest-app/src/cmr/ingest/validation/validation.clj index d9a61d0ef8..8721302349 100644 --- a/ingest-app/src/cmr/ingest/validation/validation.clj +++ b/ingest-app/src/cmr/ingest/validation/validation.clj @@ -1,9 +1,7 @@ (ns cmr.ingest.validation.validation "Provides functions to validate concept" (:require - [cheshire.core :as json] [clojure.data :as data] - [clojure.edn :as edn] [clojure.string :as string] [cmr.common-app.services.kms-lookup :as kms-lookup] [cmr.common.log :as log :refer (debug warn)] @@ -15,7 +13,6 @@ [cmr.ingest.services.messages :as msg] [cmr.ingest.validation.business-rule-validation :as bv] [cmr.transmit.config :as transmit-config] - [cmr.transmit.search :as transmit-search] [cmr.umm-spec.json-schema :as json-schema] [cmr.umm-spec.umm-json :as umm-json] [cmr.umm-spec.umm-spec-core :as umm-spec] From 52c3b014173e67eab3428070c24337d46a2391d6 Mon Sep 17 00:00:00 2001 From: Ed Olivares <34591886+eudoroolivares2016@users.noreply.github.com> Date: Mon, 31 Aug 2026 12:51:51 -0400 Subject: [PATCH 06/14] CMR-11422: Comment and documentation --- ingest-app/docs/api.md | 2 ++ ingest-app/src/cmr/ingest/api/collections.clj | 2 ++ 2 files changed, 4 insertions(+) diff --git a/ingest-app/docs/api.md b/ingest-app/docs/api.md index 927042ddff..1a519c96c3 100644 --- a/ingest-app/docs/api.md +++ b/ingest-app/docs/api.md @@ -173,6 +173,8 @@ The following fields are validated: **Note**: if cmr-validate-keywords header is not set explicitly, it will behave as if it was set to FALSE by default behind the scenes +**Note**: cmr-validate-keywords is being deprecated by provider to prevent ESDIS providers from using it and thus putting stale keywords onto the system + **Note**: that when multiple fields are present the combination of keywords are validated to match a known combination. **Note**: Among the validation fields above, [Platforms], [Instruments], [Projects], [Science Keywords], [Location Keywords] and [Data Centers] are also validated when the `Cmr-Validate-Keywords` header is not set to `true` except that validation errors will be returned to users as warnings. diff --git a/ingest-app/src/cmr/ingest/api/collections.clj b/ingest-app/src/cmr/ingest/api/collections.clj index 0c7082e7f7..a852107a04 100644 --- a/ingest-app/src/cmr/ingest/api/collections.clj +++ b/ingest-app/src/cmr/ingest/api/collections.clj @@ -25,6 +25,8 @@ "Returns a map of validation options with boolean values" [headers provider-id] (let [validate-keywords-value + ;; There is a temporary carveout for the KMS API itself until we resolve the cache delay issue either by removing the CMR cache entirely + ;; or having KMS issue a cache refresh to CMR see CMR-11524. (if (and (not (contains? headers SEND_KMS_METADATA_FIXER_HEADER)) (contains? (set (ingest-config/keyword-enforced-providers)) provider-id)) true From 0f1f7e2929db6762ef6811b79e5430278372faa0 Mon Sep 17 00:00:00 2001 From: Ed Olivares <34591886+eudoroolivares2016@users.noreply.github.com> Date: Tue, 1 Sep 2026 11:42:35 -0400 Subject: [PATCH 07/14] CMR-11422: PR feedback --- .../src/cmr/common_app/services/kms_lookup.clj | 7 ------- ingest-app/src/cmr/ingest/api/collections.clj | 13 +++++++++++-- ingest-app/src/cmr/ingest/config.clj | 6 ++++-- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/common-app-lib/src/cmr/common_app/services/kms_lookup.clj b/common-app-lib/src/cmr/common_app/services/kms_lookup.clj index d954d07386..a62f61262b 100644 --- a/common-app-lib/src/cmr/common_app/services/kms_lookup.clj +++ b/common-app-lib/src/cmr/common_app/services/kms_lookup.clj @@ -314,7 +314,6 @@ [keyword-scheme maps-by-short-name])) (select-keys gcmd-keywords-map keywords-to-lookup-by-short-name)))) -;; TODO remove this via any means necessary (def duplicate-keywords "Lookup table to account for any duplicate keywords. Will choose the preferred value. Common key is :uuid which is a field in the location-keyword map. " @@ -447,12 +446,6 @@ (def gkm gcmd-keywords-map) (generate-lookup-by-comparison-map gcmd-keywords-map :related-urls)) -(comment - -(tap> (generate-lookup-by-comparison-map gkm :related-urls)) - -) - (defn- generate-lookup-by-measurement-name "Create a map with the measurement field values defined in UMM-Var map to the KMS keywords." [gcmd-keywords-map] diff --git a/ingest-app/src/cmr/ingest/api/collections.clj b/ingest-app/src/cmr/ingest/api/collections.clj index a852107a04..b40bc56422 100644 --- a/ingest-app/src/cmr/ingest/api/collections.clj +++ b/ingest-app/src/cmr/ingest/api/collections.clj @@ -21,14 +21,23 @@ "Checks to see if the feature toggle for validate-keywords-default-true is enabled." (ingest-config/validate-keywords-default-true-enabled)) +;; Cache the enforced-providers set. NOTE: this uses `delay`, which evaluates +;; exactly once for the life of the JVM process. +(def enforced-providers-cache + (delay (set (ingest-config/keyword-enforced-providers)))) + +(defn provider-enforced? + [provider-id] + (contains? @enforced-providers-cache provider-id)) + (defn get-validation-options "Returns a map of validation options with boolean values" [headers provider-id] (let [validate-keywords-value - ;; There is a temporary carveout for the KMS API itself until we resolve the cache delay issue either by removing the CMR cache entirely + ;; There is a temporary carveout for the KMS API itself until we resolve the cache delay issue either by removing the CMR cache entirely ;; or having KMS issue a cache refresh to CMR see CMR-11524. (if (and (not (contains? headers SEND_KMS_METADATA_FIXER_HEADER)) - (contains? (set (ingest-config/keyword-enforced-providers)) provider-id)) + (provider-enforced? provider-id)) true (if validate-keywords-default-true-enabled? (if (= "false" (get headers VALIDATE_KEYWORDS_HEADER)) false true) diff --git a/ingest-app/src/cmr/ingest/config.clj b/ingest-app/src/cmr/ingest/config.clj index 416f1ce1e9..2cc991b2fc 100644 --- a/ingest-app/src/cmr/ingest/config.clj +++ b/ingest-app/src/cmr/ingest/config.clj @@ -166,5 +166,7 @@ Example \"PROV1,PROV2\", would enforce keyword validation for PROV1 and PROV2. If no providers should have keyword validation enforced, set to an empty array." {:default [] - :parser #(map (comp keyword string/trim) (string/split % #","))}) - + :parser #(->> (string/split % #",") + (remove string/blank?) + (map (comp keyword string/trim))) + }) From 1a93ed422634a05ed58825524fad94f279c4ccce Mon Sep 17 00:00:00 2001 From: Ed Olivares <34591886+eudoroolivares2016@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:44:09 -0400 Subject: [PATCH 08/14] CMR-11422: Fix unit test --- ingest-app/test/cmr/ingest/api/collections_test.clj | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ingest-app/test/cmr/ingest/api/collections_test.clj b/ingest-app/test/cmr/ingest/api/collections_test.clj index 2f09a926e4..358f0384a8 100644 --- a/ingest-app/test/cmr/ingest/api/collections_test.clj +++ b/ingest-app/test/cmr/ingest/api/collections_test.clj @@ -57,22 +57,23 @@ ;; --------------------------------------------------------------------- ;; :validate-keywords? — provider is in keyword-enforced-providers, ;; SEND_KMS_METADATA_FIXER_HEADER absent -> enforced true +;; Since we cache this value we need to redef the whole cache on these tests ;; --------------------------------------------------------------------- (deftest validate-keywords-enforced-provider-header-absent-test (with-redefs [ingest-config/keyword-enforced-providers (constantly ["PROV1" "PROV2"])] (testing "enforced provider + fixer header absent + keywords header explicitly \"false\" -> still true" - (with-redefs [v/validate-keywords-default-true-enabled? true] + (with-redefs [v/enforced-providers-cache (delay #{"PROV1" "PROV2"})] (is (= true (:validate-keywords? (get-validation-options {VALIDATE_KEYWORDS_HEADER "false"} "PROV1")))))) (testing "enforced provider + fixer header absent, default-true-enabled? false -> still true" - (with-redefs [v/validate-keywords-default-true-enabled? false] + (with-redefs [v/enforced-providers-cache (delay #{"PROV1" "PROV2"})] (is (= true (:validate-keywords? (get-validation-options {VALIDATE_KEYWORDS_HEADER "false"} "PROV1")))))) (testing "enforced provider + no headers at all -> still true" - (with-redefs [v/validate-keywords-default-true-enabled? false] + (with-redefs [v/enforced-providers-cache (delay #{"PROV1" "PROV2"})] (is (= true (:validate-keywords? (get-validation-options {} "PROV1")))))) From 4351b0f36c2b1e524ce964115189a95898139131 Mon Sep 17 00:00:00 2001 From: Ed Olivares <34591886+eudoroolivares2016@users.noreply.github.com> Date: Wed, 2 Sep 2026 18:56:36 -0400 Subject: [PATCH 09/14] CMR-11422: Accidentally left `keyword` in commit --- ingest-app/src/cmr/ingest/config.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ingest-app/src/cmr/ingest/config.clj b/ingest-app/src/cmr/ingest/config.clj index 2cc991b2fc..d4de950c7d 100644 --- a/ingest-app/src/cmr/ingest/config.clj +++ b/ingest-app/src/cmr/ingest/config.clj @@ -165,8 +165,8 @@ Should be an array of provider ids for which keywords must be present/valid. Example \"PROV1,PROV2\", would enforce keyword validation for PROV1 and PROV2. If no providers should have keyword validation enforced, set to an empty array." - {:default [] + {:default ["PROV1","PROV2"] :parser #(->> (string/split % #",") - (remove string/blank?) - (map (comp keyword string/trim))) + (remove string/blank?) + (map string/trim)) }) From 3ced734d207493a9252bfc78b46ad25b8ce921f9 Mon Sep 17 00:00:00 2001 From: Ed Olivares <34591886+eudoroolivares2016@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:59:45 -0400 Subject: [PATCH 10/14] CMR-11422: fix default --- ingest-app/src/cmr/ingest/config.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingest-app/src/cmr/ingest/config.clj b/ingest-app/src/cmr/ingest/config.clj index d4de950c7d..3c0745906f 100644 --- a/ingest-app/src/cmr/ingest/config.clj +++ b/ingest-app/src/cmr/ingest/config.clj @@ -165,7 +165,7 @@ Should be an array of provider ids for which keywords must be present/valid. Example \"PROV1,PROV2\", would enforce keyword validation for PROV1 and PROV2. If no providers should have keyword validation enforced, set to an empty array." - {:default ["PROV1","PROV2"] + {:default [] :parser #(->> (string/split % #",") (remove string/blank?) (map string/trim)) From 4d402eea5fd833c72781900a5f4cdd5a29763937 Mon Sep 17 00:00:00 2001 From: Ed Olivares <34591886+eudoroolivares2016@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:30:33 -0400 Subject: [PATCH 11/14] CMR-11422: Foolproof parameter store retrieval --- ingest-app/src/cmr/ingest/api/collections.clj | 22 ++++++++++- .../test/cmr/ingest/api/collections_test.clj | 39 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/ingest-app/src/cmr/ingest/api/collections.clj b/ingest-app/src/cmr/ingest/api/collections.clj index b40bc56422..4d9e7e55a8 100644 --- a/ingest-app/src/cmr/ingest/api/collections.clj +++ b/ingest-app/src/cmr/ingest/api/collections.clj @@ -1,6 +1,7 @@ (ns cmr.ingest.api.collections "Collection ingest functions in support of the ingest API." (:require + [clojure.string :as string] [cmr.acl.core :as acl] [cmr.common-app.api.enabled :as common-enabled] [cmr.common-app.api.launchpad-token-validation :as lt-validation] @@ -21,10 +22,29 @@ "Checks to see if the feature toggle for validate-keywords-default-true is enabled." (ingest-config/validate-keywords-default-true-enabled)) +(defn- normalize-providers + "Normalizes the raw config value into a set of trimmed, non-blank provider id strings. + Accepts a collection whose elements may each be a single id or a comma-separated + string of ids: + - [\"PROV1\" \"PROV2\"] -> #{\"PROV1\" \"PROV2\"} + - [\"PROV1,PROV2\"] -> #{\"PROV1\" \"PROV2\"} + - [\"PROV1\"] -> #{\"PROV1\"} + - [] -> #{} + This makes the module resilient to upstream config plumbing that may or may + not apply the defconfig :parser." + [provider-list] + (->> (if (string? provider-list) [provider-list] provider-list) + (mapcat #(string/split (str %) #",")) + (map string/trim) + (remove string/blank?) + set)) + ;; Cache the enforced-providers set. NOTE: this uses `delay`, which evaluates ;; exactly once for the life of the JVM process. (def enforced-providers-cache - (delay (set (ingest-config/keyword-enforced-providers)))) + "Cache the enforced-providers set. NOTE: this uses `delay`, which evaluates + exactly once for the life of the JVM process." + (delay (normalize-providers (ingest-config/keyword-enforced-providers)))) (defn provider-enforced? [provider-id] diff --git a/ingest-app/test/cmr/ingest/api/collections_test.clj b/ingest-app/test/cmr/ingest/api/collections_test.clj index 358f0384a8..7cfe08811f 100644 --- a/ingest-app/test/cmr/ingest/api/collections_test.clj +++ b/ingest-app/test/cmr/ingest/api/collections_test.clj @@ -213,3 +213,42 @@ TESTING_EXISTING_ERRORS_HEADER "true" SEND_KMS_METADATA_FIXER_HEADER "false"} "PROV1")))))) + +;; --------------------------------------------------------------------- +;; normalize-providers — directly exercise the shape-handling logic +;; that makes this module resilient to upstream parser variations. +;; --------------------------------------------------------------------- +(deftest normalize-providers-test + (testing "already-parsed collection of ids" + (is (= #{"PROV1" "PROV2"} (#'v/normalize-providers ["PROV1" "PROV2"])))) + + (testing "collection containing a single unsplit comma-string" + (is (= #{"PROV1" "PROV2"} (#'v/normalize-providers ["PROV1,PROV2"])))) + + (testing "bare comma-separated string" + (is (= #{"PROV1" "PROV2"} (#'v/normalize-providers "PROV1,PROV2")))) + + (testing "bare single-provider string" + (is (= #{"PROV1"} (#'v/normalize-providers "PROV1")))) + + (testing "single-element collection" + (is (= #{"PROV1"} (#'v/normalize-providers ["PROV1"])))) + + (testing "empty collection" + (is (= #{} (#'v/normalize-providers [])))) + + (testing "whitespace around ids is trimmed" + (is (= #{"PROV1" "PROV2"} (#'v/normalize-providers ["PROV1, PROV2"]))) + (is (= #{"PROV1" "PROV2"} (#'v/normalize-providers " PROV1 , PROV2 ")))) + + (testing "blank/empty entries from stray commas are dropped" + (is (= #{"PROV1" "PROV2"} (#'v/normalize-providers ["PROV1,,PROV2"]))) + (is (= #{"PROV1" "PROV2"} (#'v/normalize-providers "PROV1,PROV2,")))) + + (testing "duplicates collapse via set" + (is (= #{"PROV1"} (#'v/normalize-providers ["PROV1" "PROV1"]))) + (is (= #{"PROV1"} (#'v/normalize-providers "PROV1,PROV1")))) + + (testing "mixed collection (already-split ids and a comma-string) is flattened" + (is (= #{"PROV1" "PROV2" "PROV3"} + (#'v/normalize-providers ["PROV1" "PROV2,PROV3"]))))) From c90500db219d63d770275ef5366cf97d675fca44 Mon Sep 17 00:00:00 2001 From: Ed Olivares <34591886+eudoroolivares2016@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:07:34 -0400 Subject: [PATCH 12/14] CMR-11422: Extra debug line --- common-app-lib/src/cmr/common_app/services/kms_lookup.clj | 2 -- 1 file changed, 2 deletions(-) diff --git a/common-app-lib/src/cmr/common_app/services/kms_lookup.clj b/common-app-lib/src/cmr/common_app/services/kms_lookup.clj index a62f61262b..287e9837ec 100644 --- a/common-app-lib/src/cmr/common_app/services/kms_lookup.clj +++ b/common-app-lib/src/cmr/common_app/services/kms_lookup.clj @@ -443,8 +443,6 @@ (defn- generate-lookup-by-related-urls-map "Create a map with the related url comparison map as keys to the UUID for that related url." [gcmd-keywords-map] - (def gkm gcmd-keywords-map) - (generate-lookup-by-comparison-map gcmd-keywords-map :related-urls)) (defn- generate-lookup-by-measurement-name "Create a map with the measurement field values defined in UMM-Var map to the KMS keywords." From daf9b0382b02e89fceb22dc69fdc96a96647bd93 Mon Sep 17 00:00:00 2001 From: Ed Olivares <34591886+eudoroolivares2016@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:31:35 -0400 Subject: [PATCH 13/14] CMR-11422: fix formatting --- common-app-lib/src/cmr/common_app/services/kms_lookup.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/common-app-lib/src/cmr/common_app/services/kms_lookup.clj b/common-app-lib/src/cmr/common_app/services/kms_lookup.clj index 287e9837ec..ed813505c9 100644 --- a/common-app-lib/src/cmr/common_app/services/kms_lookup.clj +++ b/common-app-lib/src/cmr/common_app/services/kms_lookup.clj @@ -443,6 +443,7 @@ (defn- generate-lookup-by-related-urls-map "Create a map with the related url comparison map as keys to the UUID for that related url." [gcmd-keywords-map] + (generate-lookup-by-comparison-map gcmd-keywords-map :related-urls)) (defn- generate-lookup-by-measurement-name "Create a map with the measurement field values defined in UMM-Var map to the KMS keywords." From e7832d2522a418079e3a2e4d1869cd869b73979b Mon Sep 17 00:00:00 2001 From: Ed Olivares <34591886+eudoroolivares2016@users.noreply.github.com> Date: Fri, 4 Sep 2026 10:16:12 -0400 Subject: [PATCH 14/14] CMR-11422: format using cljfmt --- ingest-app/src/cmr/ingest/config.clj | 31 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/ingest-app/src/cmr/ingest/config.clj b/ingest-app/src/cmr/ingest/config.clj index 3c0745906f..b4e5a2b599 100644 --- a/ingest-app/src/cmr/ingest/config.clj +++ b/ingest-app/src/cmr/ingest/config.clj @@ -9,8 +9,8 @@ [cmr.oracle.connection :as conn])) (defconfig validate-keywords-default-true-enabled - "Flag for whether or not cmr-validate-keywords value is defaulted to true or false in backend when missing in ingest api headers" - {:default true :type Boolean}) + "Flag for whether or not cmr-validate-keywords value is defaulted to true or false in backend when missing in ingest api headers" + {:default true :type Boolean}) (defconfig progressive-update-enabled "Flag for whether or not collection progressive update is enabled." @@ -30,7 +30,7 @@ (defconfig granule-bulk-update-tasks-max-rows "The maximum number of rows from granule_bulk_update_tasks table that can be returned" - {:default 1000 + {:default 1000 :type Long}) (defconfig collection-bulk-update-enabled @@ -90,12 +90,12 @@ "Returns a db spec populated with config information that can be used to connect to oracle" [connection-pool-name] (conn/db-spec - connection-pool-name - (oracle-config/db-url) - (oracle-config/db-fcf-enabled) - (oracle-config/db-ons-config) - (ingest-username) - (ingest-password))) + connection-pool-name + (oracle-config/db-url) + (oracle-config/db-fcf-enabled) + (oracle-config/db-ons-config) + (ingest-username) + (ingest-password))) (defconfig ingest-queue-name "The queue containing provider events like 'index provider collections'." @@ -110,12 +110,12 @@ {:default "cmr_ingest.exchange"}) (defconfig provider-exchange-name - "The ingest exchange to which provider change and non-ingest messages are published." - {:default "cmr_ingest_provider.exchange"}) + "The ingest exchange to which provider change and non-ingest messages are published." + {:default "cmr_ingest_provider.exchange"}) (defconfig bulk-update-exchange-name - "The ingest exchange to which granule bulk update messages are published." - {:default "cmr_ingest_bulk_update.exchange"}) + "The ingest exchange to which granule bulk update messages are published." + {:default "cmr_ingest_bulk_update.exchange"}) (defconfig ingest-queue-listener-count "Number of worker threads to use for the queue listener for the provider queue" @@ -167,6 +167,5 @@ If no providers should have keyword validation enforced, set to an empty array." {:default [] :parser #(->> (string/split % #",") - (remove string/blank?) - (map string/trim)) - }) + (remove string/blank?) + (map string/trim))})