CMR-11422: Add parameter to only allow validate keyword usage by provider (Enforcement list)

common-app-lib/src/cmr/common_app/services/kms_lookup.clj

@@ -31,6 +31,10 @@
   (:import #_{:clj-kondo/ignore [:unused-import]}
            (clojure.lang ExceptionInfo)))
 
+;; ---------------------------------------------------------------------
+;; initializing kms cache keys
+;; ---------------------------------------------------------------------
+
 (def kms-short-name-cache-key
   "The key used to store the data generated from KMS into a short name index cache
   in the system hash cache map for fast lookups."
@@ -116,6 +120,11 @@
   in the system hash cache map for fast lookups."
   :kms-temporal-keywords-index)
 
+
+;; ---------------------------------------------------------------------
+;; Initializing cache instances for kms keywords
+;; ---------------------------------------------------------------------
+
 (def kms-cache-ttl
   "Time To Live value for KMS caches. nil means never expire."
   nil)
@@ -225,6 +234,10 @@
   []
   (create-kms-uuid-cache kms-temporal-keywords-cache-key))
 
+;; ---------------------------------------------------------------------
+;; Define rules for scheme lookups
+;; ---------------------------------------------------------------------
+
 (def kms-scheme->fields-for-umm-c-lookup
   "Maps the KMS keyword scheme to the list of fields that should be matched when
   comparing fields between KMS and UMM-C, UMM-G, UMM-S, UMM-T, or UMM-Var."
@@ -445,119 +458,53 @@
                                           [(normalize-for-lookup keyword-map fields)
                                            keyword-map])
                                         keyword-maps)))]])))
+;; ---------------------------------------------------------------------
+;; Create KMS indexes for cache
+;; ---------------------------------------------------------------------
+
+(def ^:private kms-index-definitions
+  "Pairs of [cache-key generate-fn] describing every KMS index. The generate-fn takes the
+   kms-keywords-map and returns the lookup map to store under that cache key. Adding a new
+   KMS index means adding one line here."
+  [[kms-short-name-cache-key           generate-lookup-by-short-name-map]
+   [kms-projects-cache-key             generate-lookup-by-project-name-map]
+   [kms-processing-level-cache-key     generate-lookup-by-processing-level-map]
+   [kms-umm-c-cache-key                generate-lookup-by-umm-c-map]
+   [kms-location-cache-key             generate-lookup-by-location-map]
+   [kms-measurement-cache-key          generate-lookup-by-measurement-name]
+   [kms-science-keywords-cache-key     generate-lookup-by-science-keywords-map]
+   [kms-platforms-cache-key            generate-lookup-by-platforms-name-map]
+   [kms-instruments-cache-key          generate-lookup-by-instruments-name-map]
+   [kms-providers-cache-key            generate-lookup-by-providers-name-map]
+   [kms-spatial-keywords-cache-key     generate-lookup-by-spatial-keywords-map]
+   [kms-concepts-cache-key             generate-lookup-by-concepts-name-map]
+   [kms-iso-topic-categories-cache-key generate-lookup-by-iso-topic-categories-name-map]
+   [kms-granule-data-format-cache-key  generate-lookup-by-granule-data-format-name-map]
+   [kms-mime-type-cache-key            generate-lookup-by-mime-type-name-map]
+   [kms-related-urls-cache-key         generate-lookup-by-related-urls-map]
+   [kms-temporal-keywords-cache-key    generate-lookup-by-temporal-keywords-name-map]])
+
+(defn- store-kms-index!
+  "Writes a single KMS lookup map to redis under cache-key. No-ops on an empty map so we
+   never clobber a populated cache with nothing."
+  [context cache-key lookup-map]
+  (when (seq lookup-map)
+    (let [cache (hash-cache/context->cache context cache-key)
+          [tm _] (util/time-execution (hash-cache/set-values cache cache-key lookup-map))]
+      (rl-util/log-redis-write-complete "create-kms-index" cache-key tm))))
 
-;; Candidate to make private as this function is currently used very little outside of the
-;; common_app package (kms_lookup.clj) and several tests. Try to not call this function in any
-;; actual code to limit how many apps directly manage KMS cache.
 (defn create-kms-index
   "Creates the KMS index structure to be used for fast lookups and stores these values in
    redis. Calling this function will CHANGE an external resource."
   [context kms-keywords-map]
-  (let [short-name-lookup-map (generate-lookup-by-short-name-map kms-keywords-map)
-        project-uuid-lookup-map (generate-lookup-by-project-name-map kms-keywords-map)
-        processing-level-uuid-lookup-map (generate-lookup-by-processing-level-map kms-keywords-map)
-        umm-c-lookup-map (generate-lookup-by-umm-c-map kms-keywords-map)
-        location-lookup-map (generate-lookup-by-location-map kms-keywords-map)
-        measurement-lookup-map (generate-lookup-by-measurement-name kms-keywords-map)
-        science-keywords-uuid-lookup-map (generate-lookup-by-science-keywords-map kms-keywords-map)
-        platforms-uuid-lookup-map (generate-lookup-by-platforms-name-map kms-keywords-map)
-        instruments-uuid-lookup-map (generate-lookup-by-instruments-name-map kms-keywords-map)
-        providers-uuid-lookup-map (generate-lookup-by-providers-name-map kms-keywords-map)
-        spatial-keywords-uuid-lookup-map (generate-lookup-by-spatial-keywords-map kms-keywords-map)
-        concepts-uuid-lookup-map (generate-lookup-by-concepts-name-map kms-keywords-map)
-        iso-topic-categories-uuid-lookup-map (generate-lookup-by-iso-topic-categories-name-map kms-keywords-map)
-        granule-data-format-uuid-lookup-map (generate-lookup-by-granule-data-format-name-map kms-keywords-map)
-        mime-type-uuid-lookup-map (generate-lookup-by-mime-type-name-map kms-keywords-map)
-        related-urls-uuid-lookup-map (generate-lookup-by-related-urls-map kms-keywords-map)
-        temporal-keywords-uuid-lookup-map (generate-lookup-by-temporal-keywords-name-map kms-keywords-map)
-        project-cache (hash-cache/context->cache context kms-projects-cache-key)
-        processing-level-cache (hash-cache/context->cache context kms-processing-level-cache-key)
-        short-name-cache (hash-cache/context->cache context kms-short-name-cache-key)
-        umm-c-cache (hash-cache/context->cache context kms-umm-c-cache-key)
-        location-cache (hash-cache/context->cache context kms-location-cache-key)
-        measurement-cache (hash-cache/context->cache context kms-measurement-cache-key)
-        science-keywords-cache (hash-cache/context->cache context kms-science-keywords-cache-key)
-        platforms-cache (hash-cache/context->cache context kms-platforms-cache-key)
-        instruments-cache (hash-cache/context->cache context kms-instruments-cache-key)
-        providers-cache (hash-cache/context->cache context kms-providers-cache-key)
-        spatial-keywords-cache (hash-cache/context->cache context kms-spatial-keywords-cache-key)
-        concepts-cache (hash-cache/context->cache context kms-concepts-cache-key)
-        iso-topic-categories-cache (hash-cache/context->cache context kms-iso-topic-categories-cache-key)
-        granule-data-format-cache (hash-cache/context->cache context kms-granule-data-format-cache-key)
-        mime-type-cache (hash-cache/context->cache context kms-mime-type-cache-key)
-        related-urls-cache (hash-cache/context->cache context kms-related-urls-cache-key)
-        temporal-keywords-cache (hash-cache/context->cache context kms-temporal-keywords-cache-key)
-        _ (rl-util/log-refresh-start (format "%s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s"
-                                             kms-short-name-cache-key
-                                             kms-umm-c-cache-key
-                                             kms-location-cache-key
-                                             kms-measurement-cache-key
-                                             kms-processing-level-cache-key
-                                             kms-science-keywords-cache-key
-                                             kms-platforms-cache-key
-                                             kms-instruments-cache-key
-                                             kms-providers-cache-key
-                                             kms-spatial-keywords-cache-key
-                                             kms-concepts-cache-key
-                                             kms-iso-topic-categories-cache-key
-                                             kms-granule-data-format-cache-key
-                                             kms-mime-type-cache-key
-                                             kms-related-urls-cache-key
-                                             kms-temporal-keywords-cache-key))]
-    ;; Only update caches that exist
-    (when-not (empty? short-name-lookup-map)
-      (let [[tm _] (util/time-execution (hash-cache/set-values short-name-cache kms-short-name-cache-key short-name-lookup-map))]
-        (rl-util/log-redis-write-complete "create-kms-index" kms-short-name-cache-key tm)))
-    (when-not (empty? project-uuid-lookup-map)
-      (let [[tm _] (util/time-execution (hash-cache/set-values project-cache kms-projects-cache-key project-uuid-lookup-map))]
-        (rl-util/log-redis-write-complete "create-kms-index" kms-projects-cache-key tm)))
-    (when-not (empty? processing-level-uuid-lookup-map)
-      (let [[tm _] (util/time-execution (hash-cache/set-values processing-level-cache kms-processing-level-cache-key processing-level-uuid-lookup-map))]
-        (rl-util/log-redis-write-complete "create-kms-index" kms-processing-level-cache-key tm)))
-    (when-not (empty? umm-c-lookup-map)
-      (let [[tm _] (util/time-execution (hash-cache/set-values umm-c-cache kms-umm-c-cache-key umm-c-lookup-map))]
-        (rl-util/log-redis-write-complete "create-kms-index" kms-umm-c-cache-key tm)))
-    (when-not (empty? location-lookup-map)
-      (let [[tm _] (util/time-execution (hash-cache/set-values location-cache kms-location-cache-key location-lookup-map))]
-        (rl-util/log-redis-write-complete "create-kms-index" kms-location-cache-key tm)))
-    (when-not (empty? measurement-lookup-map)
-      (let [[tm _] (util/time-execution (hash-cache/set-values measurement-cache kms-measurement-cache-key measurement-lookup-map))]
-        (rl-util/log-redis-write-complete "create-kms-index" kms-measurement-cache-key tm)))
-    (when-not (empty? science-keywords-uuid-lookup-map)
-      (let [[tm _] (util/time-execution (hash-cache/set-values science-keywords-cache kms-science-keywords-cache-key science-keywords-uuid-lookup-map))]
-        (rl-util/log-redis-write-complete "create-kms-index" kms-science-keywords-cache-key tm)))
-    (when-not (empty? platforms-uuid-lookup-map)
-      (let [[tm _] (util/time-execution (hash-cache/set-values platforms-cache kms-platforms-cache-key platforms-uuid-lookup-map))]
-        (rl-util/log-redis-write-complete "create-kms-index" kms-platforms-cache-key tm)))
-    (when-not (empty? instruments-uuid-lookup-map)
-      (let [[tm _] (util/time-execution (hash-cache/set-values instruments-cache kms-instruments-cache-key instruments-uuid-lookup-map))]
-        (rl-util/log-redis-write-complete "create-kms-index" kms-instruments-cache-key tm)))
-    (when-not (empty? providers-uuid-lookup-map)
-      (let [[tm _] (util/time-execution (hash-cache/set-values providers-cache kms-providers-cache-key providers-uuid-lookup-map))]
-        (rl-util/log-redis-write-complete "create-kms-index" kms-providers-cache-key tm)))
-    (when-not (empty? spatial-keywords-uuid-lookup-map)
-      (let [[tm _] (util/time-execution (hash-cache/set-values spatial-keywords-cache kms-spatial-keywords-cache-key spatial-keywords-uuid-lookup-map))]
-        (rl-util/log-redis-write-complete "create-kms-index" kms-spatial-keywords-cache-key tm)))
-    (when-not (empty? concepts-uuid-lookup-map)
-      (let [[tm _] (util/time-execution (hash-cache/set-values concepts-cache kms-concepts-cache-key concepts-uuid-lookup-map))]
-        (rl-util/log-redis-write-complete "create-kms-index" kms-concepts-cache-key tm)))
-    (when-not (empty? iso-topic-categories-uuid-lookup-map)
-      (let [[tm _] (util/time-execution (hash-cache/set-values iso-topic-categories-cache kms-iso-topic-categories-cache-key iso-topic-categories-uuid-lookup-map))]
-        (rl-util/log-redis-write-complete "create-kms-index" kms-iso-topic-categories-cache-key tm)))
-    (when-not (empty? granule-data-format-uuid-lookup-map)
-      (let [[tm _] (util/time-execution (hash-cache/set-values granule-data-format-cache kms-granule-data-format-cache-key granule-data-format-uuid-lookup-map))]
-        (rl-util/log-redis-write-complete "create-kms-index" kms-granule-data-format-cache-key tm)))
-    (when-not (empty? mime-type-uuid-lookup-map)
-      (let [[tm _] (util/time-execution (hash-cache/set-values mime-type-cache kms-mime-type-cache-key mime-type-uuid-lookup-map))]
-        (rl-util/log-redis-write-complete "create-kms-index" kms-mime-type-cache-key tm)))
-    (when-not (empty? related-urls-uuid-lookup-map)
-      (let [[tm _] (util/time-execution (hash-cache/set-values related-urls-cache kms-related-urls-cache-key related-urls-uuid-lookup-map))]
-        (rl-util/log-redis-write-complete "create-kms-index" kms-related-urls-cache-key tm)))
-    (when-not (empty? temporal-keywords-uuid-lookup-map)
-      (let [[tm _] (util/time-execution (hash-cache/set-values temporal-keywords-cache kms-temporal-keywords-cache-key temporal-keywords-uuid-lookup-map))]
-        (rl-util/log-redis-write-complete "create-kms-index" kms-temporal-keywords-cache-key tm)))
-    kms-keywords-map))
-
+  (rl-util/log-refresh-start (string/join " " (map first kms-index-definitions)))
+  (doseq [[cache-key generate-fn] kms-index-definitions]
+    (store-kms-index! context cache-key (generate-fn kms-keywords-map)))
+  kms-keywords-map)
+
+;; ---------------------------------------------------------------------
+;; Define scheme lookups
+;; ---------------------------------------------------------------------
 
 (defn- lookup-by-field
   "Generic lookup function for KMS caches that use a lowercased string as the key."

ingest-app/docs/api.md

@@ -173,6 +173,8 @@ The following fields are validated:
 
 **Note**: if cmr-validate-keywords header is not set explicitly, it will behave as if it was set to FALSE by default behind the scenes
 
+**Note**: cmr-validate-keywords is being deprecated by provider to prevent ESDIS providers from using it and thus putting stale keywords onto the system
+
 **Note**: that when multiple fields are present the combination of keywords are validated to match a known combination.
 
 **Note**: Among the validation fields above, [Platforms], [Instruments], [Projects], [Science Keywords], [Location Keywords] and [Data Centers] are also validated when the `Cmr-Validate-Keywords` header is not set to `true` except that  validation errors will be returned to users as warnings.

ingest-app/src/cmr/ingest/api/collections.clj

@@ -1,6 +1,7 @@
 (ns cmr.ingest.api.collections
   "Collection ingest functions in support of the ingest API."
   (:require
+   [clojure.string :as string]
    [cmr.acl.core :as acl]
    [cmr.common-app.api.enabled :as common-enabled]
    [cmr.common-app.api.launchpad-token-validation :as lt-validation]
@@ -21,12 +22,46 @@
   "Checks to see if the feature toggle for validate-keywords-default-true is enabled."
   (ingest-config/validate-keywords-default-true-enabled))
 
+(defn- normalize-providers
+  "Normalizes the raw config value into a set of trimmed, non-blank provider id strings.
+   Accepts a collection whose elements may each be a single id or a comma-separated
+   string of ids:
+     - [\"PROV1\" \"PROV2\"]     -> #{\"PROV1\" \"PROV2\"}
+     - [\"PROV1,PROV2\"]         -> #{\"PROV1\" \"PROV2\"}
+     - [\"PROV1\"]               -> #{\"PROV1\"}
+     - []                        -> #{}
+   This makes the module resilient to upstream config plumbing that may or may
+   not apply the defconfig :parser."
+  [provider-list]
+   (->> (if (string? provider-list) [provider-list] provider-list)
+       (mapcat #(string/split (str %) #","))
+       (map string/trim)
+       (remove string/blank?)
+       set))
+
+;; Cache the enforced-providers set. NOTE: this uses `delay`, which evaluates
+;; exactly once for the life of the JVM process.
+(def enforced-providers-cache
+  "Cache the enforced-providers set. NOTE: this uses `delay`, which evaluates
+   exactly once for the life of the JVM process."
+  (delay (normalize-providers (ingest-config/keyword-enforced-providers))))
+
+(defn provider-enforced?
+  [provider-id]
+  (contains? @enforced-providers-cache provider-id))
+
 (defn get-validation-options
   "Returns a map of validation options with boolean values"
-  [headers]
-  (let [validate-keywords-value (if validate-keywords-default-true-enabled?
-                                  (if (= "false" (get headers VALIDATE_KEYWORDS_HEADER)) false true)
-                                  (= "true" (get headers VALIDATE_KEYWORDS_HEADER)))]
+  [headers provider-id]
+  (let [validate-keywords-value
+        ;; There is a temporary carveout for the KMS API itself until we resolve the cache delay issue either by removing the CMR cache entirely
+        ;; or having KMS issue a cache refresh to CMR see CMR-11524.
+        (if (and (not (contains? headers SEND_KMS_METADATA_FIXER_HEADER))
+                 (provider-enforced? provider-id))
+          true
+          (if validate-keywords-default-true-enabled?
+            (if (= "false" (get headers VALIDATE_KEYWORDS_HEADER)) false true)
+            (= "true" (get headers VALIDATE_KEYWORDS_HEADER))))]
     {:validate-keywords? validate-keywords-value
      :validate-umm? (= "true" (get headers ENABLE_UMM_C_VALIDATION_HEADER))
      :test-existing-errors? (= "true" (get headers TESTING_EXISTING_ERRORS_HEADER))
@@ -36,7 +71,7 @@
   [provider-id native-id request]
   (let [{:keys [body content-type _params headers request-context]} request
         concept (api-core/body->concept! :collection provider-id native-id body content-type headers)
-        validation-options (get-validation-options headers)]
+        validation-options (get-validation-options headers provider-id)]
     (api-core/verify-provider-exists request-context provider-id)
     (info (format "Validating Collection %s from client %s"
                   (api-core/concept->loggable-string concept) (:client-id request-context)))
@@ -57,7 +92,7 @@
     (acl/verify-ingest-management-permission request-context :update :provider-object provider-id)
     (common-enabled/validate-write-enabled request-context "ingest")
     (let [concept (api-core/body->concept! :collection provider-id native-id body content-type headers)
-          validation-options (get-validation-options headers)
+          validation-options (get-validation-options headers provider-id)
           ;; Log the ingest attempt
           _ (info (format "Ingesting collection %s from client %s"
                           (api-core/concept->loggable-string concept)