Do not resolve params of irrelevant stages

docs/guides/parameters.md

@@ -68,6 +68,28 @@ Here is the priority used to resolve a `${param:XXX}` variable:
 
 This is especially useful in development when deploying to ephemeral stages (e.g. "feature-x"). The stage might not have any parameter, therefore it will default to the parameters set on the service. However, in other stages, like "prod", or "staging", you may override the service-level parameters with stage-level parameters to use values unique to that stage.
 
+## Resolution of other stages
+
+Variables set in `params.<stage>` sections that do not concern the current stage are not resolved. For example when deploying to `dev`, a `${ssm:/prod/secret}` variable set in `params.prod` is not fetched from SSM, and errors it may raise (missing permissions, missing environment variable, unknown variable source…) do not fail the command.
+
+Such variables are still resolved when they are explicitly referenced, e.g. with `${self:params.prod.domain}`.
+
+There are a few exceptions:
+
+- Params under `params.default` are always resolved, as they apply to every stage.
+- When a whole section is defined with a single variable (e.g. `params: ${file(./params.yml)}` or `params.prod: ${file(./prod-params.yml)}`), that variable itself is still resolved: the configuration schema requires these sections to be objects. Values nested in the result are however only resolved for the current stage.
+- Variable syntax errors are reported for all stages: a malformed variable in `params.prod` still fails the command when deploying to `dev`.
+
+One consequence is that `serverless print` displays these values unresolved, as they appear in `serverless.yml`:
+
+```yaml
+params:
+  dev:
+    domain: dev.myapp.com
+  prod:
+    domain: ${ssm:/myapp/prod/domain} # left as-is when deploying to "dev"
+```
+
 ## See also
 
 - [Referencing parameters](./variables.md#referencing-parameters) in the Variables guide for more on the `${param:XXX}` source.

lib/configuration/variables/defer-irrelevant-stage-params.js

@@ -0,0 +1,53 @@
+// Builds an `isPropertyDeferred` predicate which prevents eager resolution of `params.<stage>`
+// sections not concerning the effective stage. The "param" source only reads
+// `params.<currentStage>` and `params.default`, so resolving other sections at best wastes remote
+// calls (e.g. SSM) and at worst fails the command on errors of irrelevant stages.
+//
+// Deferred properties stay in `variablesMeta`, so explicit references (e.g.
+// `${self:params.prod.x}`) still resolve them on demand. Entries left deferred after the final
+// pass are dropped by the caller (see `scripts/serverless.js`), leaving their raw values in the
+// configuration.
+//
+// Known limitation: a whole section configured with a single variable (e.g.
+// `params.prod: ${file(./prod-params.yml)}`) is still resolved, as the configuration schema
+// requires it to be an object. Values nested in the resolved object are deferred as any other.
+
+'use strict';
+
+const isPlainObject = require('type/plain-object/is');
+const { hasOwn } = require('../../utils/safe-object');
+
+// `configuration` and `resolverConfiguration` are read live at each call (not snapshotted):
+// plugins may extend the configuration with new stages, and the effective CLI options object is
+// replaced between resolution passes
+module.exports = (configuration, resolverConfiguration) => (propertyPath) => {
+  if (!propertyPath.startsWith('params\0')) return false;
+  if (!isPlainObject(configuration.params)) return false;
+
+  const propertyPathKeys = propertyPath.split('\0');
+  // Never defer the `params.<stage>` section itself: it has to resolve for the configuration to
+  // validate (see "Known limitation" above)
+  if (propertyPathKeys.length < 3) return false;
+
+  const stageKey = propertyPathKeys[1];
+  if (stageKey === 'default') return false;
+
+  // Mirrors effective stage resolution of the "param" source
+  // (lib/configuration/variables/sources/instance-dependent/param.js)
+  const options = resolverConfiguration.options || {};
+  let stage = hasOwn(options, 'stage') && options.stage != null ? options.stage : null;
+  if (stage == null && isPlainObject(configuration.provider)) {
+    const configuredStage = configuration.provider.stage;
+    if (typeof configuredStage === 'string' || typeof configuredStage === 'number') {
+      stage = configuredStage;
+    }
+  }
+  if (stage == null) stage = 'dev';
+  stage = String(stage);
+  // The effective stage is not known yet (`provider.stage` is itself behind an unresolved
+  // variable). Deferring against it could strand params of the actual effective stage as never
+  // scheduled for resolution
+  if (stage.includes('${')) return false;
+
+  return stageKey !== stage;
+};

lib/configuration/variables/resolve.js

@@ -78,13 +78,18 @@ class VariablesResolver {
     options,
     fulfilledSources,
     propertyPathsToResolve,
+    isPropertyDeferred,
   }) {
     this.serviceDir = serviceDir;
     this.configuration = configuration;
     this.variablesMeta = variablesMeta;
     this.sources = sources;
     this.options = options;
     this.fulfilledSources = fulfilledSources;
+    // Matching properties are not scheduled for resolution in this pass. They remain in
+    // `variablesMeta`, so they still resolve on demand when another property depends on them
+    // (see `resolveDependentProperty`)
+    this.isPropertyDeferred = isPropertyDeferred || (() => false);
     this.propertyDependenciesMap = new Map();
     this.propertyResolutionNestDepthMap = new Map();
 
@@ -104,6 +109,7 @@ class VariablesResolver {
                 propertyPath === propertyPathToResolve ||
                 propertyPath.startsWith(`${propertyPathToResolve}\0`)
               ) {
+                if (this.isPropertyDeferred(propertyPath)) return null;
                 return this.resolveProperty(resolutionBatchId, propertyPath);
               }
               return null;
@@ -114,9 +120,10 @@ class VariablesResolver {
     }
 
     return Promise.all(
-      Array.from(variablesMeta.keys(), (propertyPath) =>
-        this.resolveProperty(resolutionBatchId, propertyPath)
-      )
+      Array.from(variablesMeta.keys(), (propertyPath) => {
+        if (this.isPropertyDeferred(propertyPath)) return null;
+        return this.resolveProperty(resolutionBatchId, propertyPath);
+      })
     ).then(() => {});
   }
   async resolveVariables(resolutionBatchId, propertyPath, valueMeta) {
@@ -566,12 +573,14 @@ Object.defineProperties(
           return;
         }
 
-        // Resolve variables found in resolved value
+        // Resolve variables found in resolved value. Deferred properties are registered above
+        // but not scheduled: they still resolve on demand (see `resolveDependentProperty`)
         const newResolutionBatchId = ++lastResolutionBatchId;
         await Promise.all(
-          Array.from(propertyVariablesMeta.keys()).map((subPropertyPath) =>
-            this.resolveProperty(newResolutionBatchId, subPropertyPath)
-          )
+          Array.from(propertyVariablesMeta.keys()).map((subPropertyPath) => {
+            if (this.isPropertyDeferred(subPropertyPath)) return null;
+            return this.resolveProperty(newResolutionBatchId, subPropertyPath);
+          })
         );
       },
       {
@@ -697,6 +706,7 @@ module.exports = async (data) => {
   for (const { resolve } of Object.values(data.sources)) ensurePlainFunction(resolve);
   ensurePlainObject(data.options);
   ensureSet(data.fulfilledSources);
+  ensurePlainFunction(data.isPropertyDeferred, { isOptional: true });
   ensureSet(data.propertyPathsToResolve, { isOptional: true });
   if (data.propertyPathsToResolve) {
     data.propertyPathsToResolve = new Set(Array.from(data.propertyPathsToResolve, ensureString));

scripts/serverless.js

@@ -138,6 +138,7 @@ process.once('uncaughtException', (error) => {
     const Serverless = require('../lib/serverless');
     const { safeShallowAssign } = require('../lib/utils/safe-object');
     const resolveVariables = require('../lib/configuration/variables/resolve');
+    const deferIrrelevantStageParams = require('../lib/configuration/variables/defer-irrelevant-stage-params');
     const isPropertyResolved = require('../lib/configuration/variables/is-property-resolved');
     const eventuallyReportVariableResolutionErrors = require('../lib/configuration/variables/eventually-report-resolution-errors');
     const filterSupportedOptions = require('../lib/cli/filter-supported-options');
@@ -273,6 +274,11 @@ process.once('uncaughtException', (error) => {
               fulfilledSources: new Set(['file', 'self', 'strToBool']),
               propertyPathsToResolve: new Set(['provider\0name', 'provider\0stage', 'useDotenv']),
             };
+            // Assigned after the literal, as the predicate reads `resolverConfiguration` live
+            resolverConfiguration.isPropertyDeferred = deferIrrelevantStageParams(
+              configuration,
+              resolverConfiguration
+            );
 
             await resolveVariables(resolverConfiguration);
 
@@ -571,6 +577,12 @@ process.once('uncaughtException', (error) => {
                 ? new Set(['plugins', 'provider\0name', 'provider\0stage', 'useDotenv'])
                 : null,
           };
+          // Assigned after the literal, as the predicate reads `resolverConfiguration` live
+          // (when set up above instead, `resolverConfiguration` already carries its predicate)
+          resolverConfiguration.isPropertyDeferred = deferIrrelevantStageParams(
+            configuration,
+            resolverConfiguration
+          );
         }
 
         if (commandSchema) {
@@ -644,6 +656,17 @@ process.once('uncaughtException', (error) => {
         // Having all source resolvers configured, resolve variables
         processLog.debug('resolve all variables');
         await resolveVariables(resolverConfiguration);
+
+        // Drop entries deliberately left unresolved (params of irrelevant stages), so they're not
+        // reported as resolution errors or unrecognized sources; their raw values remain in the
+        // configuration. Errored entries (e.g. syntax errors in resolved objects) are kept to be
+        // reported below. Must not run while a resolution pass is in progress
+        for (const propertyPath of Array.from(variablesMeta.keys())) {
+          if (!resolverConfiguration.isPropertyDeferred(propertyPath)) continue;
+          if (variablesMeta.get(propertyPath).error) continue;
+          variablesMeta.delete(propertyPath);
+        }
+
         if (!variablesMeta.size) return;
         if (
           eventuallyReportVariableResolutionErrorsForCliInput(

test/unit/lib/configuration/variables/defer-irrelevant-stage-params.test.js

@@ -0,0 +1,114 @@
+'use strict';
+
+const { expect } = require('chai');
+
+const deferIrrelevantStageParams = require('../../../../../lib/configuration/variables/defer-irrelevant-stage-params');
+
+const p = (...keys) => keys.join('\0');
+
+describe('test/unit/lib/configuration/variables/defer-irrelevant-stage-params.test.js', () => {
+  const buildPredicate = ({
+    params = { dev: {}, prod: {}, default: {} },
+    provider,
+    options,
+  } = {}) => deferIrrelevantStageParams({ params, provider }, { options });
+
+  it('should defer params of not effective stages', () => {
+    const isPropertyDeferred = buildPredicate({ options: { stage: 'dev' } });
+    expect(isPropertyDeferred(p('params', 'prod', 'secret'))).to.be.true;
+    expect(isPropertyDeferred(p('params', 'prod', 'nested', 'deep'))).to.be.true;
+  });
+
+  it('should not defer the stage section itself', () => {
+    // Such section has to resolve, as the configuration schema requires it to be an object
+    const isPropertyDeferred = buildPredicate({ options: { stage: 'dev' } });
+    expect(isPropertyDeferred(p('params', 'prod'))).to.be.false;
+  });
+
+  it('should not defer params of the effective stage', () => {
+    const isPropertyDeferred = buildPredicate({ options: { stage: 'prod' } });
+    expect(isPropertyDeferred(p('params', 'prod', 'secret'))).to.be.false;
+  });
+
+  it('should not defer default params', () => {
+    const isPropertyDeferred = buildPredicate({ options: { stage: 'dev' } });
+    expect(isPropertyDeferred(p('params', 'default', 'secret'))).to.be.false;
+    expect(isPropertyDeferred(p('params', 'default'))).to.be.false;
+  });
+
+  it('should not defer properties outside of "params"', () => {
+    const isPropertyDeferred = buildPredicate({ options: { stage: 'dev' } });
+    expect(isPropertyDeferred(p('provider', 'stage'))).to.be.false;
+    expect(isPropertyDeferred(p('paramsLike', 'prod'))).to.be.false;
+    expect(isPropertyDeferred('params')).to.be.false;
+  });
+
+  it('should resolve effective stage from "provider.stage" when not passed via options', () => {
+    const isPropertyDeferred = buildPredicate({ provider: { stage: 'prod' }, options: {} });
+    expect(isPropertyDeferred(p('params', 'prod', 'secret'))).to.be.false;
+    expect(isPropertyDeferred(p('params', 'dev', 'secret'))).to.be.true;
+  });
+
+  it('should prioritize stage passed via options over "provider.stage"', () => {
+    const isPropertyDeferred = buildPredicate({
+      provider: { stage: 'prod' },
+      options: { stage: 'dev' },
+    });
+    expect(isPropertyDeferred(p('params', 'prod', 'secret'))).to.be.true;
+    expect(isPropertyDeferred(p('params', 'dev', 'secret'))).to.be.false;
+  });
+
+  it('should default effective stage to "dev"', () => {
+    const isPropertyDeferred = buildPredicate({ options: {} });
+    expect(isPropertyDeferred(p('params', 'dev', 'secret'))).to.be.false;
+    expect(isPropertyDeferred(p('params', 'prod', 'secret'))).to.be.true;
+  });
+
+  it('should ignore non-primitive "provider.stage" values safely', () => {
+    const isPropertyDeferred = buildPredicate({
+      provider: { stage: { unexpected: 'object' } },
+      options: {},
+    });
+    expect(isPropertyDeferred(p('params', 'dev', 'secret'))).to.be.false;
+    expect(isPropertyDeferred(p('params', 'prod', 'secret'))).to.be.true;
+  });
+
+  it('should not defer when "provider.stage" did not resolve yet', () => {
+    // Deferring against a stage which is about to change would strand params of the effective
+    // stage as never scheduled for resolution
+    const isPropertyDeferred = buildPredicate({
+      provider: { stage: "${opt:stage, 'prod'}" },
+      options: {},
+    });
+    expect(isPropertyDeferred(p('params', 'dev', 'secret'))).to.be.false;
+    expect(isPropertyDeferred(p('params', 'prod', 'secret'))).to.be.false;
+  });
+
+  it('should not defer when stage passed via options did not resolve yet', () => {
+    const isPropertyDeferred = buildPredicate({ options: { stage: '${env:STAGE}' } });
+    expect(isPropertyDeferred(p('params', 'prod', 'secret'))).to.be.false;
+  });
+
+  it('should not defer when "params" is not a plain object', () => {
+    const isPropertyDeferred = buildPredicate({ params: null, options: { stage: 'dev' } });
+    expect(isPropertyDeferred(p('params', 'prod', 'secret'))).to.be.false;
+    const isPropertyDeferredForArray = buildPredicate({
+      params: ['unexpected'],
+      options: { stage: 'dev' },
+    });
+    expect(isPropertyDeferredForArray(p('params', '0', 'secret'))).to.be.false;
+  });
+
+  it('should read configuration and options live', () => {
+    const configuration = { params: { dev: {}, prod: {} } };
+    const resolverConfiguration = { options: {} };
+    const isPropertyDeferred = deferIrrelevantStageParams(configuration, resolverConfiguration);
+    expect(isPropertyDeferred(p('params', 'prod', 'secret'))).to.be.true;
+    // Options object is replaced between resolution passes
+    resolverConfiguration.options = { stage: 'prod' };
+    expect(isPropertyDeferred(p('params', 'prod', 'secret'))).to.be.false;
+    // Plugins may replace the "params" section during initialization
+    configuration.params = 'not-an-object';
+    expect(isPropertyDeferred(p('params', 'prod', 'secret'))).to.be.false;
+  });
+});