feat(resolver): resolve ${sls:stage}; aws: refs in fallbacks pass through; filters on resolve-later values error

packages/configorama/src/main.js

@@ -213,7 +213,7 @@ const {
 const PromiseTracker = require('./utils/PromiseTracker')
 const handleSignalEvents = require('./utils/handleSignalEvents')
 /* Utils - encoders */
-const { encodeUnknown, decodeUnknown, PASSTHROUGH_PREFIX } = require('./utils/encoders/unknown-values')
+const { encodeUnknown, decodeUnknown, findUnknownValues, PASSTHROUGH_PREFIX } = require('./utils/encoders/unknown-values')
 const { decodeEncodedValue } = require('./utils/encoders')
 const { decodeJsSyntax, hasParenthesesPlaceholder, encodeJsonForVariable, parseEncodedJson, isEncodedJson } = require('./utils/encoders/js-fixes')
 const { tagDates, reviveDates } = require('./utils/encoders/dates')
@@ -264,6 +264,7 @@ const getValueFromString = require('./resolvers/valueFromString')
 const getValueFromNumber = require('./resolvers/valueFromNumber')
 const getValueFromEnv = require('./resolvers/valueFromEnv')
 const getValueFromOptions = require('./resolvers/valueFromOptions')
+const getValueFromSls = require('./resolvers/valueFromSls')
 const getValueFromParam = require('./resolvers/valueFromParam')
 const getValueFromCron = require('./resolvers/valueFromCron')
 const getValueFromEval = require('./resolvers/valueFromEval')
@@ -528,6 +529,12 @@ class Configorama {
        * ${opt:other, "fallbackValue"}
        */
       getValueFromOptions,
+      /**
+       * Serverless stage (other sls: addresses pass through)
+       * Usage:
+       * ${sls:stage}
+       */
+      getValueFromSls,
 
       /**
        * Parameters
@@ -2733,8 +2740,19 @@ Missing Value ${missingValue} - ${matchedString}
    * @returns {any} The filtered value
    */
   applyFilters(value, filters, pathValue) {
-    // An unresolved variable kept as text (allowUnresolvedVariables) has no value to filter yet
-    if (isPassthrough(value)) return value
+    if (isPassthrough(value)) {
+      // An unknown type (${ssm:x}, ${cf:x}) gets its value later, outside configorama, so a filter
+      // here would run on the wrong text: an error. A known type left unresolved
+      // (allowUnresolvedVariables) has no value to filter yet: the filter is skipped
+      const kept = findUnknownValues(value).map(({ value: b64 }) => Buffer.from(b64, 'base64').toString('utf8'))
+      const resolvedLater = kept.filter((v) => this.isUnknownTypeAllowed(v))
+      if (!resolvedLater.length) return value
+      const names = filters.map((f) => parseFilter(f, this.config).name).join(', ')
+      const at = pathValue ? ` at "${[].concat(pathValue).join('.')}"` : ''
+      throw new Error(`Filter ${names} can't run on "${decodeUnknown(value)}"${at}: it still holds ` +
+        `${resolvedLater.join(', ')}, which is resolved later, outside configorama. ` +
+        'Remove the filter or give that variable a value first.')
+    }
     const filtered = filters.reduce((acc, filter) => {
       const { name, args } = parseFilter(filter, this.config)
       if (typeof this.filters[name] !== 'function') throw new Error(`Filter "${name}" not found`)

packages/configorama/src/resolvers/valueFromSls.js

@@ -0,0 +1,35 @@
+// Resolves ${sls:stage} the way Serverless (osls) does: --stage, else provider.stage, else 'dev'
+// Other sls: addresses (instanceId) aren't matched, so they stay unknown types for Serverless to fill in
+
+const slsStageSyntax = RegExp(/^sls:stage$/)
+
+/**
+ * Stage Serverless deploys to. A provider.stage that is itself a variable (${opt:stage, 'dev'})
+ * is returned as a self reference so it resolves to that variable's value.
+ * @param {string} variableString - sls:stage
+ * @param {Record<string, any>} options - Parsed CLI options
+ * @param {Record<string, any>} config - The config being resolved
+ * @returns {Promise<string>}
+ */
+function getValueFromSls(variableString, options, config) {
+  if (options && options.stage !== undefined && options.stage !== null) {
+    return Promise.resolve(options.stage)
+  }
+  const provider = config && config.provider
+  if (provider && provider.stage !== undefined && provider.stage !== null) {
+    return Promise.resolve('${self:provider.stage}')
+  }
+  return Promise.resolve('dev')
+}
+
+module.exports = {
+  type: 'sls',
+  source: 'config',
+  prefix: 'sls',
+  // No prefix registration: sls: addresses other than stage must remain unknown types (passthrough)
+  prefixes: [],
+  syntax: '${sls:stage}',
+  description: 'Resolves the Serverless stage: --stage, else provider.stage, else "dev". Example: ${sls:stage}',
+  match: slsStageSyntax,
+  resolver: getValueFromSls
+}

packages/configorama/src/utils/encoders/js-fixes.js

@@ -152,4 +152,5 @@ module.exports = {
   isEncodedJson,
   hasEncodedJson,
   encodeJsonArgObjects,
+  isInsideVariable,
 }

packages/configorama/src/utils/parsing/preProcess.js

@@ -7,7 +7,8 @@ const { splitByComma } = require('../strings/splitByComma')
 const { splitOnPipe } = require('../strings/splitOnPipe')
 const { getQuoteRanges } = require('../strings/quoteAware')
 const { extractVariableWrapper } = require('../variables/variableUtils')
-const { encodeJsonArgObjects } = require('../encoders/js-fixes')
+const { encodeJsonArgObjects, isInsideVariable } = require('../encoders/js-fixes')
+const { encodeUnknown } = require('../encoders/unknown-values')
 const { encodeQuotedLiterals } = require('../encoders/literal-braces')
 const { setOwn } = require('../objects')
 
@@ -35,6 +36,28 @@ function preProcess(configObject, variableSyntax, variableTypes, options = {}) {
         .filter(p => p !== 'dot.prop:' && p !== 'string:' && p !== 'number:')
     : ['self:', 'opt:', 'env:', 'file:', 'text:', 'deep:']
 
+  // Refs the syntax excludes (its (?!AWS|aws:|stageVariables) lookahead) belong to Serverless /
+  // CloudFormation / IAM and are never resolved here
+  const excludedLookahead = variableSyntax ? /\(\?!([^)]*)\)/.exec(variableSyntax.source) : null
+  const escapeRegex = (/** @type {string} */ text) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+  const excludedRefPattern = excludedLookahead && excludedLookahead[1]
+    ? new RegExp(`${escapeRegex(varPrefix)}(?:${excludedLookahead[1]})[^${escapeRegex(varSuffix[0])}]*${escapeRegex(varSuffix)}`, 'g')
+    : null
+
+  /**
+   * An excluded ref inside another variable (${env:X, 'app-${aws:accountId}'}) would stop that
+   * variable from matching at all: keep it as an unknown passthrough instead, restored verbatim at
+   * the end. Outside any variable (IAM ${aws:username}) it is plain text and left alone.
+   * @param {string} str
+   * @returns {string}
+   */
+  function encodeNestedExcludedRefs(str) {
+    if (!excludedRefPattern || str.indexOf(varPrefix) === str.lastIndexOf(varPrefix)) return str
+    return str.replace(excludedRefPattern, (ref, offset) => {
+      return isInsideVariable(str, offset, varPrefix) ? encodeUnknown(ref) : ref
+    })
+  }
+
   /**
    * Escape variables inside help() filter arguments so main resolver skips them
    * Uses base64 encoding to preserve exact original syntax (supports custom variable syntax)
@@ -378,7 +401,8 @@ function preProcess(configObject, variableSyntax, variableTypes, options = {}) {
       const hasEvalOrIf = obj.indexOf('if(') !== -1 || obj.indexOf('eval(') !== -1
       const hasComma = obj.indexOf(',') !== -1
 
-      const withHelpEscaped = hasHelp ? escapeHelpVariables(obj) : obj
+      const withExcludedEncoded = encodeNestedExcludedRefs(obj)
+      const withHelpEscaped = hasHelp ? escapeHelpVariables(withExcludedEncoded) : withExcludedEncoded
       const withBareRefsConverted = hasEvalOrIf ? convertBareRefsInIf(withHelpEscaped) : withHelpEscaped
       // Encode JSON object literals used as filter/function args so their { } don't break variable matching.
       const withJsonArgsEncoded = obj.indexOf('{') !== -1 ? encodeJsonArgObjects(withBareRefsConverted, varPrefix) : withBareRefsConverted

packages/configorama/tests/awsInFallback/awsInFallback.test.js

@@ -0,0 +1,80 @@
+/* Variables the default syntax excludes (${aws:...}, ${AWS::...}) are left for Serverless and */
+/* CloudFormation. Inside another variable's fallback they pass through instead of blocking    */
+/* that variable; outside any variable (IAM policy variables) they stay literal as before.     */
+/* eslint-disable no-template-curly-in-string */
+const { test } = require('uvu')
+const assert = require('uvu/assert')
+const { resolveYamlText } = require('../utils')
+
+const UNSET = 'CONFIGORAMA_AWS_TEST_UNSET'
+const SET = 'CONFIGORAMA_AWS_TEST_SET'
+const SERVERLESS = { allowUnknownVariableTypes: true, allowUnresolvedVariables: true, options: { stage: 'qa' } }
+
+test.before.each(() => {
+  delete process.env[UNSET]
+  process.env[SET] = 'from-env'
+})
+
+test.after(() => {
+  delete process.env[SET]
+})
+
+/**
+ * Resolve key `out`
+ * @param {string} expr - YAML value text for out
+ * @param {object} [settings]
+ * @returns {Promise<any>}
+ */
+async function resolveOut(expr, settings = SERVERLESS) {
+  const config = await resolveYamlText(`custom:\n  name: app\nout: ${JSON.stringify(expr)}\n`, settings)
+  return config.out
+}
+
+test('aws:accountId in a quoted fallback passes through', async () => {
+  assert.is(await resolveOut(`\${env:${UNSET}, 'app-\${aws:accountId}'}`), 'app-${aws:accountId}')
+})
+
+test('aws:region as the whole quoted fallback', async () => {
+  assert.is(await resolveOut(`\${env:${UNSET}, '\${aws:region}'}`), '${aws:region}')
+})
+
+test('aws: ref as an unquoted fallback item', async () => {
+  assert.is(await resolveOut(`\${env:${UNSET}, \${aws:region}}`), '${aws:region}')
+})
+
+test('aws: ref next to a self ref in the fallback', async () => {
+  assert.is(await resolveOut(`\${env:${UNSET}, '\${self:custom.name}-\${aws:accountId}-\${aws:region}'}`), 'app-${aws:accountId}-${aws:region}')
+})
+
+test('env set: the fallback holding aws: is unused', async () => {
+  assert.is(await resolveOut(`\${env:${SET}, 'app-\${aws:accountId}'}`), 'from-env')
+})
+
+test('AWS:: pseudo parameter in a fallback passes through', async () => {
+  assert.is(await resolveOut(`\${env:${UNSET}, 'arn:aws:s3:::b-\${AWS::AccountId}'}`), 'arn:aws:s3:::b-${AWS::AccountId}')
+})
+
+test('IAM policy variables outside any variable stay literal', async () => {
+  assert.is(await resolveOut('arn:aws:s3:::bucket/${aws:username}/*'), 'arn:aws:s3:::bucket/${aws:username}/*')
+  assert.is(await resolveOut('arn:aws:s3:::bucket/${aws:PrincipalTag/team}/*'), 'arn:aws:s3:::bucket/${aws:PrincipalTag/team}/*')
+  assert.is(await resolveOut('${aws:accountId}', {}), '${aws:accountId}', 'without unknown-type settings too')
+})
+
+test('IAM policy variable in Fn::Sub next to a self ref', async () => {
+  const yml = 'custom:\n  name: app\nR:\n  Fn::Sub: "arn:aws:s3:::${self:custom.name}/${aws:username}/${AWS::Region}"\n'
+  const config = await resolveYamlText(yml, SERVERLESS)
+  assert.is(config.R['Fn::Sub'], 'arn:aws:s3:::app/${aws:username}/${AWS::Region}')
+})
+
+test('a filter over a fallback holding aws:accountId is an error', async () => {
+  let error
+  try {
+    await resolveOut(`\${env:${UNSET}, 'app-\${aws:accountId}' | toUpperCase}`)
+  } catch (err) {
+    error = err
+  }
+  assert.ok(error, 'expected an error')
+  assert.match(error.message, '${aws:accountId}')
+})
+
+test.run()

packages/configorama/tests/duplicateVarFallback/duplicateVarFallback.test.js

@@ -48,9 +48,10 @@ test('controls: unchanged behavior', () => {
   assert.is(config.knownThrice, 'dev/dev/dev')
 })
 
-test('AWS pseudo params (and vars wrapping them) are left verbatim', () => {
-  assert.is(config.awsPseudoFirst, "${opt:nope, ${AWS::Region}, 'us-east-1'} and ${AWS::Region}")
-  assert.is(config.awsPseudoSecond, "${AWS::Region} and ${opt:nope, ${AWS::Region}, 'us-east-1'}")
+test('AWS pseudo params are left verbatim; a var wrapping one resolves to it as a passthrough', () => {
+  // opt:nope is missing, so the fallback list picks ${AWS::Region}, kept for CloudFormation
+  assert.is(config.awsPseudoFirst, '${AWS::Region} and ${AWS::Region}')
+  assert.is(config.awsPseudoSecond, '${AWS::Region} and ${AWS::Region}')
 })
 
 test('sync API: each occurrence resolves in its own context', () => {

packages/configorama/tests/passthroughInFallback/laterSlot.yml

@@ -1,4 +1,4 @@
 stage: dev
 # Quoted passthrough fallback after other unresolvable items in the list
-afterMissingEnv: ${env:CONFIGORAMA_TEST_UNSET, ${env:CONFIGORAMA_TEST_UNSET2}, 'sl-${sls:stage}'}
-afterMissingSelf: ${env:CONFIGORAMA_TEST_UNSET, ${self:nope}, 'c-${sls:stage}'}
+afterMissingEnv: ${env:CONFIGORAMA_TEST_UNSET, ${env:CONFIGORAMA_TEST_UNSET2}, 'sl-${sls:instanceId}'}
+afterMissingSelf: ${env:CONFIGORAMA_TEST_UNSET, ${self:nope}, 'c-${sls:instanceId}'}

packages/configorama/tests/passthroughInFallback/passthroughInFallback.test.js

@@ -1,4 +1,4 @@
-/* Unknown variable types (${sls:stage}, ${ssm:/path}) inside a quoted fallback string pass */
+/* Unknown variable types (${sls:instanceId}, ${ssm:/path}) inside a quoted fallback string pass */
 /* through verbatim when allowUnknownVariableTypes is on, with the text around them kept. */
 /* eslint-disable no-template-curly-in-string */
 const path = require('path')
@@ -29,69 +29,69 @@ test('resolves fixture', async () => {
 })
 
 test('passthrough with text around it in a quoted fallback', () => {
-  assert.is(config.textAround, 'sl-${sls:stage}-manifest')
+  assert.is(config.textAround, 'sl-${sls:instanceId}-manifest')
 })
 
 test('passthrough as the whole quoted fallback', () => {
-  assert.is(config.onlyPassthrough, '${sls:stage}')
+  assert.is(config.onlyPassthrough, '${sls:instanceId}')
 })
 
 test('double-quoted fallback', () => {
-  assert.is(config.doubleQuoted, 'sl-${sls:stage}-manifest')
+  assert.is(config.doubleQuoted, 'sl-${sls:instanceId}-manifest')
 })
 
 test('two passthroughs in one quoted fallback', () => {
-  assert.is(config.twoPassthroughs, '${ssm:/app/prefix}-${sls:stage}')
+  assert.is(config.twoPassthroughs, '${ssm:/app/prefix}-${sls:instanceId}')
 })
 
 test('self ref resolves next to a passthrough in a quoted fallback', () => {
-  assert.is(config.mixedWithSelf, 'dev-${sls:stage}')
+  assert.is(config.mixedWithSelf, 'dev-${sls:instanceId}')
 })
 
 test('self ref to a passthrough fallback value keeps it', () => {
-  assert.is(config.derived, 'sl-${sls:stage}-manifest-tables')
+  assert.is(config.derived, 'sl-${sls:instanceId}-manifest-tables')
 })
 
 test('fallback variable with text on both sides', () => {
-  assert.is(config.inText, 'prefix-sl-${sls:stage}-suffix')
+  assert.is(config.inText, 'prefix-sl-${sls:instanceId}-suffix')
 })
 
 test('outer opt: variable missing uses the passthrough fallback', () => {
-  assert.is(config.optMissing, 'sl-${sls:stage}-manifest')
+  assert.is(config.optMissing, 'sl-${sls:instanceId}-manifest')
 })
 
 test('outer opt: variable set ignores the passthrough fallback', () => {
   assert.is(config.optSet, 'qa')
 })
 
 test('inside arrays and nested objects', () => {
-  assert.equal(config.nested.list, ['a-${sls:stage}'])
+  assert.equal(config.nested.list, ['a-${sls:instanceId}'])
   assert.is(config.nested.obj.key, 'b-${ssm:/x/y}')
 })
 
 test('passthrough as an unquoted fallback variable', () => {
-  assert.is(config.bareFallback, '${sls:stage}')
+  assert.is(config.bareFallback, '${sls:instanceId}')
 })
 
 test('env set: fallback unused', () => {
   assert.is(config.envSet, 'from-env')
 })
 
 test('control: passthrough outside a fallback', () => {
-  assert.is(config.bare, 'sl-${sls:stage}-manifest')
+  assert.is(config.bare, 'sl-${sls:instanceId}-manifest')
 })
 
 test('allowUnknownVariableTypes as a type list', async () => {
   const listed = await configorama(fixture, { configDir: __dirname, ...settings, allowUnknownVariableTypes: ['sls', 'ssm'] })
-  assert.is(listed.textAround, 'sl-${sls:stage}-manifest')
-  assert.is(listed.twoPassthroughs, '${ssm:/app/prefix}-${sls:stage}')
-  assert.is(listed.bare, 'sl-${sls:stage}-manifest')
+  assert.is(listed.textAround, 'sl-${sls:instanceId}-manifest')
+  assert.is(listed.twoPassthroughs, '${ssm:/app/prefix}-${sls:instanceId}')
+  assert.is(listed.bare, 'sl-${sls:instanceId}-manifest')
 })
 
 test('quoted passthrough fallback after other unresolvable list items', async () => {
   const later = await configorama(path.join(__dirname, 'laterSlot.yml'), { configDir: __dirname, options: {}, allowUnknownVariableTypes: true })
-  assert.is(later.afterMissingEnv, 'sl-${sls:stage}')
-  assert.is(later.afterMissingSelf, 'c-${sls:stage}')
+  assert.is(later.afterMissingEnv, 'sl-${sls:instanceId}')
+  assert.is(later.afterMissingSelf, 'c-${sls:instanceId}')
 })
 
 /**
@@ -111,13 +111,13 @@ async function resolveError(file, opts) {
 
 test('unknown types not allowed: passthrough in a quoted fallback is an error, not mangled text', async () => {
   const err = await resolveError('strict.yml', {})
-  assert.match(err.message, 'Variable: "sls:stage"')
+  assert.match(err.message, 'Variable: "sls:instanceId"')
   assert.match(err.message, 'not found')
 })
 
 test('type list without the type: passthrough in a quoted fallback is an error', async () => {
   const err = await resolveError('strict.yml', { allowUnknownVariableTypes: ['ssm'] })
-  assert.match(err.message, 'Variable: "sls:stage"')
+  assert.match(err.message, 'Variable: "sls:instanceId"')
 })
 
 test('missing self ref in a quoted fallback is an error', async () => {
@@ -127,10 +127,10 @@ test('missing self ref in a quoted fallback is an error', async () => {
 
 test('sync API', () => {
   const syncConfig = configorama.sync(fixture, settings)
-  assert.is(syncConfig.textAround, 'sl-${sls:stage}-manifest')
-  assert.is(syncConfig.onlyPassthrough, '${sls:stage}')
-  assert.is(syncConfig.derived, 'sl-${sls:stage}-manifest-tables')
-  assert.is(syncConfig.inText, 'prefix-sl-${sls:stage}-suffix')
+  assert.is(syncConfig.textAround, 'sl-${sls:instanceId}-manifest')
+  assert.is(syncConfig.onlyPassthrough, '${sls:instanceId}')
+  assert.is(syncConfig.derived, 'sl-${sls:instanceId}-manifest-tables')
+  assert.is(syncConfig.inText, 'prefix-sl-${sls:instanceId}-suffix')
   assert.is(syncConfig.optSet, 'qa')
 })