From b917423c2d83c189cb556da80e873c9266f2b856 Mon Sep 17 00:00:00 2001 From: "Joe Skeen, Jr." Date: Tue, 10 Nov 2026 09:18:54 -0700 Subject: [PATCH 1/4] Allow dot-separated paths into nested JSON in awsSecret and awsParam An exact top-level key still wins, so existing keys with dots keep working. Object and array values are returned as JSON text instead of [object Object]. --- .bumpy/aws-secrets-nested-json-keys.md | 5 ++ packages/plugins/aws-secrets/README.md | 3 +- packages/plugins/aws-secrets/src/json-key.ts | 33 ++++++++++ packages/plugins/aws-secrets/src/plugin.ts | 35 ++++++----- .../plugins/aws-secrets/test/json-key.test.ts | 60 +++++++++++++++++++ .../src/content/docs/plugins/aws-secrets.mdx | 2 + 6 files changed, 121 insertions(+), 17 deletions(-) create mode 100644 .bumpy/aws-secrets-nested-json-keys.md create mode 100644 packages/plugins/aws-secrets/src/json-key.ts create mode 100644 packages/plugins/aws-secrets/test/json-key.test.ts diff --git a/.bumpy/aws-secrets-nested-json-keys.md b/.bumpy/aws-secrets-nested-json-keys.md new file mode 100644 index 000000000..5bab44dbf --- /dev/null +++ b/.bumpy/aws-secrets-nested-json-keys.md @@ -0,0 +1,5 @@ +--- +"@varlock/aws-secrets-plugin": minor +--- + +`awsSecret()` and `awsParam()` can read nested JSON values with a dot-separated key, such as `awsSecret("db-config#db.host")`. An exact top-level key with the same name still takes precedence. diff --git a/packages/plugins/aws-secrets/README.md b/packages/plugins/aws-secrets/README.md index c970b26ed..efa886daa 100644 --- a/packages/plugins/aws-secrets/README.md +++ b/packages/plugins/aws-secrets/README.md @@ -10,7 +10,7 @@ This package is a [Varlock](https://varlock.dev) [plugin](https://varlock.dev/gu - **IAM role support** - No credentials needed for AWS-hosted apps (EC2, ECS, Lambda, etc.) - **AWS CLI authentication** - Works seamlessly with `aws configure` for local development - **Auto-infer secret/parameter names** from environment variable names (uses name as-is) -- **JSON key extraction** from secrets/parameters using `#` syntax or named `key` parameter +- **JSON key extraction** from secrets/parameters using `#` syntax or named `key` parameter, including dot-separated paths into nested objects - **Name prefixing** with `namePrefix` option for organized secret management - **OIDC workload identity** - Authenticate from Vercel, GitHub Actions, and other platforms without long-lived credentials - Support for named AWS profiles @@ -225,6 +225,7 @@ Fetch a secret from AWS Secrets Manager. - Name: `"my-secret"` - Name with JSON key: `"my-secret#password"` (shorthand for key extraction) +- Name with a nested JSON key: `"my-secret#db.host"` (dot-separated path into nested objects) - ARN: `"arn:aws:secretsmanager:us-east-1:123456789012:secret:my-secret-AbCdEf"` #### `awsParam()` diff --git a/packages/plugins/aws-secrets/src/json-key.ts b/packages/plugins/aws-secrets/src/json-key.ts new file mode 100644 index 000000000..6596edb92 --- /dev/null +++ b/packages/plugins/aws-secrets/src/json-key.ts @@ -0,0 +1,33 @@ +/** Result of looking up a key in parsed JSON. */ +export type JsonKeyLookup = | { found: true, value: string } + | { found: false, availableKeys: Array }; + +function isContainer(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +function toStringValue(value: unknown): string { + return isContainer(value) ? JSON.stringify(value) : String(value); +} + +/** + * Looks up `key` in a parsed JSON value. + * + * An exact top-level key wins, so keys that contain dots keep working. Otherwise `key` is read as a + * dot-separated path: `db.host` reads `value.db.host`. Object and array results are returned as JSON text. + */ +export function lookupJsonKey(parsed: unknown, key: string): JsonKeyLookup { + if (isContainer(parsed) && Object.prototype.hasOwnProperty.call(parsed, key)) { + return { found: true, value: toStringValue(parsed[key]) }; + } + + let current: unknown = parsed; + for (const segment of key.split('.')) { + if (!isContainer(current) || !Object.prototype.hasOwnProperty.call(current, segment)) { + return { found: false, availableKeys: isContainer(current) ? Object.keys(current) : [] }; + } + current = current[segment]; + } + + return { found: true, value: toStringValue(current) }; +} diff --git a/packages/plugins/aws-secrets/src/plugin.ts b/packages/plugins/aws-secrets/src/plugin.ts index aecc050bc..60b399c41 100644 --- a/packages/plugins/aws-secrets/src/plugin.ts +++ b/packages/plugins/aws-secrets/src/plugin.ts @@ -2,6 +2,7 @@ import { type Resolver, type PluginCacheAccessor, plugin, resolveCacheTtl, } from 'varlock/plugin-lib'; import { BUNDLED_ICONS } from './bundled-icons.gen'; +import { lookupJsonKey } from './json-key'; import { createHash } from 'node:crypto'; import { @@ -469,38 +470,40 @@ class AwsPluginInstance { extractJsonKeyFromSecret(secretValue: string, jsonKey?: string): string { if (!jsonKey) return secretValue; + let parsed: unknown; try { - const parsed = JSON.parse(secretValue); - if (!(jsonKey in parsed)) { - throw new ResolutionError(`Key "${jsonKey}" not found in secret JSON`, { - tip: `Available keys: ${Object.keys(parsed).join(', ')}`, - }); - } - return String(parsed[jsonKey]); + parsed = JSON.parse(secretValue); } catch (err) { - if (err instanceof ResolutionError) throw err; throw new ResolutionError(`Failed to parse secret as JSON: ${err instanceof Error ? err.message : String(err)}`, { tip: 'Ensure the secret value is valid JSON when extracting a specific key', }); } + const result = lookupJsonKey(parsed, jsonKey); + if (!result.found) { + throw new ResolutionError(`Key "${jsonKey}" not found in secret JSON`, { + tip: `Available keys: ${result.availableKeys.join(', ')}`, + }); + } + return result.value; } extractJsonKeyFromParameter(paramValue: string, jsonKey?: string): string { if (!jsonKey) return paramValue; + let parsed: unknown; try { - const parsed = JSON.parse(paramValue); - if (!(jsonKey in parsed)) { - throw new ResolutionError(`Key "${jsonKey}" not found in parameter JSON`, { - tip: `Available keys: ${Object.keys(parsed).join(', ')}`, - }); - } - return String(parsed[jsonKey]); + parsed = JSON.parse(paramValue); } catch (err) { - if (err instanceof ResolutionError) throw err; throw new ResolutionError(`Failed to parse parameter as JSON: ${err instanceof Error ? err.message : String(err)}`, { tip: 'Ensure the parameter value is valid JSON when using the # syntax for key extraction', }); } + const result = lookupJsonKey(parsed, jsonKey); + if (!result.found) { + throw new ResolutionError(`Key "${jsonKey}" not found in parameter JSON`, { + tip: `Available keys: ${result.availableKeys.join(', ')}`, + }); + } + return result.value; } } diff --git a/packages/plugins/aws-secrets/test/json-key.test.ts b/packages/plugins/aws-secrets/test/json-key.test.ts new file mode 100644 index 000000000..8b69cebe7 --- /dev/null +++ b/packages/plugins/aws-secrets/test/json-key.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, test } from 'vitest'; +import { lookupJsonKey } from '../src/json-key'; + +describe('lookupJsonKey', () => { + test('reads a top-level key', () => { + expect(lookupJsonKey({ host: 'db.example.com' }, 'host')).toEqual({ found: true, value: 'db.example.com' }); + }); + + test('reads a dot-separated path into nested objects', () => { + const parsed = { db: { host: 'db.example.com', port: 5432 } }; + expect(lookupJsonKey(parsed, 'db.host')).toEqual({ found: true, value: 'db.example.com' }); + expect(lookupJsonKey(parsed, 'db.port')).toEqual({ found: true, value: '5432' }); + }); + + test('reads deeper paths', () => { + expect(lookupJsonKey({ a: { b: { c: 'deep' } } }, 'a.b.c')).toEqual({ found: true, value: 'deep' }); + }); + + test('an exact top-level key wins over a path with the same text', () => { + const parsed = { 'db.host': 'flat', db: { host: 'nested' } }; + expect(lookupJsonKey(parsed, 'db.host')).toEqual({ found: true, value: 'flat' }); + }); + + test('reports the keys at the level where the path stops', () => { + const parsed = { db: { host: 'x', port: 1 } }; + expect(lookupJsonKey(parsed, 'db.user')).toEqual({ found: false, availableKeys: ['host', 'port'] }); + expect(lookupJsonKey(parsed, 'cache.host')).toEqual({ found: false, availableKeys: ['db'] }); + }); + + test('a path through a scalar is not found', () => { + expect(lookupJsonKey({ db: 'postgres' }, 'db.host')).toEqual({ found: false, availableKeys: [] }); + }); + + test('empty path segments are not found', () => { + expect(lookupJsonKey({ a: { b: 1 } }, 'a..b').found).toBe(false); + expect(lookupJsonKey({ a: { b: 1 } }, 'a.').found).toBe(false); + }); + + test('inherited properties are not found', () => { + expect(lookupJsonKey({}, 'toString').found).toBe(false); + expect(lookupJsonKey({ db: {} }, 'db.constructor').found).toBe(false); + }); + + test('returns object and array values as JSON text', () => { + const parsed = { db: { hosts: ['a', 'b'], opts: { ssl: true } } }; + expect(lookupJsonKey(parsed, 'db.hosts')).toEqual({ found: true, value: '["a","b"]' }); + expect(lookupJsonKey(parsed, 'db.opts')).toEqual({ found: true, value: '{"ssl":true}' }); + }); + + test('keeps scalar string conversion for numbers, booleans, and null', () => { + const parsed = { port: 5432, enabled: false, missing: null }; + expect(lookupJsonKey(parsed, 'port')).toEqual({ found: true, value: '5432' }); + expect(lookupJsonKey(parsed, 'enabled')).toEqual({ found: true, value: 'false' }); + expect(lookupJsonKey(parsed, 'missing')).toEqual({ found: true, value: 'null' }); + }); + + test('reads a path that starts at an array index', () => { + expect(lookupJsonKey([{ host: 'first' }], '0.host')).toEqual({ found: true, value: 'first' }); + }); +}); diff --git a/packages/varlock-website/src/content/docs/plugins/aws-secrets.mdx b/packages/varlock-website/src/content/docs/plugins/aws-secrets.mdx index 28a27d30b..7ee845b0b 100644 --- a/packages/varlock-website/src/content/docs/plugins/aws-secrets.mdx +++ b/packages/varlock-website/src/content/docs/plugins/aws-secrets.mdx @@ -214,6 +214,8 @@ DB_PASSWORD=awsSecret("database-creds#password") DB_PORT=awsSecret("database-creds", key="port") ``` +Use a dot-separated path to read a nested object. If `database-creds` contains `{"db": {"host": "db.example.com"}}`, then `awsSecret("database-creds#db.host")` returns `db.example.com`. An exact top-level key with the same name takes precedence, so keys that contain dots keep working. Object and array values are returned as JSON text. + ### Name prefixing Use `namePrefix` to automatically prefix all secret/parameter names for better organization: From 29ab7de2d673db9509fe68012be5912c32189516 Mon Sep 17 00:00:00 2001 From: "Joe Skeen, Jr." Date: Tue, 10 Nov 2026 09:21:38 -0700 Subject: [PATCH 2/4] Document nested JSON keys for awsSecret with an example --- .../src/content/docs/plugins/aws-secrets.mdx | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/packages/varlock-website/src/content/docs/plugins/aws-secrets.mdx b/packages/varlock-website/src/content/docs/plugins/aws-secrets.mdx index 7ee845b0b..a0393ae5f 100644 --- a/packages/varlock-website/src/content/docs/plugins/aws-secrets.mdx +++ b/packages/varlock-website/src/content/docs/plugins/aws-secrets.mdx @@ -216,6 +216,69 @@ DB_PORT=awsSecret("database-creds", key="port") Use a dot-separated path to read a nested object. If `database-creds` contains `{"db": {"host": "db.example.com"}}`, then `awsSecret("database-creds#db.host")` returns `db.example.com`. An exact top-level key with the same name takes precedence, so keys that contain dots keep working. Object and array values are returned as JSON text. +### Nested JSON keys + +Secrets Manager stores a secret as text, and the most common layout is flat key-value pairs: one secret holds one JSON object, and each key is a single setting. Flat is the default for good reason. Every key sits at the top level, so any tool can read it, and one value can be read without parsing the rest. + +Nested objects are useful when a secret describes one thing with several parts. A database connection has a host, a port, a database name, and an SSL setting, and grouping them under `db` keeps them together. A secret that holds a whole application's configuration can have a section for the database, one for feature flags, and so on, and each section can be read on its own. + +Nested keys in Varlock are a read path, not a storage format. A secret can stay flat, be nested, or mix both. Varlock reads a dot-separated path such as `db.host` by walking the object one segment at a time. + +#### Example + +Suppose a team stores the settings for an API in one secret named `api/dev/config`: + +```json +{ + "db": { + "host": "db.dev.example.internal", + "port": 5432, + "name": "api", + "ssl": true + }, + "featureFlags": { + "newSearch": false + }, + "logLevel": "info" +} +``` + +The project's `.env.schema` reads individual values from that secret: + +```env-spec title=".env.schema" +# @plugin(@varlock/aws-secrets-plugin) +# @initAws(region=us-east-1) +# --- + +# @type=string +DB_HOST=awsSecret("api/dev/config#db.host") + +# @type=number +DB_PORT=awsSecret("api/dev/config#db.port") + +# @type=boolean +NEW_SEARCH=awsSecret("api/dev/config#featureFlags.newSearch") + +# The named key parameter reads paths the same way +LOG_LEVEL=awsSecret("api/dev/config", key="logLevel") +``` + +Walking through it: + +1. **Store the secret.** Save the JSON above as the secret string of `api/dev/config`. Varlock doesn't need registering anything in the secret. It only needs the name. +2. **Point each setting at a path.** `db.host` is the segment `db`, then the key `host`. `featureFlags.newSearch` goes one level deeper in the same way. +3. **Types apply after the read.** The value is read as text, then `@type` checks and converts it. `DB_PORT` is declared as a number, so the `5432` in the JSON becomes a number in the environment. +4. **Use `#` or `key=` for the same result.** `awsSecret("name#db.host")` and `awsSecret("name", key="db.host")` run the same lookup. +5. **Objects and arrays come back as JSON text.** `awsSecret("api/dev/config#db")` returns the whole `db` object as a JSON string. Parse it in code, or read the fields you need as separate settings. + +If a path doesn't exist, Varlock stops with an error that names the missing key and lists the keys available where the walk stopped. A typo such as `db.hots` lists `host`, `port`, `name`, and `ssl` in the hint. + +#### Precedence and edge cases + +An exact top-level key wins over a path with the same text. If the secret holds both `"db.host": "flat"` and `"db": {"host": "nested"}`, then `#db.host` returns `flat`. Existing flat keys that contain dots keep working after an upgrade. + +Empty path segments are not found. `db..host` and a trailing dot both fail the same way a missing key does. + ### Name prefixing Use `namePrefix` to automatically prefix all secret/parameter names for better organization: From 0ceb55cee13950a5d3864b2c867b9ac4b6810fe8 Mon Sep 17 00:00:00 2001 From: "Joe Skeen, Jr." Date: Tue, 10 Nov 2026 09:59:57 -0700 Subject: [PATCH 3/4] Correct nested key docs: whole-secret parse and empty path segments --- .../varlock-website/src/content/docs/plugins/aws-secrets.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/varlock-website/src/content/docs/plugins/aws-secrets.mdx b/packages/varlock-website/src/content/docs/plugins/aws-secrets.mdx index a0393ae5f..90f86d807 100644 --- a/packages/varlock-website/src/content/docs/plugins/aws-secrets.mdx +++ b/packages/varlock-website/src/content/docs/plugins/aws-secrets.mdx @@ -218,7 +218,7 @@ Use a dot-separated path to read a nested object. If `database-creds` contains ` ### Nested JSON keys -Secrets Manager stores a secret as text, and the most common layout is flat key-value pairs: one secret holds one JSON object, and each key is a single setting. Flat is the default for good reason. Every key sits at the top level, so any tool can read it, and one value can be read without parsing the rest. +Secrets Manager stores a secret as text, and the most common layout is flat key-value pairs: one secret holds one JSON object, and each key is a single setting. Flat is the default for good reason. Every key sits at the top level, so any tool can read it, and a value needs no walk through nested objects. Varlock still parses the whole secret before it looks up a key, so the layout affects how you read the data, not how much is parsed. Nested objects are useful when a secret describes one thing with several parts. A database connection has a host, a port, a database name, and an SSL setting, and grouping them under `db` keeps them together. A secret that holds a whole application's configuration can have a section for the database, one for feature flags, and so on, and each section can be read on its own. @@ -277,7 +277,7 @@ If a path doesn't exist, Varlock stops with an error that names the missing key An exact top-level key wins over a path with the same text. If the secret holds both `"db.host": "flat"` and `"db": {"host": "nested"}`, then `#db.host` returns `flat`. Existing flat keys that contain dots keep working after an upgrade. -Empty path segments are not found. `db..host` and a trailing dot both fail the same way a missing key does. +A path with an empty segment resolves only when the JSON has an empty-string key at that position. For example, `db..host` reads `{"db": {"": {"host": "value"}}}`. In any other secret, `db..host` and a trailing dot fail the same way a missing key does. ### Name prefixing From 718414d09906a1717bd9049e66b0d710a050d84b Mon Sep 17 00:00:00 2001 From: Theo Ephraim Date: Thu, 12 Nov 2026 18:05:06 +0000 Subject: [PATCH 4/4] Trim nested JSON key docs to a short example and the key rules --- .../src/content/docs/plugins/aws-secrets.mdx | 62 +++---------------- 1 file changed, 10 insertions(+), 52 deletions(-) diff --git a/packages/varlock-website/src/content/docs/plugins/aws-secrets.mdx b/packages/varlock-website/src/content/docs/plugins/aws-secrets.mdx index 90f86d807..b9ad22094 100644 --- a/packages/varlock-website/src/content/docs/plugins/aws-secrets.mdx +++ b/packages/varlock-website/src/content/docs/plugins/aws-secrets.mdx @@ -214,70 +214,28 @@ DB_PASSWORD=awsSecret("database-creds#password") DB_PORT=awsSecret("database-creds", key="port") ``` -Use a dot-separated path to read a nested object. If `database-creds` contains `{"db": {"host": "db.example.com"}}`, then `awsSecret("database-creds#db.host")` returns `db.example.com`. An exact top-level key with the same name takes precedence, so keys that contain dots keep working. Object and array values are returned as JSON text. +#### Nested JSON keys -### Nested JSON keys - -Secrets Manager stores a secret as text, and the most common layout is flat key-value pairs: one secret holds one JSON object, and each key is a single setting. Flat is the default for good reason. Every key sits at the top level, so any tool can read it, and a value needs no walk through nested objects. Varlock still parses the whole secret before it looks up a key, so the layout affects how you read the data, not how much is parsed. - -Nested objects are useful when a secret describes one thing with several parts. A database connection has a host, a port, a database name, and an SSL setting, and grouping them under `db` keeps them together. A secret that holds a whole application's configuration can have a section for the database, one for feature flags, and so on, and each section can be read on its own. - -Nested keys in Varlock are a read path, not a storage format. A secret can stay flat, be nested, or mix both. Varlock reads a dot-separated path such as `db.host` by walking the object one segment at a time. - -#### Example - -Suppose a team stores the settings for an API in one secret named `api/dev/config`: +Use a dot-separated path to read a value inside a nested object. Both the `#` shorthand and the `key` parameter accept paths. If `api/config` contains: ```json { - "db": { - "host": "db.dev.example.internal", - "port": 5432, - "name": "api", - "ssl": true - }, - "featureFlags": { - "newSearch": false - }, - "logLevel": "info" + "db": { "host": "db.example.internal", "port": 5432 }, + "featureFlags": { "newSearch": false } } ``` -The project's `.env.schema` reads individual values from that secret: - ```env-spec title=".env.schema" -# @plugin(@varlock/aws-secrets-plugin) -# @initAws(region=us-east-1) -# --- - -# @type=string -DB_HOST=awsSecret("api/dev/config#db.host") - +DB_HOST=awsSecret("api/config#db.host") # @type=number -DB_PORT=awsSecret("api/dev/config#db.port") - +DB_PORT=awsSecret("api/config", key="db.port") # @type=boolean -NEW_SEARCH=awsSecret("api/dev/config#featureFlags.newSearch") - -# The named key parameter reads paths the same way -LOG_LEVEL=awsSecret("api/dev/config", key="logLevel") +NEW_SEARCH=awsSecret("api/config#featureFlags.newSearch") ``` -Walking through it: - -1. **Store the secret.** Save the JSON above as the secret string of `api/dev/config`. Varlock doesn't need registering anything in the secret. It only needs the name. -2. **Point each setting at a path.** `db.host` is the segment `db`, then the key `host`. `featureFlags.newSearch` goes one level deeper in the same way. -3. **Types apply after the read.** The value is read as text, then `@type` checks and converts it. `DB_PORT` is declared as a number, so the `5432` in the JSON becomes a number in the environment. -4. **Use `#` or `key=` for the same result.** `awsSecret("name#db.host")` and `awsSecret("name", key="db.host")` run the same lookup. -5. **Objects and arrays come back as JSON text.** `awsSecret("api/dev/config#db")` returns the whole `db` object as a JSON string. Parse it in code, or read the fields you need as separate settings. - -If a path doesn't exist, Varlock stops with an error that names the missing key and lists the keys available where the walk stopped. A typo such as `db.hots` lists `host`, `port`, `name`, and `ssl` in the hint. - -#### Precedence and edge cases - -An exact top-level key wins over a path with the same text. If the secret holds both `"db.host": "flat"` and `"db": {"host": "nested"}`, then `#db.host` returns `flat`. Existing flat keys that contain dots keep working after an upgrade. - -A path with an empty segment resolves only when the JSON has an empty-string key at that position. For example, `db..host` reads `{"db": {"": {"host": "value"}}}`. In any other secret, `db..host` and a trailing dot fail the same way a missing key does. +- An exact top-level key wins over a path with the same text, so existing keys that contain dots (e.g. `"db.host"`) keep working. +- Object and array values are returned as JSON text, e.g. `#db` returns `{"host":"db.example.internal","port":5432}`. +- If a path doesn't exist, the error lists the keys available at the level where the lookup stopped. ### Name prefixing