Feature/aws provider update

README.md

@@ -26,16 +26,26 @@ npm run generate-provider
 ```
 output openapi3 specs (`components/schemas`) are written to `opneapi/src/aws/v00.00.00000/services`
 
-### 5. Test all metadata routes (services, resources, methods) in the provider:
+### 5. Test resource coverage in the provider:
+
+Verifies that every supported resource type in `provider-dev/config/cc_supported_resources.js` is accounted for in the generated provider:
+
+```bash
+npm run test-resource-coverage
+```
+
+### 6. Test all metadata routes (services, resources, methods) in the provider:
 
 ```bash
 PROVIDER_REGISTRY_ROOT_DIR="$(pwd)/openapi"
 npm run start-server -- --provider awscc --registry $PROVIDER_REGISTRY_ROOT_DIR
-npm run test-meta-routes -- awscc --ignore-no-methods
+npm run test-meta-routes -- awscc --ignore-no-methods --skip-resources awscc.tagging.tagged_resources
 npm run stop-server
 ```
 
-### 6. Testing locally with `stackql`
+*(`awscc.tagging.tagged_resources` is a native resource in a statically defined service; `DESCRIBE` does not return columns for it so it is excluded from the metadata route tests)*
+
+### 7. Testing locally with `stackql`
 1. ensure the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables are set
 2. download the latest `stackql` binary, for example `curl -L https://bit.ly/stackql-zip -O && unzip stackql-zip` for Linux systems
 3. run the following:
@@ -75,14 +85,14 @@ and TagFilters = '[{"Key": "StackName", "Values": ["stackql-serverless"]}]'
 where key = 'StackName' and value = 'stackql-serverless';
 ```
 
-### 6. Generate web docs:
+### 8. Generate web docs:
 
 ```bash
 npm run generate-docs
 ```
 output markdown docs are written to `website/docs`
 
-### 7. Test web docs locally
+### 9. Test web docs locally
 
 ```bash
 cd website
@@ -93,7 +103,7 @@ yarn build
 yarn start
 ```
 
-### 8. Publish web docs to GitHub Pages
+### 10. Publish web docs to GitHub Pages
 
 Under __Pages__ in the repository, in the __Build and deployment__ section select __GitHub Actions__ as the __Source__. In Netlify DNS create the following records:
 

bin/generate-provider.js

@@ -198,6 +198,11 @@ async function processService(servicePrefix, outputFilename) {
 
   const files = findFiles(docsDir, servicePrefix);
   const serviceTitle = path.basename(outputFilename, '.yaml');
+
+  // pre-read all supported docs so every resource schema name in the service is
+  // known before merging definitions, regardless of file processing order
+  const supportedDocs = [];
+  const reservedNames = new Set();
   for (const file of files) {
     const content = await fs.promises.readFile(file);
     const jsonContent = JSON.parse(content);
@@ -206,12 +211,17 @@ async function processService(servicePrefix, outputFilename) {
       console.log(`Skipping unsupported resource type: ${jsonContent.typeName}`);
       continue;
     }
+    supportedDocs.push(jsonContent);
+    reservedNames.add(jsonContent.typeName.split("::").pop());
+  }
 
+  for (const jsonContent of supportedDocs) {
     const componentName = jsonContent.typeName.split("::").pop();
     const openAPIComponent = convertToOpenAPI(
       jsonContent,
       componentName,
-      openAPI.components.schemas
+      openAPI.components.schemas,
+      reservedNames
     );
     Object.assign(openAPI.components.schemas, openAPIComponent);
   }
@@ -237,21 +247,16 @@ async function processService(servicePrefix, outputFilename) {
 
   const cleanedOpenAPI = cleanOpenAPISpec(openAPI);
 
-  if(serviceTitle == 'ec2'){
-    // fix bug with self referencing object
-    delete cleanedOpenAPI.components.schemas.SseSpecification.$ref;
-    cleanedOpenAPI.components.schemas.SseSpecification['type'] = 'object';
-    cleanedOpenAPI.components.schemas.SseSpecification['properties'] = {
-      KmsKeyArn: {
-        description: 'KMS Key Arn used to encrypt the group policy',
-        type: 'string'
-      },
-      CustomerManagedKeyEnabled: {
-        description: 'Whether to encrypt the policy with the provided key or disable encryption',
-        type: 'boolean'
+  // break self referencing schemas (broken upstream definitions, e.g. the
+  // SseSpecification definition in AWS::EC2::VerifiedAccessTrustProvider)
+  for (const [schemaName, schema] of Object.entries(cleanedOpenAPI.components.schemas || {})) {
+    if (schema && schema.$ref === `#/components/schemas/${schemaName}`) {
+      console.log(`Breaking self referencing schema ${schemaName} in ${serviceTitle}`);
+      delete schema.$ref;
+      if (!schema.type) {
+        schema.type = 'object';
       }
     }
-    cleanedOpenAPI.components.schemas.SseSpecification['additionalProperties'] = false;
   }
 
   // const finalAPI = addAdditionalRoutes(cleanedOpenAPI, serviceTitle);

bin/test-meta-routes.cjs

@@ -24,6 +24,7 @@ let verbose = false;
 let outputFormat = 'json';
 let timeoutMs = 60000; // Default timeout: 60 seconds
 let ignoreNoMethods = false;
+let skipResources = new Set();
 
 for (let i = 0; i < args.length; i++) {
   if (args[i].startsWith('--')) {
@@ -46,7 +47,10 @@ for (let i = 0; i < args.length; i++) {
         break;
       case '--ignore-no-methods':
         ignoreNoMethods = true;
-        break;        
+        break;
+      case '--skip-resources':
+        skipResources = new Set(args[++i].split(',').map(r => r.trim()).filter(r => r.length > 0));
+        break;
       case '--help':
         console.log(`
 Usage: test-meta-routes.js <provider> [OPTIONS]
@@ -62,6 +66,7 @@ Options:
   --format FORMAT           Output format: json, csv, markdown (default: json)
   --timeout MILLISECONDS    Query timeout in milliseconds (default: 60000)
   --ignore-no-methods       Skip resources with no methods (e.g., views)
+  --skip-resources LIST     Comma separated list of fully qualified resource names to skip
   --help                    Display this help message
         `);
         process.exit(0);
@@ -213,9 +218,15 @@ async function testMetaRoutes() {
       // for each resource
       for (const resource of resources) {
         const resourceName = resource.name;
-        console.log(`\n  🔹 Testing resource: ${resourceName}`);
-
+
         const resourceFQRN = `${provider}.${serviceName}.${resourceName}`;
+
+        if (skipResources.has(resourceFQRN)) {
+          console.log(`\n  🔹 Skipping resource: ${resourceName} (in --skip-resources)`);
+          continue;
+        }
+
+        console.log(`\n  🔹 Testing resource: ${resourceName}`);
         const resourceData = {
           name: resourceName,
           service: serviceName,

bin/test-resource-coverage.js

@@ -0,0 +1,112 @@
+// bin/test-resource-coverage.js
+//
+// Verifies that every resource type in provider-dev/config/cc_supported_resources.js
+// is present in the generated provider (openapi/src/awscc/.../services), with an
+// exact accounting for types that cannot be generated:
+//   - no schema file in provider-dev/downloaded
+//   - schema has no handlers (no Cloud Control operations to generate)
+// Fails if any supported type is missing for an unexplained reason, if the totals
+// do not reconcile, or if the provider contains a type not in the supported list.
+
+import * as fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { dirname } from 'path';
+import { load } from 'js-yaml';
+import { resourceTypes } from '../provider-dev/config/cc_supported_resources.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+const downloadedDir = path.join(__dirname, '../provider-dev/downloaded');
+const servicesDir = path.join(__dirname, '../openapi/src/awscc/v00.00.00000/services');
+
+const supported = new Set(resourceTypes);
+
+// index downloaded schemas by typeName
+const downloaded = {};
+for (const file of fs.readdirSync(downloadedDir)) {
+  if (!file.endsWith('.json')) continue;
+  const doc = JSON.parse(fs.readFileSync(path.join(downloadedDir, file), 'utf8'));
+  if (doc.typeName) {
+    downloaded[doc.typeName] = {
+      file,
+      hasHandlers: !!(doc.handlers && Object.keys(doc.handlers).length > 0),
+    };
+  }
+}
+
+// collect distinct x-cfn-type-name values from generated resources
+const generated = new Set();
+for (const file of fs.readdirSync(servicesDir)) {
+  if (!file.endsWith('.yaml')) continue;
+  const doc = load(fs.readFileSync(path.join(servicesDir, file), 'utf8'));
+  const resources = (doc.components && doc.components['x-stackQL-resources']) || {};
+  for (const resourceDef of Object.values(resources)) {
+    if (resourceDef && resourceDef['x-cfn-type-name']) {
+      generated.add(resourceDef['x-cfn-type-name']);
+    }
+  }
+}
+
+const noSchema = [];
+const noHandlers = [];
+const unexplainedMissing = [];
+
+for (const typeName of [...supported].sort()) {
+  if (generated.has(typeName)) continue;
+  if (!downloaded[typeName]) {
+    noSchema.push(typeName);
+  } else if (!downloaded[typeName].hasHandlers) {
+    noHandlers.push(typeName);
+  } else {
+    unexplainedMissing.push(typeName);
+  }
+}
+
+const notSupported = [...generated].filter(t => !supported.has(t)).sort();
+
+console.log('Resource coverage summary');
+console.log('-------------------------');
+console.log(`supported resource types (cc_supported_resources.js): ${supported.size}`);
+console.log(`resource types in generated provider:                 ${generated.size}`);
+console.log(`excluded - no schema in provider-dev/downloaded:      ${noSchema.length}`);
+console.log(`excluded - schema has no handlers:                    ${noHandlers.length}`);
+console.log(`missing for an unexplained reason:                    ${unexplainedMissing.length}`);
+console.log(`generated but not in supported list:                  ${notSupported.length}`);
+
+if (noSchema.length > 0) {
+  console.log('\ntypes with no downloaded schema:');
+  noSchema.forEach(t => console.log(`  ${t}`));
+}
+
+if (noHandlers.length > 0) {
+  console.log('\ntypes with no handlers in schema (nothing to generate):');
+  noHandlers.forEach(t => console.log(`  ${t}`));
+}
+
+let failed = false;
+
+if (unexplainedMissing.length > 0) {
+  console.error('\nFAIL: supported types missing from the generated provider:');
+  unexplainedMissing.forEach(t => console.error(`  ${t}`));
+  failed = true;
+}
+
+if (notSupported.length > 0) {
+  console.error('\nFAIL: generated types not present in the supported list:');
+  notSupported.forEach(t => console.error(`  ${t}`));
+  failed = true;
+}
+
+const reconciled = generated.size + noSchema.length + noHandlers.length + unexplainedMissing.length === supported.size;
+if (!reconciled) {
+  console.error(`\nFAIL: totals do not reconcile: ${generated.size} generated + ${noSchema.length} no schema + ${noHandlers.length} no handlers != ${supported.size} supported`);
+  failed = true;
+}
+
+if (failed) {
+  process.exit(1);
+}
+
+console.log('\nPASS: all supported resource types are accounted for in the generated provider');

lib/utils/openapi-utils.js

@@ -86,7 +86,11 @@ function processObjectProperties(object, parentKey, grandparentKey) {
   let result = {};
 
   for (const [key, val] of Object.entries(object)) {
-    if (isReferenceKey(key, val)) {
+    if (key === "x-stackQL-resources") {
+      // stackql resource config is not a JSON schema; the key prefixing
+      // heuristic misfires on resources named 'schemas' or 'properties'
+      result[key] = val;
+    } else if (isReferenceKey(key, val)) {
       result[key] = updateReference(val);
     } else if (needsPrefix(key, parentKey, grandparentKey)) {
       result[`x-${key}`] = processProperties(val, key, parentKey);
@@ -104,9 +108,51 @@ function processArrayProperties(array, parentKey, grandparentKey) {
   );
 }
 
-export function convertToOpenAPI(input, componentName, schemaDefinitions) {
+function resolveDefinitionCollisions(input, formattedComponentName, schemaDefinitions, reservedNames) {
+  // definitions and top level resource schemas share one flat components.schemas
+  // namespace per service; a definition that collides with a resource schema name
+  // (or with a differing definition from a sibling file) is renamed to
+  // <ComponentName>_<Key> and its $refs are rewritten to match
+  const renames = {};
+  let current = input;
+
+  while (true) {
+    let added = false;
+    for (const [key, value] of Object.entries(current.definitions || {})) {
+      if (renames[key]) continue;
+      const collidesWithResource = reservedNames.has(key);
+      const collidesWithExisting = key in schemaDefinitions && !_.isEqual(schemaDefinitions[key], value);
+      if (collidesWithResource || collidesWithExisting) {
+        renames[key] = `${formattedComponentName}_${key}`;
+        added = true;
+      }
+    }
+    if (!added) break;
+
+    // re-apply all renames to the original input so ref rewrites stay consistent
+    let serialized = JSON.stringify(input);
+    for (const [oldKey, newKey] of Object.entries(renames)) {
+      serialized = serialized.split(`"#/definitions/${oldKey}"`).join(`"#/definitions/${newKey}"`);
+    }
+    current = JSON.parse(serialized);
+  }
+
+  if (Object.keys(renames).length > 0) {
+    console.log(`Renamed colliding definition(s) in ${formattedComponentName}: ${Object.entries(renames).map(([o, n]) => `${o} -> ${n}`).join(', ')}`);
+  }
+
+  return { input: current, renames };
+}
+
+export function convertToOpenAPI(input, componentName, schemaDefinitions, reservedNames) {
 
   const formattedComponentName = formatComponentName(componentName);
+
+  let definitionRenames = {};
+  if (reservedNames) {
+    ({ input, renames: definitionRenames } = resolveDefinitionCollisions(input, formattedComponentName, schemaDefinitions, reservedNames));
+  }
+
   const requiredProperties = input.required;
   const openAPIComponent = createOpenAPIComponent(
     formattedComponentName,
@@ -232,7 +278,7 @@ export function convertToOpenAPI(input, componentName, schemaDefinitions) {
   }
 
   for (const [key, value] of Object.entries(input.definitions || {})) {
-    schemaDefinitions[key] = value;
+    schemaDefinitions[definitionRenames[key] || key] = value;
   }
 
   return { [formattedComponentName]: openAPIComponent };

lib/utils/stackql-utils.js

@@ -2,6 +2,10 @@
 
 import pluralize from 'pluralize';
 
+// pluralize returns 'schemata' for 'schema', which collides with the
+// information_schema.schemata table handling in stackql
+pluralize.addIrregularRule('schema', 'schemas');
+
 const providerName = 'awscc';
 
 function fixReservedWordColumnAlias(columnAlias) {
@@ -19,7 +23,9 @@ function fixReservedWordColumnAlias(columnAlias) {
     case 'force':
       return `_${columnAlias}`;
     case 'match':
-      return `_${columnAlias}`;                      
+      return `_${columnAlias}`;
+    case 'registry':
+      return `_${columnAlias}`;
     default:
       return columnAlias;
   }

openapi/src/awscc/v00.00.00000/services/accessanalyzer.yaml

@@ -694,7 +694,7 @@ components:
       id: awscc.accessanalyzer.analyzers
       x-cfn-schema-name: Analyzer
       x-cfn-type-name: AWS::AccessAnalyzer::Analyzer
-      x-identifiers:
+      x-identifiers: &ref_0
         - Arn
       x-type: cloud_control
       methods:
@@ -790,8 +790,7 @@ components:
       id: awscc.accessanalyzer.analyzers_list_only
       x-cfn-schema-name: Analyzer
       x-cfn-type-name: AWS::AccessAnalyzer::Analyzer
-      x-identifiers:
-        - Arn
+      x-identifiers: *ref_0
       x-type: cloud_control_view
       methods: {}
       sqlVerbs: