feat: Support one CI variable key in many environment scopes

Readme.md

@@ -99,6 +99,33 @@ The module accepts the following inputs:
 
 Refer to the variables.tf file for more details.
 
+## CI variables in many environment scopes
+
+A group or project can have one variable key in more than one `environment_scope`.
+Each key and `environment_scope` pair must be unique in its group or project. If it is not unique, the plan fails.
+
+The module puts masked and unmasked variables in different resources (`this_sensitive` and `this`).
+The `for_each` key of a variable depends on how many times its key occurs in its resource:
+
+| Key occurs | Project key | Group key |
+|---|---|---|
+| Once | `<namespace>-<project>-<KEY>` | `<group path>-<KEY>` |
+| More than once | `<namespace>-<project>-<KEY>:<environment_scope>` | `<group path>-<KEY>:<environment_scope>` |
+
+Thus the addresses of current variables do not change.
+
+**Caution:** If you add a second scope for a key that occurs once, the address of the first variable changes.
+The same occurs when you remove scopes until the key occurs once.
+Without a `moved` block, Terraform destroys the variable and creates it again.
+Add a `moved` block in your root module before you apply:
+
+```hcl
+moved {
+  from = module.gitlab.gitlab_project_variable.this["infra-api-UI_BUCKET"]
+  to   = module.gitlab.gitlab_project_variable.this["infra-api-UI_BUCKET:ui-uat"]
+}
+```
+
 ## Outputs
 
 The module provides the following outputs:

examples/terraform/projects/alpha_projects.yaml

@@ -3,7 +3,16 @@ projects:
     create: true
     namespace: "alpha"
     visibility: private
-    settings: {}
+    settings:
+      # One key in two environment scopes. The module adds the scope to the for_each key:
+      # "alpha-project-beta-DEPLOY_ROLE_ARN:uat" and "alpha-project-beta-DEPLOY_ROLE_ARN:production".
+      variables:
+        - key: "DEPLOY_ROLE_ARN"
+          value: "arn:aws:iam::111111111111:role/deploy"
+          environment_scope: "uat"
+        - key: "DEPLOY_ROLE_ARN"
+          value: "arn:aws:iam::222222222222:role/deploy"
+          environment_scope: "production"
   - name: project-gamma
     create: true
     namespace: "alpha"

main.tf

@@ -426,55 +426,87 @@ resource "gitlab_group_saml_link" "this" {
   saml_group_name = each.value.saml_link.saml_group_name
 }
 
+# CI variables of groups and projects.
+# The for_each key of a variable is "<owner>-<key>" when the key occurs once in its resource.
+# When the key occurs more than once in its resource, the key gets the suffix ":<environment_scope>".
+# Thus the addresses from before this rule do not change.
+# The validation of var.gitlab_groups and var.gitlab_projects makes sure that each key and environment_scope pair is unique.
+locals {
+  ci_variables = concat(
+    flatten([
+      for group in var.gitlab_groups : [
+        for variable in lookup(group.settings, "variables", []) : {
+          kind      = "group"
+          owner     = contains(keys(group), "parent") ? "${group.parent}/${group.name}" : group.name
+          parent    = lookup(group, "parent", null)
+          base_key  = contains(keys(group), "parent") ? "${group.parent}/${group.name}-${variable.key}" : "${group.name}-${variable.key}"
+          scope     = lookup(variable, "environment_scope", "*")
+          is_masked = lookup(variable, "masked", false) == true
+          variable  = variable
+        }
+      ]
+    ]),
+    flatten([
+      for project in var.gitlab_projects : [
+        for variable in lookup(project.settings, "variables", []) : {
+          kind      = "project"
+          owner     = "${project.namespace}/${project.name}"
+          parent    = null
+          base_key  = "${project.namespace}-${project.name}-${variable.key}"
+          scope     = lookup(variable, "environment_scope", "*")
+          is_masked = lookup(variable, "masked", false) == true
+          variable  = variable
+        }
+      ]
+    ]),
+  )
+
+  # Count each base key in its resource: the kind and the masked flag select the resource.
+  ci_variable_key_counts = {
+    for v in local.ci_variables : "${v.kind}:${v.is_masked}:${v.base_key}" => v.base_key...
+  }
+
+  ci_variables_keyed = [
+    for v in local.ci_variables : merge(v, {
+      key = (
+        length(local.ci_variable_key_counts["${v.kind}:${v.is_masked}:${v.base_key}"]) > 1
+        ? "${v.base_key}:${v.scope}"
+        : v.base_key
+      )
+    })
+  ]
+
+  group_variables_sensitive   = { for v in local.ci_variables_keyed : v.key => v if v.kind == "group" && v.is_masked }
+  group_variables             = { for v in local.ci_variables_keyed : v.key => v if v.kind == "group" && !v.is_masked }
+  project_variables_sensitive = { for v in local.ci_variables_keyed : v.key => v if v.kind == "project" && v.is_masked }
+  project_variables           = { for v in local.ci_variables_keyed : v.key => v if v.kind == "project" && !v.is_masked }
+}
+
 # Create GitLab Group Variables
 resource "gitlab_group_variable" "this_sensitive" {
-  for_each = merge([
-    for group in var.gitlab_groups : {
-      for variable in lookup(group.settings, "variables", []) : (
-        contains(keys(group), "parent")
-        ? "${group.parent}/${group.name}-${variable.key}" # Include parent in the key if it exists
-        : "${group.name}-${variable.key}"                 # Fallback to group name only if no parent
-        ) => {
-        group_name = contains(keys(group), "parent") ? "${group.parent}/${group.name}" : group.name
-        parent     = lookup(group, "parent", null)
-        variable   = variable
-      } if lookup(variable, "masked", false) == true # Only sensitive ones
-    }
-  ]...)
+  for_each = local.group_variables_sensitive
 
-  group             = each.value.parent == null ? gitlab_group.parent_groups[each.value.group_name].id : gitlab_group.subgroups[each.value.group_name].id
+  group             = each.value.parent == null ? gitlab_group.parent_groups[each.value.owner].id : gitlab_group.subgroups[each.value.owner].id
   key               = each.value.variable.key
   value             = sensitive(each.value.variable.value)
   protected         = lookup(each.value.variable, "protected", false)
   masked            = lookup(each.value.variable, "masked", false)
   hidden            = lookup(each.value.variable, "hidden", false)
-  environment_scope = lookup(each.value.variable, "environment_scope", "*")
+  environment_scope = each.value.scope
   description       = lookup(each.value.variable, "description", null)
   raw               = lookup(each.value.variable, "raw", false)
   variable_type     = lookup(each.value.variable, "variable_type", "env_var")
 }
 
 resource "gitlab_group_variable" "this" {
-  for_each = merge([
-    for group in var.gitlab_groups : {
-      for variable in lookup(group.settings, "variables", []) : (
-        contains(keys(group), "parent")
-        ? "${group.parent}/${group.name}-${variable.key}" # Include parent in the key if it exists
-        : "${group.name}-${variable.key}"                 # Fallback to group name only if no parent
-        ) => {
-        group_name = contains(keys(group), "parent") ? "${group.parent}/${group.name}" : group.name
-        parent     = lookup(group, "parent", null)
-        variable   = variable
-      } if lookup(variable, "masked", false) == false # Only non-sensitive ones
-    }
-  ]...)
+  for_each = local.group_variables
 
-  group             = each.value.parent == null ? gitlab_group.parent_groups[each.value.group_name].id : gitlab_group.subgroups[each.value.group_name].id
+  group             = each.value.parent == null ? gitlab_group.parent_groups[each.value.owner].id : gitlab_group.subgroups[each.value.owner].id
   key               = each.value.variable.key
   value             = each.value.variable.value
   protected         = lookup(each.value.variable, "protected", false)
   masked            = lookup(each.value.variable, "masked", false)
-  environment_scope = lookup(each.value.variable, "environment_scope", "*")
+  environment_scope = each.value.scope
   description       = lookup(each.value.variable, "description", null)
   raw               = lookup(each.value.variable, "raw", false)
   variable_type     = lookup(each.value.variable, "variable_type", "env_var")
@@ -1177,48 +1209,30 @@ resource "gitlab_project_share_group" "this" {
 }
 
 resource "gitlab_project_variable" "this_sensitive" {
-  for_each = merge([
-    for project in var.gitlab_projects : {
-      for variable in lookup(project.settings, "variables", []) :
-      "${project.namespace}-${project.name}-${variable.key}" => {
-        project_name      = project.name
-        project_namespace = project.namespace
-        variable          = variable
-      } if lookup(variable, "masked", false) == true # Only sensitive ones
-    }
-  ]...)
+  for_each = local.project_variables_sensitive
 
-  project           = gitlab_project.this["${each.value.project_namespace}/${each.value.project_name}"].id
+  project           = gitlab_project.this[each.value.owner].id
   key               = each.value.variable.key
   value             = sensitive(each.value.variable.value)
   protected         = lookup(each.value.variable, "protected", false)
   masked            = lookup(each.value.variable, "masked", false)
   hidden            = lookup(each.value.variable, "hidden", false)
-  environment_scope = lookup(each.value.variable, "environment_scope", "*")
+  environment_scope = each.value.scope
   description       = lookup(each.value.variable, "description", null)
   raw               = lookup(each.value.variable, "raw", false)
   variable_type     = lookup(each.value.variable, "variable_type", "env_var")
 }
 
 
 resource "gitlab_project_variable" "this" {
-  for_each = merge([
-    for project in var.gitlab_projects : {
-      for variable in lookup(project.settings, "variables", []) :
-      "${project.namespace}-${project.name}-${variable.key}" => {
-        project_name      = project.name
-        project_namespace = project.namespace
-        variable          = variable
-      } if lookup(variable, "masked", false) == false # Only non-sensitive ones
-    }
-  ]...)
+  for_each = local.project_variables
 
-  project           = gitlab_project.this["${each.value.project_namespace}/${each.value.project_name}"].id
+  project           = gitlab_project.this[each.value.owner].id
   key               = each.value.variable.key
   value             = each.value.variable.value
   protected         = lookup(each.value.variable, "protected", false)
   masked            = lookup(each.value.variable, "masked", false)
-  environment_scope = lookup(each.value.variable, "environment_scope", "*")
+  environment_scope = each.value.scope
   description       = lookup(each.value.variable, "description", null)
   raw               = lookup(each.value.variable, "raw", false)
   variable_type     = lookup(each.value.variable, "variable_type", "env_var")

tests/ci_variables.tftest.hcl

@@ -0,0 +1,160 @@
+# Unit tests for the for_each keys of CI variables. They use a mock provider and do not call GitLab.
+
+mock_provider "gitlab" {
+  # Some arguments take a project ID as a number. A random mock ID is not a number.
+  mock_resource "gitlab_project" {
+    defaults = { id = "1" }
+  }
+  mock_data "gitlab_project" {
+    defaults = { id = "1" }
+  }
+}
+
+variables {
+  gitlab_groups = [
+    {
+      name   = "alpha"
+      create = true
+      settings = {
+        variables = [
+          { key = "UNIQUE", value = "a" },
+          { key = "ROLE", value = "uat", environment_scope = "uat" },
+          { key = "ROLE", value = "prod", environment_scope = "production" },
+        ]
+      }
+    },
+  ]
+  gitlab_projects = [
+    {
+      name      = "api"
+      namespace = "alpha"
+      create    = true
+      settings = {
+        variables = [
+          { key = "UNIQUE", value = "a", environment_scope = "uat" },
+          { key = "SPLIT", value = "plain", environment_scope = "uat" },
+          { key = "SPLIT", value = "secret-value", environment_scope = "production", masked = true },
+          { key = "BUCKET", value = "uat", environment_scope = "uat" },
+          { key = "BUCKET", value = "prod", environment_scope = "production" },
+          { key = "TOKEN", value = "secret-uat", environment_scope = "uat", masked = true },
+          { key = "TOKEN", value = "secret-prod", environment_scope = "production", masked = true },
+        ]
+      }
+    },
+  ]
+}
+
+run "unique_keys_keep_the_old_address" {
+  command = plan
+
+  assert {
+    condition     = contains(keys(gitlab_project_variable.this), "alpha-api-UNIQUE")
+    error_message = "A unique key with a non-* scope must keep the address without the scope."
+  }
+
+  assert {
+    condition     = contains(keys(gitlab_group_variable.this), "alpha-UNIQUE")
+    error_message = "A unique group key must keep the address without the scope."
+  }
+
+  assert {
+    condition = (
+      contains(keys(gitlab_project_variable.this), "alpha-api-SPLIT") &&
+      contains(keys(gitlab_project_variable.this_sensitive), "alpha-api-SPLIT")
+    )
+    error_message = "A key that is unique in each resource must keep the address without the scope."
+  }
+}
+
+run "repeated_keys_get_the_scope" {
+  command = plan
+
+  assert {
+    condition = toset(keys(gitlab_project_variable.this)) == toset([
+      "alpha-api-UNIQUE", "alpha-api-SPLIT", "alpha-api-BUCKET:uat", "alpha-api-BUCKET:production",
+    ])
+    error_message = "Repeated unmasked project keys must get the scope suffix."
+  }
+
+  assert {
+    condition = toset(keys(gitlab_project_variable.this_sensitive)) == toset([
+      "alpha-api-SPLIT", "alpha-api-TOKEN:uat", "alpha-api-TOKEN:production",
+    ])
+    error_message = "Repeated masked project keys must get the scope suffix."
+  }
+
+  assert {
+    condition = toset(keys(gitlab_group_variable.this)) == toset([
+      "alpha-UNIQUE", "alpha-ROLE:uat", "alpha-ROLE:production",
+    ])
+    error_message = "Repeated group keys must get the scope suffix."
+  }
+
+  assert {
+    condition     = gitlab_project_variable.this["alpha-api-BUCKET:production"].environment_scope == "production"
+    error_message = "The scoped resource must use its environment_scope."
+  }
+}
+
+run "duplicate_project_key_and_scope_fails" {
+  command = plan
+
+  variables {
+    gitlab_projects = [
+      {
+        name      = "api"
+        namespace = "alpha"
+        create    = true
+        settings = {
+          variables = [
+            { key = "DUP", value = "plain", environment_scope = "uat" },
+            { key = "DUP", value = "secret-value", environment_scope = "uat", masked = true },
+          ]
+        }
+      },
+    ]
+  }
+
+  expect_failures = [var.gitlab_projects]
+}
+
+run "duplicate_group_key_and_default_scope_fails" {
+  command = plan
+
+  variables {
+    gitlab_groups = [
+      {
+        name     = "alpha"
+        create   = true
+        settings = { variables = [{ key = "DUP", value = "a" }, { key = "DUP", value = "b", environment_scope = "*" }] }
+      },
+    ]
+  }
+
+  expect_failures = [var.gitlab_groups]
+}
+
+run "example_creates_both_scoped_variables" {
+  command = plan
+
+  variables {
+    gitlab_groups = concat(
+      yamldecode(file("examples/terraform/groups/alpha.yaml")).groups,
+      yamldecode(file("examples/terraform/groups/roles.yaml")).groups,
+    )
+    gitlab_projects = yamldecode(file("examples/terraform/projects/alpha_projects.yaml")).projects
+  }
+
+  assert {
+    condition = alltrue([
+      for key in ["alpha-project-beta-DEPLOY_ROLE_ARN:uat", "alpha-project-beta-DEPLOY_ROLE_ARN:production"] :
+      contains(keys(gitlab_project_variable.this), key)
+    ])
+    error_message = "The example must create both scoped variables."
+  }
+
+  assert {
+    condition     = contains(keys(gitlab_project_variable.this), "alpha-project-alpha-VAR1")
+    error_message = "The unique key of the example must keep the address without the scope."
+  }
+}