[evict] Wire up limit flags in infra and documentation

DEVELOPMENT.md

@@ -23,6 +23,39 @@ Each project should contain the following elements:
 
 1. If applicable, design documentation may be captured in a `DESIGN.md` or a `design` directory.
 
+## Changing command-line flags or commands
+
+The binaries in [`cmds/`](cmds) (`core-service`, `db-manager migrate`, `db-manager evict`, ...) are part of the deployments.
+When adding, renaming or removing a flag or command, the change usually has to be propagated to the locations below:
+
+1. **Command documentation**: the flag reference and usage notes in [`docs/`](docs), e.g. [`docs/operations/cleanup.md`](docs/operations/cleanup.md) for `evict`.
+
+2. **Helm chart** ([`deploy/services/helm-charts/dss`](deploy/services/helm-charts/dss)):
+
+   - `values.yaml`
+   - `values.schema.json`
+   - `templates/*.yaml`
+
+3. **Tanka** ([`deploy/services/tanka`](deploy/services/tanka)):
+
+   - `metadata_base.libsonnet`
+   - The component's `.libsonnet` (e.g. `evict.libsonnet`)
+   - `examples/*/main.jsonnet`
+
+4. **Terraform** ([`deploy/infrastructure`](deploy/infrastructure)):
+
+   - `utils/definitions/<variable>.tf`
+   - `utils/variables.py`
+   - Run [`utils/generate_terraform_variables.sh`](deploy/infrastructure/utils/generate_terraform_variables.sh) to regenerate the `variables.gen.tf` and `TFVARS.gen.md` files.
+   - `modules/terraform-*-dss/main.tf`
+   - `dependencies/terraform-commons-dss/helm.tf`
+   - `dependencies/terraform-commons-dss/tanka.tf`
+
+5. **Local development and tests**:
+
+   - [`build/dev/docker-compose_dss.yaml`](build/dev/docker-compose_dss.yaml) and [`build/dev/startup`](build/dev/startup), if the flag is needed to run locally.
+   - Test helpers e.g. in [`test/evict/evict_helper.py`](test/evict/evict_helper.py), with a test case.
+
 ## Continuous integration
 
 Continuous integration shall be configured to run the following targets when applicable:

deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf

@@ -81,6 +81,7 @@ resource "local_file" "helm_chart_values" {
             schedule   = var.evict_scd_schedule
             ttl : var.evict_scd_ttl
             timeout : var.evict_scd_timeout
+            limit : var.evict_scd_limit
             operationalIntents : var.evict_scd_operational_intents
             subscriptions : var.evict_scd_subscriptions
           }
@@ -89,6 +90,7 @@ resource "local_file" "helm_chart_values" {
             schedule   = var.evict_rid_schedule
             ttl : var.evict_rid_ttl
             timeout : var.evict_rid_timeout
+            limit : var.evict_rid_limit
             ISAs : var.evict_rid_isas
             subscriptions : var.evict_rid_subscriptions
           }
@@ -294,6 +296,7 @@ resource "local_file" "helm_chart_values" {
             schedule   = var.evict_scd_schedule
             ttl : var.evict_scd_ttl
             timeout : var.evict_scd_timeout
+            limit : var.evict_scd_limit
             operationalIntents : var.evict_scd_operational_intents
             subscriptions : var.evict_scd_subscriptions
           }
@@ -302,6 +305,7 @@ resource "local_file" "helm_chart_values" {
             schedule   = var.evict_rid_schedule
             ttl : var.evict_rid_ttl
             timeout : var.evict_rid_timeout
+            limit : var.evict_rid_limit
             ISAs : var.evict_rid_isas
             subscriptions : var.evict_rid_subscriptions
           }

deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf

@@ -52,12 +52,14 @@ resource "local_file" "tanka_config_main" {
     VAR_EVICT_SCD_SCHEDULE                   = var.evict_scd_schedule
     VAR_EVICT_SCD_TTL                        = var.evict_scd_ttl
     VAR_EVICT_SCD_TIMEOUT                    = var.evict_scd_timeout
+    VAR_EVICT_SCD_LIMIT                      = var.evict_scd_limit
     VAR_EVICT_SCD_ENABLE_OPERATIONAL_INTENTS = var.evict_scd_operational_intents
     VAR_EVICT_SCD_ENABLE_SUBSCRIPTIONS       = var.evict_scd_subscriptions
     VAR_EVICT_ENABLE_RID_CRON                = var.evict_enable_rid_cron
     VAR_EVICT_RID_SCHEDULE                   = var.evict_rid_schedule
     VAR_EVICT_RID_TTL                        = var.evict_rid_ttl
     VAR_EVICT_RID_TIMEOUT                    = var.evict_rid_timeout
+    VAR_EVICT_RID_LIMIT                      = var.evict_rid_limit
     VAR_EVICT_RID_ENABLE_ISAS                = var.evict_rid_isas
     VAR_EVICT_RID_ENABLE_SUBSCRIPTIONS       = var.evict_rid_subscriptions
     VAR_PROMETHEUS_EXPOSE_EXTERNAL           = var.prometheus_hostname != ""

deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp

@@ -77,6 +77,7 @@ local metadata = metadataBase {
       schedule: "${VAR_EVICT_SCD_SCHEDULE}",
       ttl: "${VAR_EVICT_SCD_TTL}",
       timeout: "${VAR_EVICT_SCD_TIMEOUT}",
+      limit: ${VAR_EVICT_SCD_LIMIT},
       operational_intents: ${VAR_EVICT_SCD_ENABLE_OPERATIONAL_INTENTS},
       subscriptions: ${VAR_EVICT_SCD_ENABLE_SUBSCRIPTIONS},
     },
@@ -85,6 +86,7 @@ local metadata = metadataBase {
       schedule: "${VAR_EVICT_RID_SCHEDULE}",
       ttl: "${VAR_EVICT_RID_TTL}",
       timeout: "${VAR_EVICT_RID_TIMEOUT}",
+      limit: ${VAR_EVICT_RID_LIMIT},
       ISAs: ${VAR_EVICT_RID_ENABLE_ISAS},
       subscriptions: ${VAR_EVICT_RID_ENABLE_SUBSCRIPTIONS},
     },

deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf

@@ -439,6 +439,24 @@ variable "evict_scd_timeout" {
 }
 
 
+variable "evict_scd_limit" {
+  type        = number
+  description = <<-EOT
+  Maximum number of entities deleted by each run of the SCD eviction command.
+  Set to 0 to delete all expired entities in a single run.
+
+  Example: `1000`
+  EOT
+
+  default = 0
+
+  validation {
+    condition     = var.evict_scd_limit >= 0
+    error_message = "evict_scd_limit must be equal to or greater than 0."
+  }
+}
+
+
 variable "evict_scd_operational_intents" {
   type        = bool
   description = <<-EOT
@@ -507,6 +525,24 @@ variable "evict_rid_timeout" {
 }
 
 
+variable "evict_rid_limit" {
+  type        = number
+  description = <<-EOT
+  Maximum number of entities deleted by each run of the RID eviction command.
+  Set to 0 to delete all expired entities in a single run.
+
+  Example: `1000`
+  EOT
+
+  default = 0
+
+  validation {
+    condition     = var.evict_rid_limit >= 0
+    error_message = "evict_rid_limit must be equal to or greater than 0."
+  }
+}
+
+
 variable "evict_rid_isas" {
   type        = bool
   description = <<-EOT

deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md

@@ -217,6 +217,12 @@ Use <code>latest</code> to use the latest schema version.</p>
                 <td>evict_rid_isas (<code>bool</code>)</td>
                 <td><p>Set this to true to enable cleanup of RID ISAs.</p>
 <br/>Default value: <code>true</code></td>
+            </tr><tr>
+                <td>evict_rid_limit (<code>number</code>)</td>
+                <td><p>Maximum number of entities deleted by each run of the RID eviction command.
+Set to 0 to delete all expired entities in a single run.</p>
+<p>Example: <code>1000</code></p>
+<br/>Default value: <code>0</code></td>
             </tr><tr>
                 <td>evict_rid_schedule (<code>string</code>)</td>
                 <td><p>When the RID cleanup job shall be performed; expressed in cron format (https://crontab.guru/).</p>
@@ -235,6 +241,12 @@ Leave empty to use the default value of the command.</p>
                 <td>evict_rid_ttl (<code>string</code>)</td>
                 <td><p>How long expired RID items should stay before being automatically removed; expressed in Go duration format (https://pkg.go.dev/time#ParseDuration).</p>
 <br/>Default value: <code>"30m"</code></td>
+            </tr><tr>
+                <td>evict_scd_limit (<code>number</code>)</td>
+                <td><p>Maximum number of entities deleted by each run of the SCD eviction command.
+Set to 0 to delete all expired entities in a single run.</p>
+<p>Example: <code>1000</code></p>
+<br/>Default value: <code>0</code></td>
             </tr><tr>
                 <td>evict_scd_operational_intents (<code>bool</code>)</td>
                 <td><p>Set this to true to enable cleanup of SCD operational intents.</p>

deploy/infrastructure/modules/terraform-aws-dss/main.tf

@@ -52,12 +52,14 @@ module "terraform-commons-dss" {
   evict_scd_schedule                   = var.evict_scd_schedule
   evict_scd_ttl                        = var.evict_scd_ttl
   evict_scd_timeout                    = var.evict_scd_timeout
+  evict_scd_limit                      = var.evict_scd_limit
   evict_scd_operational_intents        = var.evict_scd_operational_intents
   evict_scd_subscriptions              = var.evict_scd_subscriptions
   evict_enable_rid_cron                = var.evict_enable_rid_cron
   evict_rid_schedule                   = var.evict_rid_schedule
   evict_rid_ttl                        = var.evict_rid_ttl
   evict_rid_timeout                    = var.evict_rid_timeout
+  evict_rid_limit                      = var.evict_rid_limit
   evict_rid_isas                       = var.evict_rid_isas
   evict_rid_subscriptions              = var.evict_rid_subscriptions
   enable_monitoring                    = var.enable_monitoring

deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf

@@ -566,6 +566,24 @@ variable "evict_scd_timeout" {
 }
 
 
+variable "evict_scd_limit" {
+  type        = number
+  description = <<-EOT
+  Maximum number of entities deleted by each run of the SCD eviction command.
+  Set to 0 to delete all expired entities in a single run.
+
+  Example: `1000`
+  EOT
+
+  default = 0
+
+  validation {
+    condition     = var.evict_scd_limit >= 0
+    error_message = "evict_scd_limit must be equal to or greater than 0."
+  }
+}
+
+
 variable "evict_scd_operational_intents" {
   type        = bool
   description = <<-EOT
@@ -634,6 +652,24 @@ variable "evict_rid_timeout" {
 }
 
 
+variable "evict_rid_limit" {
+  type        = number
+  description = <<-EOT
+  Maximum number of entities deleted by each run of the RID eviction command.
+  Set to 0 to delete all expired entities in a single run.
+
+  Example: `1000`
+  EOT
+
+  default = 0
+
+  validation {
+    condition     = var.evict_rid_limit >= 0
+    error_message = "evict_rid_limit must be equal to or greater than 0."
+  }
+}
+
+
 variable "evict_rid_isas" {
   type        = bool
   description = <<-EOT

deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md

@@ -174,6 +174,12 @@ Use <code>latest</code> to use the latest schema version.</p>
                 <td>evict_rid_isas (<code>bool</code>)</td>
                 <td><p>Set this to true to enable cleanup of RID ISAs.</p>
 <br/>Default value: <code>true</code></td>
+            </tr><tr>
+                <td>evict_rid_limit (<code>number</code>)</td>
+                <td><p>Maximum number of entities deleted by each run of the RID eviction command.
+Set to 0 to delete all expired entities in a single run.</p>
+<p>Example: <code>1000</code></p>
+<br/>Default value: <code>0</code></td>
             </tr><tr>
                 <td>evict_rid_schedule (<code>string</code>)</td>
                 <td><p>When the RID cleanup job shall be performed; expressed in cron format (https://crontab.guru/).</p>
@@ -192,6 +198,12 @@ Leave empty to use the default value of the command.</p>
                 <td>evict_rid_ttl (<code>string</code>)</td>
                 <td><p>How long expired RID items should stay before being automatically removed; expressed in Go duration format (https://pkg.go.dev/time#ParseDuration).</p>
 <br/>Default value: <code>"30m"</code></td>
+            </tr><tr>
+                <td>evict_scd_limit (<code>number</code>)</td>
+                <td><p>Maximum number of entities deleted by each run of the SCD eviction command.
+Set to 0 to delete all expired entities in a single run.</p>
+<p>Example: <code>1000</code></p>
+<br/>Default value: <code>0</code></td>
             </tr><tr>
                 <td>evict_scd_operational_intents (<code>bool</code>)</td>
                 <td><p>Set this to true to enable cleanup of SCD operational intents.</p>

deploy/infrastructure/modules/terraform-google-dss/main.tf

@@ -52,12 +52,14 @@ module "terraform-commons-dss" {
   evict_scd_schedule                   = var.evict_scd_schedule
   evict_scd_ttl                        = var.evict_scd_ttl
   evict_scd_timeout                    = var.evict_scd_timeout
+  evict_scd_limit                      = var.evict_scd_limit
   evict_scd_operational_intents        = var.evict_scd_operational_intents
   evict_scd_subscriptions              = var.evict_scd_subscriptions
   evict_enable_rid_cron                = var.evict_enable_rid_cron
   evict_rid_schedule                   = var.evict_rid_schedule
   evict_rid_ttl                        = var.evict_rid_ttl
   evict_rid_timeout                    = var.evict_rid_timeout
+  evict_rid_limit                      = var.evict_rid_limit
   evict_rid_isas                       = var.evict_rid_isas
   evict_rid_subscriptions              = var.evict_rid_subscriptions
   enable_monitoring                    = var.enable_monitoring

deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf

@@ -555,6 +555,24 @@ variable "evict_scd_timeout" {
 }
 
 
+variable "evict_scd_limit" {
+  type        = number
+  description = <<-EOT
+  Maximum number of entities deleted by each run of the SCD eviction command.
+  Set to 0 to delete all expired entities in a single run.
+
+  Example: `1000`
+  EOT
+
+  default = 0
+
+  validation {
+    condition     = var.evict_scd_limit >= 0
+    error_message = "evict_scd_limit must be equal to or greater than 0."
+  }
+}
+
+
 variable "evict_scd_operational_intents" {
   type        = bool
   description = <<-EOT
@@ -623,6 +641,24 @@ variable "evict_rid_timeout" {
 }
 
 
+variable "evict_rid_limit" {
+  type        = number
+  description = <<-EOT
+  Maximum number of entities deleted by each run of the RID eviction command.
+  Set to 0 to delete all expired entities in a single run.
+
+  Example: `1000`
+  EOT
+
+  default = 0
+
+  validation {
+    condition     = var.evict_rid_limit >= 0
+    error_message = "evict_rid_limit must be equal to or greater than 0."
+  }
+}
+
+
 variable "evict_rid_isas" {
   type        = bool
   description = <<-EOT

deploy/infrastructure/utils/definitions/evict_rid_limit.tf

@@ -0,0 +1,16 @@
+variable "evict_rid_limit" {
+  type        = number
+  description = <<-EOT
+  Maximum number of entities deleted by each run of the RID eviction command.
+  Set to 0 to delete all expired entities in a single run.
+
+  Example: `1000`
+  EOT
+
+  default = 0
+
+  validation {
+    condition     = var.evict_rid_limit >= 0
+    error_message = "evict_rid_limit must be equal to or greater than 0."
+  }
+}

deploy/infrastructure/utils/definitions/evict_scd_limit.tf

@@ -0,0 +1,16 @@
+variable "evict_scd_limit" {
+  type        = number
+  description = <<-EOT
+  Maximum number of entities deleted by each run of the SCD eviction command.
+  Set to 0 to delete all expired entities in a single run.
+
+  Example: `1000`
+  EOT
+
+  default = 0
+
+  validation {
+    condition     = var.evict_scd_limit >= 0
+    error_message = "evict_scd_limit must be equal to or greater than 0."
+  }
+}

deploy/infrastructure/utils/variables.py

@@ -61,12 +61,14 @@
     "evict_scd_schedule",
     "evict_scd_ttl",
     "evict_scd_timeout",
+    "evict_scd_limit",
     "evict_scd_operational_intents",
     "evict_scd_subscriptions",
     "evict_enable_rid_cron",
     "evict_rid_schedule",
     "evict_rid_ttl",
     "evict_rid_timeout",
+    "evict_rid_limit",
     "evict_rid_isas",
     "evict_rid_subscriptions",
     "enable_monitoring",