From 172043eb2b94e281b8059edba50f3e8ef0d1a3f0 Mon Sep 17 00:00:00 2001 From: Mariem Baccari Date: Mon, 5 Oct 2026 12:38:14 +0200 Subject: [PATCH] [evict] Wire up limit flags in infra and documentation --- DEVELOPMENT.md | 33 ++++++ .../terraform-commons-dss/helm.tf | 4 + .../terraform-commons-dss/tanka.tf | 2 + .../templates/main.jsonnet.tmp | 2 + .../terraform-commons-dss/variables.gen.tf | 36 +++++++ .../modules/terraform-aws-dss/TFVARS.gen.md | 12 +++ .../modules/terraform-aws-dss/main.tf | 2 + .../terraform-aws-dss/variables.gen.tf | 36 +++++++ .../terraform-google-dss/TFVARS.gen.md | 12 +++ .../modules/terraform-google-dss/main.tf | 2 + .../terraform-google-dss/variables.gen.tf | 36 +++++++ .../utils/definitions/evict_rid_limit.tf | 16 +++ .../utils/definitions/evict_scd_limit.tf | 16 +++ deploy/infrastructure/utils/variables.py | 2 + deploy/operations/ci/aws-1/variables.gen.tf | 36 +++++++ .../helm-charts/dss/templates/dss-evict.yaml | 6 ++ .../helm-charts/dss/values.schema.json | 12 ++- deploy/services/helm-charts/dss/values.yaml | 2 + deploy/services/tanka/evict.libsonnet | 2 + .../tanka/examples/minikube/main.jsonnet | 2 + .../tanka/examples/minimum/main.jsonnet | 2 + deploy/services/tanka/metadata_base.libsonnet | 2 + docs/operations/cleanup.md | 8 +- test/evict/evict_helper.py | 46 ++++++-- test/evict/query_helper.py | 6 +- test/evict/tests/test_limit.py | 102 ++++++++++++++++++ 26 files changed, 424 insertions(+), 13 deletions(-) create mode 100644 deploy/infrastructure/utils/definitions/evict_rid_limit.tf create mode 100644 deploy/infrastructure/utils/definitions/evict_scd_limit.tf create mode 100644 test/evict/tests/test_limit.py diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index dcc1e73fe..7e65447a4 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -23,6 +23,39 @@ Each project should contain the following elements: 1. If applicable, design documentation may be captured in a `DESIGN.md` or a `design` directory. +## Changing command-line flags or commands + +The binaries in [`cmds/`](cmds) (`core-service`, `db-manager migrate`, `db-manager evict`, ...) are part of the deployments. +When adding, renaming or removing a flag or command, the change usually has to be propagated to the locations below: + +1. **Command documentation**: the flag reference and usage notes in [`docs/`](docs), e.g. [`docs/operations/cleanup.md`](docs/operations/cleanup.md) for `evict`. + +2. **Helm chart** ([`deploy/services/helm-charts/dss`](deploy/services/helm-charts/dss)): + + - `values.yaml` + - `values.schema.json` + - `templates/*.yaml` + +3. **Tanka** ([`deploy/services/tanka`](deploy/services/tanka)): + + - `metadata_base.libsonnet` + - The component's `.libsonnet` (e.g. `evict.libsonnet`) + - `examples/*/main.jsonnet` + +4. **Terraform** ([`deploy/infrastructure`](deploy/infrastructure)): + + - `utils/definitions/.tf` + - `utils/variables.py` + - Run [`utils/generate_terraform_variables.sh`](deploy/infrastructure/utils/generate_terraform_variables.sh) to regenerate the `variables.gen.tf` and `TFVARS.gen.md` files. + - `modules/terraform-*-dss/main.tf` + - `dependencies/terraform-commons-dss/helm.tf` + - `dependencies/terraform-commons-dss/tanka.tf` + +5. **Local development and tests**: + + - [`build/dev/docker-compose_dss.yaml`](build/dev/docker-compose_dss.yaml) and [`build/dev/startup`](build/dev/startup), if the flag is needed to run locally. + - Test helpers e.g. in [`test/evict/evict_helper.py`](test/evict/evict_helper.py), with a test case. + ## Continuous integration Continuous integration shall be configured to run the following targets when applicable: diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf index 12e86e805..50b801b87 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf @@ -81,6 +81,7 @@ resource "local_file" "helm_chart_values" { schedule = var.evict_scd_schedule ttl : var.evict_scd_ttl timeout : var.evict_scd_timeout + limit : var.evict_scd_limit operationalIntents : var.evict_scd_operational_intents subscriptions : var.evict_scd_subscriptions } @@ -89,6 +90,7 @@ resource "local_file" "helm_chart_values" { schedule = var.evict_rid_schedule ttl : var.evict_rid_ttl timeout : var.evict_rid_timeout + limit : var.evict_rid_limit ISAs : var.evict_rid_isas subscriptions : var.evict_rid_subscriptions } @@ -294,6 +296,7 @@ resource "local_file" "helm_chart_values" { schedule = var.evict_scd_schedule ttl : var.evict_scd_ttl timeout : var.evict_scd_timeout + limit : var.evict_scd_limit operationalIntents : var.evict_scd_operational_intents subscriptions : var.evict_scd_subscriptions } @@ -302,6 +305,7 @@ resource "local_file" "helm_chart_values" { schedule = var.evict_rid_schedule ttl : var.evict_rid_ttl timeout : var.evict_rid_timeout + limit : var.evict_rid_limit ISAs : var.evict_rid_isas subscriptions : var.evict_rid_subscriptions } diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf index 0f98c3f90..788123feb 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/tanka.tf @@ -52,12 +52,14 @@ resource "local_file" "tanka_config_main" { VAR_EVICT_SCD_SCHEDULE = var.evict_scd_schedule VAR_EVICT_SCD_TTL = var.evict_scd_ttl VAR_EVICT_SCD_TIMEOUT = var.evict_scd_timeout + VAR_EVICT_SCD_LIMIT = var.evict_scd_limit VAR_EVICT_SCD_ENABLE_OPERATIONAL_INTENTS = var.evict_scd_operational_intents VAR_EVICT_SCD_ENABLE_SUBSCRIPTIONS = var.evict_scd_subscriptions VAR_EVICT_ENABLE_RID_CRON = var.evict_enable_rid_cron VAR_EVICT_RID_SCHEDULE = var.evict_rid_schedule VAR_EVICT_RID_TTL = var.evict_rid_ttl VAR_EVICT_RID_TIMEOUT = var.evict_rid_timeout + VAR_EVICT_RID_LIMIT = var.evict_rid_limit VAR_EVICT_RID_ENABLE_ISAS = var.evict_rid_isas VAR_EVICT_RID_ENABLE_SUBSCRIPTIONS = var.evict_rid_subscriptions VAR_PROMETHEUS_EXPOSE_EXTERNAL = var.prometheus_hostname != "" diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp b/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp index 28f128d7f..eeca5f9f1 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/templates/main.jsonnet.tmp @@ -77,6 +77,7 @@ local metadata = metadataBase { schedule: "${VAR_EVICT_SCD_SCHEDULE}", ttl: "${VAR_EVICT_SCD_TTL}", timeout: "${VAR_EVICT_SCD_TIMEOUT}", + limit: ${VAR_EVICT_SCD_LIMIT}, operational_intents: ${VAR_EVICT_SCD_ENABLE_OPERATIONAL_INTENTS}, subscriptions: ${VAR_EVICT_SCD_ENABLE_SUBSCRIPTIONS}, }, @@ -85,6 +86,7 @@ local metadata = metadataBase { schedule: "${VAR_EVICT_RID_SCHEDULE}", ttl: "${VAR_EVICT_RID_TTL}", timeout: "${VAR_EVICT_RID_TIMEOUT}", + limit: ${VAR_EVICT_RID_LIMIT}, ISAs: ${VAR_EVICT_RID_ENABLE_ISAS}, subscriptions: ${VAR_EVICT_RID_ENABLE_SUBSCRIPTIONS}, }, diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf index 778a587f5..d4012be3a 100644 --- a/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf +++ b/deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf @@ -439,6 +439,24 @@ variable "evict_scd_timeout" { } +variable "evict_scd_limit" { + type = number + description = <<-EOT + Maximum number of entities deleted by each run of the SCD eviction command. + Set to 0 to delete all expired entities in a single run. + + Example: `1000` + EOT + + default = 0 + + validation { + condition = var.evict_scd_limit >= 0 + error_message = "evict_scd_limit must be equal to or greater than 0." + } +} + + variable "evict_scd_operational_intents" { type = bool description = <<-EOT @@ -507,6 +525,24 @@ variable "evict_rid_timeout" { } +variable "evict_rid_limit" { + type = number + description = <<-EOT + Maximum number of entities deleted by each run of the RID eviction command. + Set to 0 to delete all expired entities in a single run. + + Example: `1000` + EOT + + default = 0 + + validation { + condition = var.evict_rid_limit >= 0 + error_message = "evict_rid_limit must be equal to or greater than 0." + } +} + + variable "evict_rid_isas" { type = bool description = <<-EOT diff --git a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md index 71231e280..15a42ee72 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md +++ b/deploy/infrastructure/modules/terraform-aws-dss/TFVARS.gen.md @@ -217,6 +217,12 @@ Use latest to use the latest schema version.

evict_rid_isas (bool)

Set this to true to enable cleanup of RID ISAs.


Default value: true + + evict_rid_limit (number) +

Maximum number of entities deleted by each run of the RID eviction command. +Set to 0 to delete all expired entities in a single run.

+

Example: 1000

+
Default value: 0 evict_rid_schedule (string)

When the RID cleanup job shall be performed; expressed in cron format (https://crontab.guru/).

@@ -235,6 +241,12 @@ Leave empty to use the default value of the command.

evict_rid_ttl (string)

How long expired RID items should stay before being automatically removed; expressed in Go duration format (https://pkg.go.dev/time#ParseDuration).


Default value: "30m" + + evict_scd_limit (number) +

Maximum number of entities deleted by each run of the SCD eviction command. +Set to 0 to delete all expired entities in a single run.

+

Example: 1000

+
Default value: 0 evict_scd_operational_intents (bool)

Set this to true to enable cleanup of SCD operational intents.

diff --git a/deploy/infrastructure/modules/terraform-aws-dss/main.tf b/deploy/infrastructure/modules/terraform-aws-dss/main.tf index 289f18a6c..23d577b1b 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/main.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/main.tf @@ -52,12 +52,14 @@ module "terraform-commons-dss" { evict_scd_schedule = var.evict_scd_schedule evict_scd_ttl = var.evict_scd_ttl evict_scd_timeout = var.evict_scd_timeout + evict_scd_limit = var.evict_scd_limit evict_scd_operational_intents = var.evict_scd_operational_intents evict_scd_subscriptions = var.evict_scd_subscriptions evict_enable_rid_cron = var.evict_enable_rid_cron evict_rid_schedule = var.evict_rid_schedule evict_rid_ttl = var.evict_rid_ttl evict_rid_timeout = var.evict_rid_timeout + evict_rid_limit = var.evict_rid_limit evict_rid_isas = var.evict_rid_isas evict_rid_subscriptions = var.evict_rid_subscriptions enable_monitoring = var.enable_monitoring diff --git a/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf b/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf index f156bd93e..c20e043de 100644 --- a/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf +++ b/deploy/infrastructure/modules/terraform-aws-dss/variables.gen.tf @@ -566,6 +566,24 @@ variable "evict_scd_timeout" { } +variable "evict_scd_limit" { + type = number + description = <<-EOT + Maximum number of entities deleted by each run of the SCD eviction command. + Set to 0 to delete all expired entities in a single run. + + Example: `1000` + EOT + + default = 0 + + validation { + condition = var.evict_scd_limit >= 0 + error_message = "evict_scd_limit must be equal to or greater than 0." + } +} + + variable "evict_scd_operational_intents" { type = bool description = <<-EOT @@ -634,6 +652,24 @@ variable "evict_rid_timeout" { } +variable "evict_rid_limit" { + type = number + description = <<-EOT + Maximum number of entities deleted by each run of the RID eviction command. + Set to 0 to delete all expired entities in a single run. + + Example: `1000` + EOT + + default = 0 + + validation { + condition = var.evict_rid_limit >= 0 + error_message = "evict_rid_limit must be equal to or greater than 0." + } +} + + variable "evict_rid_isas" { type = bool description = <<-EOT diff --git a/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md b/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md index d3fdc093a..e757c4e67 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md +++ b/deploy/infrastructure/modules/terraform-google-dss/TFVARS.gen.md @@ -174,6 +174,12 @@ Use latest to use the latest schema version.

evict_rid_isas (bool)

Set this to true to enable cleanup of RID ISAs.


Default value: true + + evict_rid_limit (number) +

Maximum number of entities deleted by each run of the RID eviction command. +Set to 0 to delete all expired entities in a single run.

+

Example: 1000

+
Default value: 0 evict_rid_schedule (string)

When the RID cleanup job shall be performed; expressed in cron format (https://crontab.guru/).

@@ -192,6 +198,12 @@ Leave empty to use the default value of the command.

evict_rid_ttl (string)

How long expired RID items should stay before being automatically removed; expressed in Go duration format (https://pkg.go.dev/time#ParseDuration).


Default value: "30m" + + evict_scd_limit (number) +

Maximum number of entities deleted by each run of the SCD eviction command. +Set to 0 to delete all expired entities in a single run.

+

Example: 1000

+
Default value: 0 evict_scd_operational_intents (bool)

Set this to true to enable cleanup of SCD operational intents.

diff --git a/deploy/infrastructure/modules/terraform-google-dss/main.tf b/deploy/infrastructure/modules/terraform-google-dss/main.tf index da5b8a7a1..8cf9088db 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/main.tf +++ b/deploy/infrastructure/modules/terraform-google-dss/main.tf @@ -52,12 +52,14 @@ module "terraform-commons-dss" { evict_scd_schedule = var.evict_scd_schedule evict_scd_ttl = var.evict_scd_ttl evict_scd_timeout = var.evict_scd_timeout + evict_scd_limit = var.evict_scd_limit evict_scd_operational_intents = var.evict_scd_operational_intents evict_scd_subscriptions = var.evict_scd_subscriptions evict_enable_rid_cron = var.evict_enable_rid_cron evict_rid_schedule = var.evict_rid_schedule evict_rid_ttl = var.evict_rid_ttl evict_rid_timeout = var.evict_rid_timeout + evict_rid_limit = var.evict_rid_limit evict_rid_isas = var.evict_rid_isas evict_rid_subscriptions = var.evict_rid_subscriptions enable_monitoring = var.enable_monitoring diff --git a/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf b/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf index ad9723988..ad306b72d 100644 --- a/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf +++ b/deploy/infrastructure/modules/terraform-google-dss/variables.gen.tf @@ -555,6 +555,24 @@ variable "evict_scd_timeout" { } +variable "evict_scd_limit" { + type = number + description = <<-EOT + Maximum number of entities deleted by each run of the SCD eviction command. + Set to 0 to delete all expired entities in a single run. + + Example: `1000` + EOT + + default = 0 + + validation { + condition = var.evict_scd_limit >= 0 + error_message = "evict_scd_limit must be equal to or greater than 0." + } +} + + variable "evict_scd_operational_intents" { type = bool description = <<-EOT @@ -623,6 +641,24 @@ variable "evict_rid_timeout" { } +variable "evict_rid_limit" { + type = number + description = <<-EOT + Maximum number of entities deleted by each run of the RID eviction command. + Set to 0 to delete all expired entities in a single run. + + Example: `1000` + EOT + + default = 0 + + validation { + condition = var.evict_rid_limit >= 0 + error_message = "evict_rid_limit must be equal to or greater than 0." + } +} + + variable "evict_rid_isas" { type = bool description = <<-EOT diff --git a/deploy/infrastructure/utils/definitions/evict_rid_limit.tf b/deploy/infrastructure/utils/definitions/evict_rid_limit.tf new file mode 100644 index 000000000..c0dab4e93 --- /dev/null +++ b/deploy/infrastructure/utils/definitions/evict_rid_limit.tf @@ -0,0 +1,16 @@ +variable "evict_rid_limit" { + type = number + description = <<-EOT + Maximum number of entities deleted by each run of the RID eviction command. + Set to 0 to delete all expired entities in a single run. + + Example: `1000` + EOT + + default = 0 + + validation { + condition = var.evict_rid_limit >= 0 + error_message = "evict_rid_limit must be equal to or greater than 0." + } +} diff --git a/deploy/infrastructure/utils/definitions/evict_scd_limit.tf b/deploy/infrastructure/utils/definitions/evict_scd_limit.tf new file mode 100644 index 000000000..815f7bdea --- /dev/null +++ b/deploy/infrastructure/utils/definitions/evict_scd_limit.tf @@ -0,0 +1,16 @@ +variable "evict_scd_limit" { + type = number + description = <<-EOT + Maximum number of entities deleted by each run of the SCD eviction command. + Set to 0 to delete all expired entities in a single run. + + Example: `1000` + EOT + + default = 0 + + validation { + condition = var.evict_scd_limit >= 0 + error_message = "evict_scd_limit must be equal to or greater than 0." + } +} diff --git a/deploy/infrastructure/utils/variables.py b/deploy/infrastructure/utils/variables.py index 2d42f18d9..be8457e76 100755 --- a/deploy/infrastructure/utils/variables.py +++ b/deploy/infrastructure/utils/variables.py @@ -61,12 +61,14 @@ "evict_scd_schedule", "evict_scd_ttl", "evict_scd_timeout", + "evict_scd_limit", "evict_scd_operational_intents", "evict_scd_subscriptions", "evict_enable_rid_cron", "evict_rid_schedule", "evict_rid_ttl", "evict_rid_timeout", + "evict_rid_limit", "evict_rid_isas", "evict_rid_subscriptions", "enable_monitoring", diff --git a/deploy/operations/ci/aws-1/variables.gen.tf b/deploy/operations/ci/aws-1/variables.gen.tf index f156bd93e..c20e043de 100644 --- a/deploy/operations/ci/aws-1/variables.gen.tf +++ b/deploy/operations/ci/aws-1/variables.gen.tf @@ -566,6 +566,24 @@ variable "evict_scd_timeout" { } +variable "evict_scd_limit" { + type = number + description = <<-EOT + Maximum number of entities deleted by each run of the SCD eviction command. + Set to 0 to delete all expired entities in a single run. + + Example: `1000` + EOT + + default = 0 + + validation { + condition = var.evict_scd_limit >= 0 + error_message = "evict_scd_limit must be equal to or greater than 0." + } +} + + variable "evict_scd_operational_intents" { type = bool description = <<-EOT @@ -634,6 +652,24 @@ variable "evict_rid_timeout" { } +variable "evict_rid_limit" { + type = number + description = <<-EOT + Maximum number of entities deleted by each run of the RID eviction command. + Set to 0 to delete all expired entities in a single run. + + Example: `1000` + EOT + + default = 0 + + validation { + condition = var.evict_rid_limit >= 0 + error_message = "evict_rid_limit must be equal to or greater than 0." + } +} + + variable "evict_rid_isas" { type = bool description = <<-EOT diff --git a/deploy/services/helm-charts/dss/templates/dss-evict.yaml b/deploy/services/helm-charts/dss/templates/dss-evict.yaml index eecb88edb..618c6ab4b 100644 --- a/deploy/services/helm-charts/dss/templates/dss-evict.yaml +++ b/deploy/services/helm-charts/dss/templates/dss-evict.yaml @@ -41,6 +41,9 @@ spec: - --scd_ttl={{ $.Values.dss.conf.evict.scd.ttl }} {{- if $.Values.dss.conf.evict.scd.timeout }} - --timeout={{ $.Values.dss.conf.evict.scd.timeout }} + {{- end }} + {{- if $.Values.dss.conf.evict.scd.limit }} + - --scd_limit={{ $.Values.dss.conf.evict.scd.limit }} {{- end }} - --locality={{ $.Values.dss.conf.locality }} - --delete=true @@ -100,6 +103,9 @@ spec: - --rid_ttl={{ $.Values.dss.conf.evict.rid.ttl }} {{- if $.Values.dss.conf.evict.rid.timeout }} - --timeout={{ $.Values.dss.conf.evict.rid.timeout }} + {{- end }} + {{- if $.Values.dss.conf.evict.rid.limit }} + - --rid_limit={{ $.Values.dss.conf.evict.rid.limit }} {{- end }} - --locality={{ $.Values.dss.conf.locality }} - --delete=true diff --git a/deploy/services/helm-charts/dss/values.schema.json b/deploy/services/helm-charts/dss/values.schema.json index bb88df3b0..5e9fa6401 100644 --- a/deploy/services/helm-charts/dss/values.schema.json +++ b/deploy/services/helm-charts/dss/values.schema.json @@ -316,6 +316,11 @@ "type": "string", "description": "Go duration format (https://pkg.go.dev/time#ParseDuration). Empty to use the command default." }, + "limit": { + "type": "integer", + "minimum": 0, + "description": "Maximum number of entities deleted per run. 0 means unlimited." + }, "operationalIntents": { "type": "boolean" }, @@ -324,7 +329,7 @@ } } }, - "scd": { + "rid": { "description": "RID Related configuration", "type": "object", "properties": { @@ -343,6 +348,11 @@ "type": "string", "description": "Go duration format (https://pkg.go.dev/time#ParseDuration). Empty to use the command default." }, + "limit": { + "type": "integer", + "minimum": 0, + "description": "Maximum number of entities deleted per run. 0 means unlimited." + }, "ISAs": { "type": "boolean" }, diff --git a/deploy/services/helm-charts/dss/values.yaml b/deploy/services/helm-charts/dss/values.yaml index 605adca3b..e9f72b9ad 100644 --- a/deploy/services/helm-charts/dss/values.yaml +++ b/deploy/services/helm-charts/dss/values.yaml @@ -9,6 +9,7 @@ dss: schedule: "0 2 * * *" ttl: 2688h timeout: "" + limit: 0 operationalIntents: true subscriptions: true rid: @@ -16,6 +17,7 @@ dss: schedule: "*/30 * * * *" ttl: 30m timeout: "" + limit: 0 ISAs: true subscriptions: true diff --git a/deploy/services/tanka/evict.libsonnet b/deploy/services/tanka/evict.libsonnet index 6a888c7a7..5e4f02923 100644 --- a/deploy/services/tanka/evict.libsonnet +++ b/deploy/services/tanka/evict.libsonnet @@ -32,6 +32,7 @@ local datastoreparameters = import 'datastoreparameters.libsonnet'; locality: metadata.locality, delete: true, } + (if metadata.evict.scd.timeout != "" then { timeout: metadata.evict.scd.timeout } else {}) + + (if metadata.evict.scd.limit > 0 then { scd_limit: metadata.evict.scd.limit } else {}) + datastoreparameters.all(metadata), volumeMounts: volumes.all(metadata).backendMounts, }], @@ -69,6 +70,7 @@ local datastoreparameters = import 'datastoreparameters.libsonnet'; locality: metadata.locality, delete: true, } + (if metadata.evict.rid.timeout != "" then { timeout: metadata.evict.rid.timeout } else {}) + + (if metadata.evict.rid.limit > 0 then { rid_limit: metadata.evict.rid.limit } else {}) + datastoreparameters.all(metadata), volumeMounts: volumes.all(metadata).backendMounts, }], diff --git a/deploy/services/tanka/examples/minikube/main.jsonnet b/deploy/services/tanka/examples/minikube/main.jsonnet index 5f5217b92..f01e68c27 100644 --- a/deploy/services/tanka/examples/minikube/main.jsonnet +++ b/deploy/services/tanka/examples/minikube/main.jsonnet @@ -51,6 +51,7 @@ local metadata = metadataBase { schedule: "0 2 * * *", ttl: "2688h", timeout: "", + limit: 0, operational_intents: true, subscriptions: true, }, @@ -59,6 +60,7 @@ local metadata = metadataBase { schedule: "*/30 * * * *", ttl: "30m", timeout: "", + limit: 0, ISAs: true, subscriptions: true, }, diff --git a/deploy/services/tanka/examples/minimum/main.jsonnet b/deploy/services/tanka/examples/minimum/main.jsonnet index 5d7e93b50..29facaa2f 100644 --- a/deploy/services/tanka/examples/minimum/main.jsonnet +++ b/deploy/services/tanka/examples/minimum/main.jsonnet @@ -76,6 +76,7 @@ local metadata = metadataBase { schedule: "VAR_EVICT_SCD_SCHEDULE", ttl: "VAR_EVICT_SCD_TTL", timeout: "VAR_EVICT_SCD_TIMEOUT", + limit: 0, // <-- this integer value is VAR_EVICT_SCD_LIMIT operational_intents: true, // <-- this boolean value is VAR_EVICT_SCD_ENABLE_OPERATIONAL_INTENTS subscriptions: true, // <-- this boolean value is VAR_EVICT_SCD_ENABLE_SUBSCRIPTIONS }, @@ -84,6 +85,7 @@ local metadata = metadataBase { schedule: "VAR_EVICT_RID_SCHEDULE", ttl: "VAR_EVICT_RID_TTL", timeout: "VAR_EVICT_RID_TIMEOUT", + limit: 0, // <-- this integer value is VAR_EVICT_RID_LIMIT ISAs: true, // <-- this boolean value is VAR_EVICT_RID_ENABLE_ISAS subscriptions: true, // <-- this boolean value is VAR_EVICT_RID_ENABLE_SUBSCRIPTIONS }, diff --git a/deploy/services/tanka/metadata_base.libsonnet b/deploy/services/tanka/metadata_base.libsonnet index 7b8ec6ec9..4122b2423 100644 --- a/deploy/services/tanka/metadata_base.libsonnet +++ b/deploy/services/tanka/metadata_base.libsonnet @@ -102,6 +102,7 @@ schedule: "0 2 * * *", ttl: "2688h", timeout: "", + limit: 0, operational_intents: true, subscriptions: true, }, @@ -110,6 +111,7 @@ schedule: "*/30 * * * *", ttl: "30m", timeout: "", + limit: 0, ISAs: true, subscriptions: true, }, diff --git a/docs/operations/cleanup.md b/docs/operations/cleanup.md index c0940c41a..bba11f8f4 100644 --- a/docs/operations/cleanup.md +++ b/docs/operations/cleanup.md @@ -22,6 +22,8 @@ By default, the tool only lists expired entities. Deletion is opt-in via the `-- - Ensure a backup of the data is available. - Double-check the TTL values passed to `--rid_ttl` and `--scd_ttl`. + A preview run (without `--delete`) and a later `--delete` run are two separate invocations, not one atomic operation. Entities can expire, be created or be removed between the preview and the delete, and the delete run re-evaluates the threshold at its own execution time. With `--rid_limit`/`--scd_limit`, the delete run removes only part of the entities listed by the preview. Keep the gap between the two runs short and re-review if in doubt. + Expiration of entities is preferably determined through their end times. In the unusual event that an end time is not available, the last update time is used instead. ## Why and when to run the cleanup @@ -35,7 +37,6 @@ There is no single correct interval or TTL. Reasonable values depend on context The defaults shipped with the deployment tooling (`30m` TTL on RID running every 30 min, `2688h` ≈ 56-day TTL on SCD running nightly when enabled) are starting points, not recommendations. Validate production values against the criteria above. - ## Performance impact Each entity type (SCD operational intents, SCD subscriptions, RID ISAs, RID subscriptions) is evicted with a single statement. Entity types are deleted independently, so contention or failure on one does not affect the others. There is no risk of data inconsistency in this case - the cleanup may simply be retried. @@ -44,7 +45,7 @@ To mitigate contention: - Run the cleanup during low-intensity periods (e.g. at night). - Clean up iteratively, starting with a lower TTL and progressively increasing it. -If this becomes a recurring issue, batching removals could be considered as a future improvement. +- Use `--rid_limit`/`--scd_limit` to bound the size of each deletion if a batch still causes performance issues. ## Changes in locality @@ -67,8 +68,10 @@ Flags: -h, --help help for evict --locality string self-identification string of this DSS instance --rid_isa set this flag to true to check for expired RID ISAs (default true) + --rid_limit int maximum number of RID entities deleted, defaults to unlimited --rid_sub set this flag to true to check for expired RID subscriptions (default true) --rid_ttl duration time-to-live duration used for determining RID entries expiration, defaults to 30 minutes (default 30m0s) + --scd_limit int maximum number of SCD entities deleted, defaults to unlimited --scd_oir set this flag to true to check for expired SCD operational intents (default true) --scd_sub set this flag to true to check for expired SCD subscriptions (default true) --scd_ttl duration time-to-live duration used for determining SCD entries expiration, defaults to 2*56 days (default 2688h0m0s) @@ -89,6 +92,7 @@ Global Flags: Notes: - By default, expired entities are only listed - `--delete` is required to actually remove them. +- `--rid_limit`/`--scd_limit` can only be used together with `--delete` and bound the number of entities deleted by a run, per entity type (e.g. `--scd_limit=1000` deletes up to 1000 operational intents and up to 1000 SCD subscriptions). Which expired entities are deleted first is unspecified. Run the command again to delete the remaining ones. A run without `--delete` always lists all expired entities. - `--rid_ttl` and `--scd_ttl` accept durations formatted as [Go `time.Duration` strings](https://pkg.go.dev/time#ParseDuration), e.g. `24h`. - `--timeout` accepts the same duration format and bounds the total execution time of the command. - The datastore connection flags match those of the `core-service` command. diff --git a/test/evict/evict_helper.py b/test/evict/evict_helper.py index e4161d093..6b2c6ceae 100644 --- a/test/evict/evict_helper.py +++ b/test/evict/evict_helper.py @@ -16,6 +16,8 @@ def run_evict( rid_sub: bool = False, scd_ttl: str | None = None, rid_ttl: str | None = None, + scd_limit: int | None = None, + rid_limit: int | None = None, locality: str = "local_dev", delete: bool = False, ): @@ -58,6 +60,18 @@ def run_evict( str(rid_ttl).lower(), ] + if scd_limit is not None: + command += [ + "--scd_limit", + str(scd_limit), + ] + + if rid_limit is not None: + command += [ + "--rid_limit", + str(rid_limit), + ] + process = subprocess.run( " ".join(command), shell=True, capture_output=True, timeout=5 ) @@ -68,16 +82,32 @@ def run_evict( self.logger.error(process.stderr.decode("utf-8")) sys.exit(1) - def evict_scd_operational_intents(self, ttl: str, delete: bool): - self.run_evict(scd_oir=True, delete=delete, scd_ttl=ttl) + def evict_scd_operational_intents( + self, ttl: str, delete: bool, limit: int | None = None + ): + self.run_evict(scd_oir=True, delete=delete, scd_ttl=ttl, scd_limit=limit) - def evict_scd_subscriptions(self, ttl: str, delete: bool): - self.run_evict(scd_sub=True, delete=delete, scd_ttl=ttl) + def evict_scd_subscriptions(self, ttl: str, delete: bool, limit: int | None = None): + self.run_evict(scd_sub=True, delete=delete, scd_ttl=ttl, scd_limit=limit) - def evict_rid_ISAs(self, ttl: str, delete: bool, locality: str = "local_dev"): - self.run_evict(rid_isa=True, delete=delete, rid_ttl=ttl, locality=locality) + def evict_rid_ISAs( + self, + ttl: str, + delete: bool, + locality: str = "local_dev", + limit: int | None = None, + ): + self.run_evict( + rid_isa=True, delete=delete, rid_ttl=ttl, locality=locality, rid_limit=limit + ) def evict_rid_subscriptions( - self, ttl: str, delete: bool, locality: str = "local_dev" + self, + ttl: str, + delete: bool, + locality: str = "local_dev", + limit: int | None = None, ): - self.run_evict(rid_sub=True, delete=delete, rid_ttl=ttl, locality=locality) + self.run_evict( + rid_sub=True, delete=delete, rid_ttl=ttl, locality=locality, rid_limit=limit + ) diff --git a/test/evict/query_helper.py b/test/evict/query_helper.py index fcb94f604..d090baf76 100644 --- a/test/evict/query_helper.py +++ b/test/evict/query_helper.py @@ -104,7 +104,9 @@ def get_scd_subscription(self, id: str) -> dict[str, Any] | None: return json.loads(r.read()) - def create_scd_op_intent(self, until: datetime) -> dict[str, Any] | None: + def create_scd_op_intent( + self, until: datetime, lat: float = 22.910168434185902, lng: float = 56 + ) -> dict[str, Any] | None: r = self.do_dss_put_query( f"http://localhost:8082/dss/v1/operational_intent_references/{uuid.uuid4()}", { @@ -125,7 +127,7 @@ def create_scd_op_intent(self, until: datetime) -> dict[str, Any] | None: }, "outline_circle": { "radius": {"units": "M", "value": 100}, - "center": {"lat": 22.910168434185902, "lng": 56}, + "center": {"lat": lat, "lng": lng}, }, }, "time_start": { diff --git a/test/evict/tests/test_limit.py b/test/evict/tests/test_limit.py new file mode 100644 index 000000000..2186cec67 --- /dev/null +++ b/test/evict/tests/test_limit.py @@ -0,0 +1,102 @@ +import sys +from datetime import datetime, timedelta, UTC +import time +import logging +from typing import Any, Callable + +from evict_helper import EvictHelper +from query_helper import QueryHelper + +ENTITY_COUNT = 3 +LIMIT = 2 + + +def check_limit( + logger: logging.Logger, + name: str, + create: Callable[[datetime], dict[str, Any] | None], + get_id: Callable[[dict[str, Any]], str], + get: Callable[[str], dict[str, Any] | None], + evict: Callable[..., None], +): + logger.debug(f"Evicting all {name} older than 1s so only test entities expire") + evict("1s", delete=True) + + t = datetime.now(UTC) + timedelta(seconds=1) + + logger.debug(f"Creating {ENTITY_COUNT} test {name}") + ids: list[str] = [] + for _ in range(ENTITY_COUNT): + entity = create(t) + if not entity: + logger.error(f"❌ Unable to create {name}") + sys.exit(1) + ids.append(get_id(entity)) + + logger.debug(f"Waiting 3s so the {name} expire") + _ = sys.stdout.flush() + time.sleep(3) + + logger.debug(f"Evicting {name} older than 1s with a limit of {LIMIT}") + evict("1s", delete=True, limit=LIMIT) + + remaining = [i for i in ids if get(i)] + if len(remaining) != ENTITY_COUNT - LIMIT: + logger.error( + f"❌ Expected {ENTITY_COUNT - LIMIT} {name} to remain after evicting with a limit of {LIMIT}, got {remaining}" + ) + sys.exit(1) + + logger.debug(f"Evicting {name} older than 1s without limit") + evict("1s", delete=True, limit=0) + + remaining = [i for i in ids if get(i)] + if remaining: + logger.error(f"❌ Test {name} {remaining} shall have been deleted by evict") + sys.exit(1) + + +def test_limit(qh: QueryHelper, eh: EvictHelper): + logger = logging.getLogger("test_limit") + + logger.info("📋 Limit test") + + check_limit( + logger, + "RID ISAs", + qh.create_rid_ISA, + lambda isa: str(isa["service_area"]["id"]), + qh.get_rid_ISA, + eh.evict_rid_ISAs, + ) + + check_limit( + logger, + "RID subscriptions", + qh.create_rid_subscription, + lambda sub: str(sub["subscription"]["id"]), + qh.get_rid_subscription, + eh.evict_rid_subscriptions, + ) + + # Operational intents are spread apart so they don't conflict with each other + op_intent_lngs = iter(56 + 0.01 * i for i in range(ENTITY_COUNT)) + check_limit( + logger, + "SCD operational intents", + lambda until: qh.create_scd_op_intent(until, lng=next(op_intent_lngs)), + lambda op_intent: str(op_intent["operational_intent_reference"]["id"]), + qh.get_scd_op_intent, + eh.evict_scd_operational_intents, + ) + + check_limit( + logger, + "SCD subscriptions", + qh.create_scd_subscription, + lambda sub: str(sub["subscription"]["id"]), + qh.get_scd_subscription, + eh.evict_scd_subscriptions, + ) + + logger.info("✅ Limit test successful :)")