diff --git a/docker/.env b/docker/.env deleted file mode 100644 index a4441b26..00000000 --- a/docker/.env +++ /dev/null @@ -1 +0,0 @@ -MOUNT_PATH=../../default.txt diff --git a/docker/default_input b/docker/default_input new file mode 100644 index 00000000..e69de29b diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 203e8b15..6e045151 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -47,69 +47,30 @@ services: extends: file: "docker-compose/dev/docker-compose.monitoring.yml" service: monitoring_agent - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy - clickhouse-server: - condition: service_healthy profiles: ["dev", "dev-gpu"] monitoring_agent: extends: file: "docker-compose/prod/docker-compose.monitoring.yml" service: monitoring_agent - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy - clickhouse-server: - condition: service_healthy profiles: ["prod", "prod-gpu"] logserver-dev: extends: file: "docker-compose/dev/docker-compose.pipeline.yml" service: logserver - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy profiles: ["dev", "dev-gpu"] logserver: extends: file: "docker-compose/prod/docker-compose.pipeline.yml" service: logserver - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy profiles: ["prod", "prod-gpu"] logcollector-dev: extends: file: "docker-compose/dev/docker-compose.pipeline.yml" service: logcollector - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy profiles: ["dev", "dev-gpu"] @@ -117,65 +78,30 @@ services: extends: file: "docker-compose/prod/docker-compose.pipeline.yml" service: logcollector - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy profiles: ["prod", "prod-gpu"] prefilter-dev: extends: file: "docker-compose/dev/docker-compose.pipeline.yml" service: prefilter - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy profiles: ["dev", "dev-gpu"] prefilter: extends: file: "docker-compose/prod/docker-compose.pipeline.yml" service: prefilter - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy profiles: ["prod", "prod-gpu"] inspector-dev: extends: file: "docker-compose/dev/docker-compose.pipeline.yml" service: inspector - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy profiles: ["dev", "dev-gpu"] inspector: extends: file: "docker-compose/prod/docker-compose.pipeline.yml" service: inspector - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy profiles: ["prod", "prod-gpu"] @@ -183,82 +109,40 @@ services: extends: file: "docker-compose/dev/docker-compose.pipeline.yml" service: detector - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy profiles: ["dev"] detector-dev-gpu: extends: file: "docker-compose/dev/docker-compose.pipeline.yml" service: detector-gpu - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy profiles: ["dev-gpu"] detector: extends: file: "docker-compose/prod/docker-compose.pipeline.yml" service: detector - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy profiles: ["prod"] detector-gpu: extends: file: "docker-compose/prod/docker-compose.pipeline.yml" service: detector-gpu - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy profiles: ["prod-gpu"] alerter-dev: extends: file: "docker-compose/dev/docker-compose.pipeline.yml" service: alerter - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy profiles: ["dev", "dev-gpu"] alerter: extends: file: "docker-compose/prod/docker-compose.pipeline.yml" service: alerter - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy profiles: ["prod", "prod-gpu"] zeek: - image: ghcr.io/astraos-de/hamstring-zeek:2.0.1 + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-zeek:${HAMSTRING_ZEEK_IMAGE_TAG:-2.0.1}" cap_add: - NET_ADMIN network_mode: host diff --git a/docker/docker-compose/base/docker-compose.clickhouse.yml b/docker/docker-compose/base/docker-compose.clickhouse.yml index fda33561..4e60c21f 100644 --- a/docker/docker-compose/base/docker-compose.clickhouse.yml +++ b/docker/docker-compose/base/docker-compose.clickhouse.yml @@ -1,6 +1,6 @@ services: clickhouse-server: - image: clickhouse/clickhouse-server:26.5-alpine + image: "${CLICKHOUSE_IMAGE_REPOSITORY:-clickhouse/clickhouse-server}:${CLICKHOUSE_IMAGE_TAG:-26.5-alpine}" container_name: clickhouse-server volumes: - ../../create_tables:/docker-entrypoint-initdb.d diff --git a/docker/docker-compose/base/docker-compose.grafana.yml b/docker/docker-compose/base/docker-compose.grafana.yml index 422c00ed..92d18ce1 100644 --- a/docker/docker-compose/base/docker-compose.grafana.yml +++ b/docker/docker-compose/base/docker-compose.grafana.yml @@ -1,6 +1,6 @@ services: grafana: - image: grafana/grafana:13.0.3-slim + image: "${GRAFANA_IMAGE_REPOSITORY:-grafana/grafana}:${GRAFANA_IMAGE_TAG:-13.0.3-slim}" container_name: grafana networks: - hamstring diff --git a/docker/docker-compose/base/docker-compose.kafka-exporter.yml b/docker/docker-compose/base/docker-compose.kafka-exporter.yml index 90e093f7..9c22b07d 100644 --- a/docker/docker-compose/base/docker-compose.kafka-exporter.yml +++ b/docker/docker-compose/base/docker-compose.kafka-exporter.yml @@ -1,6 +1,6 @@ services: kafka-exporter: - image: danielqsj/kafka-exporter:latest + image: "${KAFKA_EXPORTER_IMAGE_REPOSITORY:-danielqsj/kafka-exporter}:${KAFKA_EXPORTER_IMAGE_TAG:-latest}" container_name: kafka-exporter ports: - "9308:9308" diff --git a/docker/docker-compose/base/docker-compose.kafka.yml b/docker/docker-compose/base/docker-compose.kafka.yml index 49bd65f1..01a9688d 100644 --- a/docker/docker-compose/base/docker-compose.kafka.yml +++ b/docker/docker-compose/base/docker-compose.kafka.yml @@ -16,7 +16,7 @@ x-kafka-broker-env: &kafka-broker-env services: kafka1: - image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:8.2.2}" + image: "${KAFKA_IMAGE_REPOSITORY:-confluentinc/cp-kafka}:${KAFKA_IMAGE_TAG:-8.2.2}" container_name: kafka1 restart: "unless-stopped" networks: @@ -37,7 +37,7 @@ services: - kafka-broker-data1:/var/lib/kafka/data kafka2: - image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:8.2.2}" + image: "${KAFKA_IMAGE_REPOSITORY:-confluentinc/cp-kafka}:${KAFKA_IMAGE_TAG:-8.2.2}" container_name: kafka2 restart: "unless-stopped" networks: @@ -59,7 +59,7 @@ services: - kafka-broker-data2:/var/lib/kafka/data kafka3: - image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:8.2.2}" + image: "${KAFKA_IMAGE_REPOSITORY:-confluentinc/cp-kafka}:${KAFKA_IMAGE_TAG:-8.2.2}" container_name: kafka3 restart: "unless-stopped" networks: diff --git a/docker/docker-compose/base/docker-compose.prometheus.yml b/docker/docker-compose/base/docker-compose.prometheus.yml index 703c547a..8a2c4a2b 100644 --- a/docker/docker-compose/base/docker-compose.prometheus.yml +++ b/docker/docker-compose/base/docker-compose.prometheus.yml @@ -1,6 +1,6 @@ services: prometheus: - image: prom/prometheus + image: "${PROMETHEUS_IMAGE_REPOSITORY:-prom/prometheus}:${PROMETHEUS_IMAGE_TAG:-latest}" container_name: prometheus ports: - "9088:9090" diff --git a/docker/docker-compose/dev/docker-compose.monitoring.yml b/docker/docker-compose/dev/docker-compose.monitoring.yml index 103e433f..bbd6f5b8 100644 --- a/docker/docker-compose/dev/docker-compose.monitoring.yml +++ b/docker/docker-compose/dev/docker-compose.monitoring.yml @@ -6,20 +6,10 @@ services: restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy - clickhouse-server: - condition: service_healthy - grafana: - condition: service_healthy networks: - hamstring environment: - GROUP_ID=monitoring_agent + - HAMSTRING_WAIT_FOR=kafka,clickhouse - CLICKHOUSE_USER=${CLICKHOUSE_USER:-default} - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-hamstring} diff --git a/docker/docker-compose/dev/docker-compose.pipeline.yml b/docker/docker-compose/dev/docker-compose.pipeline.yml index 946789ce..16b646f8 100644 --- a/docker/docker-compose/dev/docker-compose.pipeline.yml +++ b/docker/docker-compose/dev/docker-compose.pipeline.yml @@ -4,10 +4,10 @@ services: context: ../../.. dockerfile: docker/dockerfiles/Dockerfile.logserver volumes: - - "${MOUNT_PATH:?MOUNT_PATH not set}:/opt/file.txt" - ../../../config.yaml:/app/config.yaml environment: - GROUP_ID=log_storage + - HAMSTRING_WAIT_FOR=kafka deploy: mode: "replicated" replicas: 1 @@ -28,6 +28,7 @@ services: hamstring: environment: - GROUP_ID=log_collection + - HAMSTRING_WAIT_FOR=kafka prefilter: build: context: ../../.. @@ -42,6 +43,7 @@ services: replicas: 1 environment: - GROUP_ID=log_filtering + - HAMSTRING_WAIT_FOR=kafka inspector: build: context: ../../.. @@ -65,6 +67,7 @@ services: environment: - GROUP_ID=data_inspection - NUMBER_OF_INSTANCES=1 + - HAMSTRING_WAIT_FOR=kafka detector: build: context: ../../.. @@ -79,6 +82,7 @@ services: replicas: 1 environment: - GROUP_ID=data_analysis + - HAMSTRING_WAIT_FOR=kafka detector-gpu: build: @@ -100,6 +104,7 @@ services: capabilities: [gpu] environment: - GROUP_ID=data_analysis + - HAMSTRING_WAIT_FOR=kafka - NVIDIA_VISIBLE_DEVICES=all - NVIDIA_DRIVER_CAPABILITIES=compute,utility @@ -110,7 +115,7 @@ services: restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml - - ../../../logs:/opt/logs + - "${ALERTER_LOGS_PATH:-../../../logs}:/opt/logs" networks: hamstring: deploy: @@ -118,3 +123,4 @@ services: replicas: 1 environment: - GROUP_ID=data_alerting + - HAMSTRING_WAIT_FOR=kafka diff --git a/docker/docker-compose/dev/docker-compose.yml b/docker/docker-compose/dev/docker-compose.yml index 56e6badd..8b470d37 100644 --- a/docker/docker-compose/dev/docker-compose.yml +++ b/docker/docker-compose/dev/docker-compose.yml @@ -1,100 +1,65 @@ services: kafka1: extends: - file: "docker-compose/base/docker-compose.kafka.yml" + file: "../base/docker-compose.kafka.yml" service: kafka1 kafka2: extends: - file: "docker-compose/base/docker-compose.kafka.yml" + file: "../base/docker-compose.kafka.yml" service: kafka2 kafka3: extends: - file: "docker-compose/base/docker-compose.kafka.yml" + file: "../base/docker-compose.kafka.yml" service: kafka3 logserver: extends: - file: "docker-compose/base/docker-compose.pipeline.yml" + file: "docker-compose.pipeline.yml" service: logserver - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy logcollector: extends: - file: "docker-compose/base/docker-compose.pipeline.yml" + file: "docker-compose.pipeline.yml" service: logcollector - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy prefilter: extends: - file: "docker-compose/base/docker-compose.pipeline.yml" + file: "docker-compose.pipeline.yml" service: prefilter - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy inspector: extends: - file: "docker-compose/base/docker-compose.pipeline.yml" + file: "docker-compose.pipeline.yml" service: inspector - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy detector: extends: - file: "docker-compose/base/docker-compose.pipeline.yml" + file: "docker-compose.pipeline.yml" service: detector - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy clickhouse-server: extends: - file: "docker-compose/prod/docker-compose.clickhouse.yml" + file: "../base/docker-compose.clickhouse.yml" service: clickhouse-server grafana: extends: - file: "docker-compose/prod/docker-compose.grafana.yml" + file: "../base/docker-compose.grafana.yml" service: grafana prometheus: extends: - file: "docker-compose/prod/docker-compose.prometheus.yml" + file: "../base/docker-compose.prometheus.yml" service: prometheus kafka-exporter: extends: - file: "docker-compose/base/docker-compose.kafka-exporter.yml" + file: "../base/docker-compose.kafka-exporter.yml" service: kafka-exporter depends_on: kafka1: @@ -107,17 +72,8 @@ services: condition: service_healthy monitoring_agent: extends: - file: "docker-compose/base/docker-compose.monitoring.yml" + file: "docker-compose.monitoring.yml" service: monitoring_agent - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy - clickhouse-server: - condition: service_healthy networks: hamstring: diff --git a/docker/docker-compose/prod/docker-compose.monitoring.yml b/docker/docker-compose/prod/docker-compose.monitoring.yml index 77c46167..37178aa5 100644 --- a/docker/docker-compose/prod/docker-compose.monitoring.yml +++ b/docker/docker-compose/prod/docker-compose.monitoring.yml @@ -1,23 +1,13 @@ services: monitoring_agent: - image: ghcr.io/astraos-de/hamstring-monitoring:v2.2.0 + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-monitoring:${HAMSTRING_MONITORING_IMAGE_TAG:-v2.2.0}" restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy - clickhouse-server: - condition: service_healthy - grafana: - condition: service_healthy networks: - hamstring environment: - GROUP_ID=monitoring_agent + - HAMSTRING_WAIT_FOR=kafka,clickhouse - CLICKHOUSE_USER=${CLICKHOUSE_USER:-default} - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-hamstring} diff --git a/docker/docker-compose/prod/docker-compose.pipeline.yml b/docker/docker-compose/prod/docker-compose.pipeline.yml index 60fa083f..33c82d46 100644 --- a/docker/docker-compose/prod/docker-compose.pipeline.yml +++ b/docker/docker-compose/prod/docker-compose.pipeline.yml @@ -1,16 +1,16 @@ services: logserver: - image: ghcr.io/astraos-de/hamstring-logserver:v2.2.0 + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-logserver:${HAMSTRING_LOGSERVER_IMAGE_TAG:-v2.2.0}" restart: "unless-stopped" networks: hamstring: volumes: - - "${MOUNT_PATH:?MOUNT_PATH not set}:/opt/file.txt" - ../../../config.yaml:/app/config.yaml environment: - GROUP_ID=log_storage + - HAMSTRING_WAIT_FOR=kafka logcollector: - image: ghcr.io/astraos-de/hamstring-logcollector:v2.2.0 + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-logcollector:${HAMSTRING_LOGCOLLECTOR_IMAGE_TAG:-v2.2.0}" restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml @@ -18,9 +18,10 @@ services: hamstring: environment: - GROUP_ID=log_collection + - HAMSTRING_WAIT_FOR=kafka prefilter: - image: ghcr.io/astraos-de/hamstring-prefilter:v2.2.0 + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-prefilter:${HAMSTRING_PREFILTER_IMAGE_TAG:-v2.2.0}" restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml @@ -31,9 +32,10 @@ services: replicas: 1 environment: - GROUP_ID=log_filtering + - HAMSTRING_WAIT_FOR=kafka inspector: - image: ghcr.io/astraos-de/hamstring-inspector:v2.2.0 + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-inspector:${HAMSTRING_INSPECTOR_IMAGE_TAG:-v2.2.0}" restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml @@ -52,9 +54,10 @@ services: environment: - GROUP_ID=data_inspection - NUMBER_OF_INSTANCES=1 + - HAMSTRING_WAIT_FOR=kafka detector: - image: ghcr.io/astraos-de/hamstring-detector:v2.2.0 + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-detector:${HAMSTRING_DETECTOR_IMAGE_TAG:-v2.2.0}" restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml @@ -65,9 +68,10 @@ services: replicas: 1 environment: - GROUP_ID=data_analysis + - HAMSTRING_WAIT_FOR=kafka detector-gpu: - image: ghcr.io/astraos-de/hamstring-detector:v2.2.0 + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-detector:${HAMSTRING_DETECTOR_IMAGE_TAG:-v2.2.0}" restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml @@ -84,15 +88,16 @@ services: capabilities: [gpu] environment: - GROUP_ID=data_analysis + - HAMSTRING_WAIT_FOR=kafka - NVIDIA_VISIBLE_DEVICES=all - NVIDIA_DRIVER_CAPABILITIES=compute,utility alerter: - image: ghcr.io/astraos-de/hamstring-alerter:v2.2.0 + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-alerter:${HAMSTRING_ALERTER_IMAGE_TAG:-v2.2.0}" restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml - - /opt/logs:/opt/logs + - "${ALERTER_LOGS_PATH:-/opt/logs}:/opt/logs" networks: hamstring: deploy: @@ -100,3 +105,4 @@ services: replicas: 1 environment: - GROUP_ID=data_alerting + - HAMSTRING_WAIT_FOR=kafka diff --git a/docker/docker-compose/prod/docker-compose.yml b/docker/docker-compose/prod/docker-compose.yml index 879101eb..8b470d37 100644 --- a/docker/docker-compose/prod/docker-compose.yml +++ b/docker/docker-compose/prod/docker-compose.yml @@ -1,100 +1,65 @@ services: kafka1: extends: - file: "docker-compose/base/docker-compose.kafka.yml" + file: "../base/docker-compose.kafka.yml" service: kafka1 kafka2: extends: - file: "docker-compose/base/docker-compose.kafka.yml" + file: "../base/docker-compose.kafka.yml" service: kafka2 kafka3: extends: - file: "docker-compose/base/docker-compose.kafka.yml" + file: "../base/docker-compose.kafka.yml" service: kafka3 logserver: extends: - file: "docker-compose/prod/docker-compose.pipeline.yml" + file: "docker-compose.pipeline.yml" service: logserver - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy logcollector: extends: - file: "docker-compose/prod/docker-compose.pipeline.yml" + file: "docker-compose.pipeline.yml" service: logcollector - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy prefilter: extends: - file: "docker-compose/prod/docker-compose.pipeline.yml" + file: "docker-compose.pipeline.yml" service: prefilter - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy inspector: extends: - file: "docker-compose/prod/docker-compose.pipeline.yml" + file: "docker-compose.pipeline.yml" service: inspector - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy detector: extends: - file: "docker-compose/prod/docker-compose.pipeline.yml" + file: "docker-compose.pipeline.yml" service: detector - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy clickhouse-server: extends: - file: "docker-compose/prod/docker-compose.clickhouse.yml" + file: "../base/docker-compose.clickhouse.yml" service: clickhouse-server grafana: extends: - file: "docker-compose/prod/docker-compose.grafana.yml" + file: "../base/docker-compose.grafana.yml" service: grafana prometheus: extends: - file: "docker-compose/prod/docker-compose.prometheus.yml" + file: "../base/docker-compose.prometheus.yml" service: prometheus kafka-exporter: extends: - file: "docker-compose/base/docker-compose.kafka-exporter.yml" + file: "../base/docker-compose.kafka-exporter.yml" service: kafka-exporter depends_on: kafka1: @@ -107,17 +72,8 @@ services: condition: service_healthy monitoring_agent: extends: - file: "docker-compose/prod/docker-compose.monitoring.yml" + file: "docker-compose.monitoring.yml" service: monitoring_agent - depends_on: - kafka1: - condition: service_healthy - kafka2: - condition: service_healthy - kafka3: - condition: service_healthy - clickhouse-server: - condition: service_healthy networks: hamstring: diff --git a/docker/docker_swarm/docker-compose.swarm.yml b/docker/docker_swarm/docker-compose.swarm.yml index dd354518..682e5fb8 100644 --- a/docker/docker_swarm/docker-compose.swarm.yml +++ b/docker/docker_swarm/docker-compose.swarm.yml @@ -16,9 +16,19 @@ x-kafka-broker-env: &kafka-broker-env KAFKA_LOG_SEGMENT_BYTES: "${KAFKA_LOG_SEGMENT_BYTES:-1073741824}" KAFKA_LOG_CLEANUP_POLICY: "${KAFKA_LOG_CLEANUP_POLICY:-delete}" +x-hamstring-kafka-wait-env: &hamstring-kafka-wait-env + HAMSTRING_WAIT_FOR: kafka + HAMSTRING_WAIT_INITIAL_DELAY_SECONDS: "${HAMSTRING_WAIT_INITIAL_DELAY_SECONDS:-30}" + HAMSTRING_WAIT_TIMEOUT_SECONDS: "${HAMSTRING_WAIT_TIMEOUT_SECONDS:-180}" + +x-hamstring-kafka-clickhouse-wait-env: &hamstring-kafka-clickhouse-wait-env + HAMSTRING_WAIT_FOR: kafka,clickhouse + HAMSTRING_WAIT_INITIAL_DELAY_SECONDS: "${HAMSTRING_WAIT_INITIAL_DELAY_SECONDS:-30}" + HAMSTRING_WAIT_TIMEOUT_SECONDS: "${HAMSTRING_WAIT_TIMEOUT_SECONDS:-180}" + services: kafka1: - image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:8.2.2}" + image: "${KAFKA_IMAGE_REPOSITORY:-confluentinc/cp-kafka}:${KAFKA_IMAGE_TAG:-8.2.2}" networks: [hamstring] ports: - target: 8097 @@ -33,11 +43,14 @@ services: volumes: - kafka-broker-data1:/var/lib/kafka/data deploy: + placement: + constraints: + - "${KAFKA1_PLACEMENT_CONSTRAINT:-node.platform.os == linux}" restart_policy: condition: any kafka2: - image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:8.2.2}" + image: "${KAFKA_IMAGE_REPOSITORY:-confluentinc/cp-kafka}:${KAFKA_IMAGE_TAG:-8.2.2}" networks: [hamstring] ports: - target: 8098 @@ -53,11 +66,14 @@ services: volumes: - kafka-broker-data2:/var/lib/kafka/data deploy: + placement: + constraints: + - "${KAFKA2_PLACEMENT_CONSTRAINT:-node.platform.os == linux}" restart_policy: condition: any kafka3: - image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:8.2.2}" + image: "${KAFKA_IMAGE_REPOSITORY:-confluentinc/cp-kafka}:${KAFKA_IMAGE_TAG:-8.2.2}" networks: [hamstring] ports: - target: 8099 @@ -73,11 +89,14 @@ services: volumes: - kafka-broker-data3:/var/lib/kafka/data deploy: + placement: + constraints: + - "${KAFKA3_PLACEMENT_CONSTRAINT:-node.platform.os == linux}" restart_policy: condition: any clickhouse-server: - image: "${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.5-alpine}" + image: "${CLICKHOUSE_IMAGE_REPOSITORY:-clickhouse/clickhouse-server}:${CLICKHOUSE_IMAGE_TAG:-26.5-alpine}" networks: - hamstring environment: @@ -102,11 +121,14 @@ services: timeout: 5s retries: 3 deploy: + placement: + constraints: + - "${CLICKHOUSE_PLACEMENT_CONSTRAINT:-node.platform.os == linux}" restart_policy: condition: any grafana: - image: "${GRAFANA_IMAGE:-grafana/grafana:13.0.3-slim}" + image: "${GRAFANA_IMAGE_REPOSITORY:-grafana/grafana}:${GRAFANA_IMAGE_TAG:-13.0.3-slim}" networks: - hamstring ports: @@ -131,11 +153,14 @@ services: start_period: 120s retries: 3 deploy: + placement: + constraints: + - "${GRAFANA_PLACEMENT_CONSTRAINT:-node.platform.os == linux}" restart_policy: condition: any prometheus: - image: "${PROMETHEUS_IMAGE:-prom/prometheus}" + image: "${PROMETHEUS_IMAGE_REPOSITORY:-prom/prometheus}:${PROMETHEUS_IMAGE_TAG:-latest}" networks: - hamstring ports: @@ -143,14 +168,19 @@ services: published: "${PROMETHEUS_PORT:-9088}" protocol: tcp mode: ingress - volumes: - - "${HAMSTRING_ROOT:-../..}/docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml" + configs: + - source: prometheus_config + target: /etc/prometheus/prometheus.yml + mode: 0444 deploy: + placement: + constraints: + - "${PROMETHEUS_PLACEMENT_CONSTRAINT:-node.platform.os == linux}" restart_policy: condition: any kafka-exporter: - image: "${KAFKA_EXPORTER_IMAGE:-danielqsj/kafka-exporter}" + image: "${KAFKA_EXPORTER_IMAGE_REPOSITORY:-danielqsj/kafka-exporter}:${KAFKA_EXPORTER_IMAGE_TAG:-latest}" command: - --kafka.server=kafka1:19092 - --kafka.server=kafka2:19093 @@ -163,78 +193,98 @@ services: protocol: tcp mode: ingress deploy: + placement: + constraints: + - "${KAFKA_EXPORTER_PLACEMENT_CONSTRAINT:-node.platform.os == linux}" restart_policy: condition: any monitoring_agent: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-monitoring:${HAMSTRING_IMAGE_TAG:-v2.2.0}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-monitoring:${HAMSTRING_MONITORING_IMAGE_TAG:-v2.2.0}" command: ["src/monitoring/monitoring_agent.py"] networks: - hamstring volumes: - "${HAMSTRING_ROOT:-../..}/config.yaml:/app/config.yaml" environment: + <<: *hamstring-kafka-clickhouse-wait-env GROUP_ID: "${MONITORING_AGENT_GROUP_ID:-monitoring_agent}" HAMSTRING_CONFIG_CHECKSUM: "${HAMSTRING_CONFIG_CHECKSUM:-d9bc812b76c2b58c633b412a6c07f981f3c4d787}" CLICKHOUSE_USER: "${CLICKHOUSE_USER:-default}" CLICKHOUSE_PASSWORD: "${CLICKHOUSE_PASSWORD:-hamstring}" deploy: + placement: + constraints: + - "${MONITORING_AGENT_PLACEMENT_CONSTRAINT:-node.platform.os == linux}" restart_policy: condition: any logserver: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-logserver:${HAMSTRING_IMAGE_TAG:-v2.2.0}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-logserver:${HAMSTRING_LOGSERVER_IMAGE_TAG:-v2.2.0}" command: ["src/logserver/server.py"] networks: - hamstring volumes: - - "${HAMSTRING_ROOT:-../..}/docker/default.txt:/opt/file.txt" + - "${LOGSERVER_INPUT_PATH:-../../docker/default_input}:/opt/file.txt:ro" - "${HAMSTRING_ROOT:-../..}/config.yaml:/app/config.yaml" environment: + <<: *hamstring-kafka-wait-env GROUP_ID: "${LOGSERVER_GROUP_ID:-log_storage}" HAMSTRING_CONFIG_CHECKSUM: "${HAMSTRING_CONFIG_CHECKSUM:-d9bc812b76c2b58c633b412a6c07f981f3c4d787}" deploy: + placement: + constraints: + - "${LOGSERVER_PLACEMENT_CONSTRAINT:-node.platform.os == linux}" restart_policy: condition: any logcollector: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-logcollector:${HAMSTRING_IMAGE_TAG:-v2.2.0}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-logcollector:${HAMSTRING_LOGCOLLECTOR_IMAGE_TAG:-v2.2.0}" command: ["src/logcollector/collector.py"] networks: - hamstring volumes: - "${HAMSTRING_ROOT:-../..}/config.yaml:/app/config.yaml" environment: + <<: *hamstring-kafka-wait-env GROUP_ID: "${LOGCOLLECTOR_GROUP_ID:-log_collection}" HAMSTRING_CONFIG_CHECKSUM: "${HAMSTRING_CONFIG_CHECKSUM:-d9bc812b76c2b58c633b412a6c07f981f3c4d787}" deploy: + placement: + constraints: + - "${LOGCOLLECTOR_PLACEMENT_CONSTRAINT:-node.platform.os == linux}" restart_policy: condition: any prefilter: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-prefilter:${HAMSTRING_IMAGE_TAG:-v2.2.0}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-prefilter:${HAMSTRING_PREFILTER_IMAGE_TAG:-v2.2.0}" command: ["src/prefilter/prefilter.py"] networks: - hamstring volumes: - "${HAMSTRING_ROOT:-../..}/config.yaml:/app/config.yaml" environment: + <<: *hamstring-kafka-wait-env GROUP_ID: "${PREFILTER_GROUP_ID:-log_filtering}" HAMSTRING_CONFIG_CHECKSUM: "${HAMSTRING_CONFIG_CHECKSUM:-d9bc812b76c2b58c633b412a6c07f981f3c4d787}" deploy: mode: replicated replicas: ${PREFILTER_REPLICAS:-1} + placement: + constraints: + - "${PREFILTER_PLACEMENT_CONSTRAINT:-node.platform.os == linux}" restart_policy: condition: any inspector: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-inspector:${HAMSTRING_IMAGE_TAG:-v2.2.0}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-inspector:${HAMSTRING_INSPECTOR_IMAGE_TAG:-v2.2.0}" command: ["src/inspector/inspector.py"] networks: - hamstring volumes: - "${HAMSTRING_ROOT:-../..}/config.yaml:/app/config.yaml" environment: + <<: *hamstring-kafka-wait-env GROUP_ID: "${INSPECTOR_GROUP_ID:-data_inspection}" NUMBER_OF_INSTANCES: "${INSPECTOR_NUMBER_OF_INSTANCES:-1}" HAMSTRING_CONFIG_CHECKSUM: "${HAMSTRING_CONFIG_CHECKSUM:-d9bc812b76c2b58c633b412a6c07f981f3c4d787}" @@ -248,17 +298,21 @@ services: reservations: cpus: "${INSPECTOR_CPU_RESERVATION:-1}" memory: "${INSPECTOR_MEMORY_RESERVATION:-256M}" + placement: + constraints: + - "${INSPECTOR_PLACEMENT_CONSTRAINT:-node.platform.os == linux}" restart_policy: condition: any detector: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-detector:${HAMSTRING_IMAGE_TAG:-v2.2.0}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-detector:${HAMSTRING_DETECTOR_IMAGE_TAG:-v2.2.0}" command: ["src/detector/detector.py"] networks: - hamstring volumes: - "${HAMSTRING_ROOT:-../..}/config.yaml:/app/config.yaml" environment: + <<: *hamstring-kafka-wait-env GROUP_ID: "${DETECTOR_GROUP_ID:-data_analysis}" NVIDIA_DRIVER_CAPABILITIES: "${NVIDIA_DRIVER_CAPABILITIES:-compute,utility}" NVIDIA_VISIBLE_DEVICES: "${NVIDIA_VISIBLE_DEVICES:-all}" @@ -273,28 +327,35 @@ services: # - discrete_resource_spec: # kind: NVIDIA-GPU # value: 1 + placement: + constraints: + - "${DETECTOR_PLACEMENT_CONSTRAINT:-node.platform.os == linux}" restart_policy: condition: any alerter: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-alerter:${HAMSTRING_IMAGE_TAG:-v2.2.0}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-alerter:${HAMSTRING_ALERTER_IMAGE_TAG:-v2.2.0}" command: ["src/alerter/alerter.py"] networks: - hamstring volumes: - "${HAMSTRING_ROOT:-../..}/config.yaml:/app/config.yaml" - - "${HAMSTRING_ROOT:-../..}/docker/logs:/opt/logs" + - alerter_logs:/opt/logs environment: + <<: *hamstring-kafka-wait-env GROUP_ID: "${ALERTER_GROUP_ID:-data_alerting}" HAMSTRING_CONFIG_CHECKSUM: "${HAMSTRING_CONFIG_CHECKSUM:-d9bc812b76c2b58c633b412a6c07f981f3c4d787}" deploy: mode: replicated replicas: ${ALERTER_REPLICAS:-1} + placement: + constraints: + - "${ALERTER_PLACEMENT_CONSTRAINT:-node.platform.os == linux}" restart_policy: condition: any zeek: - image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-zeek:${HAMSTRING_IMAGE_TAG:-2.0.1}" + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-zeek:${HAMSTRING_ZEEK_IMAGE_TAG:-2.0.1}" cap_add: - NET_ADMIN networks: @@ -309,6 +370,9 @@ services: deploy: mode: replicated replicas: ${ZEEK_REPLICAS:-1} + placement: + constraints: + - "${ZEEK_PLACEMENT_CONSTRAINT:-node.platform.os == linux}" restart_policy: condition: any @@ -321,10 +385,15 @@ networks: attachable: true volumes: + alerter_logs: ch_data: ch_logs: kafka-broker-data1: kafka-broker-data2: kafka-broker-data3: +configs: + prometheus_config: + file: "${PROMETHEUS_CONFIG_FILE:-../../docker/prometheus/prometheus.yml}" + # code: language=yaml diff --git a/docker/dockerfiles/Dockerfile.alerter b/docker/dockerfiles/Dockerfile.alerter index 891aac85..f79038c6 100644 --- a/docker/dockerfiles/Dockerfile.alerter +++ b/docker/dockerfiles/Dockerfile.alerter @@ -19,4 +19,5 @@ COPY --from=build-venv /venv /venv COPY src/base /app/src/base COPY src/alerter /app/src/alerter WORKDIR /app -ENTRYPOINT ["/venv/bin/python3", "src/alerter/alerter.py"] +ENTRYPOINT ["/venv/bin/python3", "-m", "src.base.service_entrypoint"] +CMD ["src/alerter/alerter.py"] diff --git a/docker/dockerfiles/Dockerfile.detector b/docker/dockerfiles/Dockerfile.detector index 3b3cef9d..eab7014d 100644 --- a/docker/dockerfiles/Dockerfile.detector +++ b/docker/dockerfiles/Dockerfile.detector @@ -19,4 +19,5 @@ COPY --from=build-venv /venv /venv COPY src/base /app/src/base COPY src/detector /app/src/detector WORKDIR /app -ENTRYPOINT ["/venv/bin/python3", "src/detector/detector.py"] +ENTRYPOINT ["/venv/bin/python3", "-m", "src.base.service_entrypoint"] +CMD ["src/detector/detector.py"] diff --git a/docker/dockerfiles/Dockerfile.inspector b/docker/dockerfiles/Dockerfile.inspector index 6fc5a48f..7bf99f55 100644 --- a/docker/dockerfiles/Dockerfile.inspector +++ b/docker/dockerfiles/Dockerfile.inspector @@ -19,4 +19,5 @@ COPY --from=build-venv /venv /venv COPY src/base /app/src/base COPY src/inspector /app/src/inspector WORKDIR /app -ENTRYPOINT ["/venv/bin/python3", "src/inspector/inspector.py"] +ENTRYPOINT ["/venv/bin/python3", "-m", "src.base.service_entrypoint"] +CMD ["src/inspector/inspector.py"] diff --git a/docker/dockerfiles/Dockerfile.logcollector b/docker/dockerfiles/Dockerfile.logcollector index ff2c9b89..e9a56964 100644 --- a/docker/dockerfiles/Dockerfile.logcollector +++ b/docker/dockerfiles/Dockerfile.logcollector @@ -19,4 +19,5 @@ COPY --from=build-venv /venv /venv COPY src/base /app/src/base COPY src/logcollector /app/src/logcollector WORKDIR /app -ENTRYPOINT ["/venv/bin/python3", "src/logcollector/collector.py"] +ENTRYPOINT ["/venv/bin/python3", "-m", "src.base.service_entrypoint"] +CMD ["src/logcollector/collector.py"] diff --git a/docker/dockerfiles/Dockerfile.logserver b/docker/dockerfiles/Dockerfile.logserver index cb2ac556..dd356761 100644 --- a/docker/dockerfiles/Dockerfile.logserver +++ b/docker/dockerfiles/Dockerfile.logserver @@ -19,4 +19,5 @@ COPY --from=build-venv /venv /venv COPY src/base /app/src/base COPY src/logserver /app/src/logserver WORKDIR /app -ENTRYPOINT ["/venv/bin/python3", "src/logserver/server.py"] +ENTRYPOINT ["/venv/bin/python3", "-m", "src.base.service_entrypoint"] +CMD ["src/logserver/server.py"] diff --git a/docker/dockerfiles/Dockerfile.monitoring b/docker/dockerfiles/Dockerfile.monitoring index 402505d3..33d50cfb 100644 --- a/docker/dockerfiles/Dockerfile.monitoring +++ b/docker/dockerfiles/Dockerfile.monitoring @@ -20,4 +20,5 @@ COPY src/base /app/src/base COPY src/monitoring /app/src/monitoring COPY config.yaml /app WORKDIR /app -ENTRYPOINT ["/venv/bin/python3", "src/monitoring/monitoring_agent.py"] +ENTRYPOINT ["/venv/bin/python3", "-m", "src.base.service_entrypoint"] +CMD ["src/monitoring/monitoring_agent.py"] diff --git a/docker/dockerfiles/Dockerfile.prefilter b/docker/dockerfiles/Dockerfile.prefilter index cb279ee4..304bea74 100644 --- a/docker/dockerfiles/Dockerfile.prefilter +++ b/docker/dockerfiles/Dockerfile.prefilter @@ -19,4 +19,5 @@ COPY --from=build-venv /venv /venv COPY src/base /app/src/base COPY src/prefilter /app/src/prefilter WORKDIR /app -ENTRYPOINT ["/venv/bin/python3", "src/prefilter/prefilter.py"] +ENTRYPOINT ["/venv/bin/python3", "-m", "src.base.service_entrypoint"] +CMD ["src/prefilter/prefilter.py"] diff --git a/docs/configuration.rst b/docs/configuration.rst index 6d7af4d6..8805f126 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -116,6 +116,67 @@ Pipeline Configuration The following parameters control the behavior of each stage of the HAMSTRING pipeline, including the functionality of the modules. +``pipeline.acceleration`` +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Controls optional hardware acceleration for modules that support accelerated execution. Values from +``default`` are used unless a configured inspector or detector overrides them with its own +``acceleration`` block. + +.. list-table:: Acceleration options + :header-rows: 1 + :widths: 30 20 50 + + * - Parameter + - Default Value + - Description + * - ``enabled`` + - ``true`` + - Enables accelerated execution where a module supports it. + * - ``fallback_to_cpu`` + - ``true`` + - Allows a module to continue on CPU if the requested accelerator is unavailable. + * - ``log_device`` + - ``true`` + - Logs the selected acceleration device during startup. + * - ``default.device`` + - ``auto`` + - Default device selection. Use ``auto`` for automatic detection, or a module-supported device name. + * - ``default.backend`` + - ``auto`` + - Default acceleration backend. Use ``auto`` for module-specific automatic backend selection. + * - ``default.batch_size`` + - ``auto`` + - Default accelerated inference batch size. + +``pipeline.resilience`` +^^^^^^^^^^^^^^^^^^^^^^^ + +Controls retry behavior around transient startup and infrastructure errors. + +.. list-table:: Retry options + :header-rows: 1 + :widths: 30 20 50 + + * - Parameter + - Default Value + - Description + * - ``retry.initial_delay_seconds`` + - ``1.0`` + - Delay before the first retry. + * - ``retry.max_delay_seconds`` + - ``30.0`` + - Maximum delay between retries. + * - ``retry.backoff_multiplier`` + - ``2.0`` + - Multiplier used for exponential backoff. + * - ``retry.jitter_seconds`` + - ``0.25`` + - Random jitter added to retry delays. + * - ``retry.log_every_attempts`` + - ``5`` + - Log every nth retry attempt while a dependency is still unavailable. + ``pipeline.scaling`` ^^^^^^^^^^^^^^^^^^^^ @@ -330,7 +391,8 @@ With this example and the hybrid detector config shown above, the detector start - ``"/opt/file.txt"`` - Path of the input file, to which data is appended during usage. - Keep this setting unchanged when using Docker; modify the ``MOUNT_PATH`` in ``docker/.env`` instead. + Keep this setting unchanged when using Docker. Mount a different host file with + ``LOGSERVER_INPUT_PATH`` instead. ``pipeline.log_collection`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -466,6 +528,9 @@ To entirely skip the anomaly detection phase, you can set ``inspector_module_nam * - send_to_alerter - ``true`` - Set to ``false`` to disable the detector-to-alerter Kafka output while still allowing detector-to-detector forwarding. + * - acceleration + - See ``pipeline.acceleration`` + - Optional per-detector acceleration override. ``pipeline.alerting`` @@ -487,6 +552,12 @@ To entirely skip the anomaly detection phase, you can set ``inspector_module_nam * - log_file_path - ``"/opt/logs/alerts.txt"`` - Local file path where alerts will be appended if ``log_to_file`` is enabled. + * - log_rotation.enabled + - ``true`` + - Enables rotation for the alert log file. + * - log_rotation.retention_days + - ``7`` + - Number of days to retain rotated alert log files. * - external_kafka_topic - ``"hamstring_alerts"`` - Name of the external Kafka topic where alerts will be sent if ``log_to_kafka`` is enabled. @@ -494,6 +565,23 @@ To entirely skip the anomaly detection phase, you can set ``inspector_module_nam - ``[]`` - List of custom alerter plugins to execute. Each plugin must specify ``name``, ``alerter_module_name``, and ``alerter_class_name``. +``pipeline.monitoring`` +^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: ``monitoring`` Parameters + :header-rows: 1 + :widths: 30 20 50 + + * - Parameter + - Default Value + - Description + * - clickhouse_connector.batch_size + - ``50`` + - Number of monitoring rows written to ClickHouse in one batch. + * - clickhouse_connector.batch_timeout + - ``2.0`` + - Maximum time in seconds before a partial monitoring batch is written. + ``pipeline.zeek`` ^^^^^^^^^^^^^^^^^ @@ -530,8 +618,9 @@ The following parameters control the infrastructure of the software. - Default Value - Description * - kafka_brokers - - ``hostname: kafka1, port: 8097, node_ip: 0.0.0.0``, ``hostname: kafka2, port: 8098, node_ip: 0.0.0.0``, ``hostname: kafka3, port: 8099, node_ip: 0.0.0.0`` - - Hostnames and ports of the Kafka brokers, given as list. The node ip is crucial and needs to be set to the actual IP of the system where the Kafka broker will be running on. + - ``hostname: kafka1, internal_port: 19092, external_port: 8097, node_ip: 127.0.0.1``, ``hostname: kafka2, internal_port: 19093, external_port: 8098, node_ip: 127.0.0.1``, ``hostname: kafka3, internal_port: 19094, external_port: 8099, node_ip: 127.0.0.1`` + - Kafka broker endpoints. Containers use ``hostname`` and ``internal_port`` on the Docker network; + host-network clients use ``node_ip`` and ``external_port``. * - kafka_topics_prefix - Not given here - Kafka topic name prefixes given as strings. These prefix name are used to construct the actual topic names based on the instance name (e.g. a collector instance name) that produces for the given stage. @@ -554,3 +643,384 @@ The following parameters control the infrastructure of the software. * - monitoring.clickhouse_server.hostname - ``clickhouse-server`` - Hostname of the ClickHouse server. Used by Grafana. + * - monitoring.clickhouse_server.http_port + - ``8123`` + - Optional HTTP port used by the container entrypoint when waiting for ClickHouse. If omitted, + HAMSTRING uses ``8123``. + +Deployment Environment Variables +................................ + +The Docker Compose and Docker Swarm files expose the following environment variables. Values shown here +are the defaults used when no override is provided. + +Service readiness +^^^^^^^^^^^^^^^^^ + +HAMSTRING application images use ``src.base.service_entrypoint`` as their container entrypoint. The +entrypoint waits for selected infrastructure before starting the module script. + +.. list-table:: Readiness variables + :header-rows: 1 + :widths: 30 25 45 + + * - Variable + - Default + - Description + * - ``HAMSTRING_WAIT_FOR`` + - unset + - Comma-separated dependencies to wait for. Supported values are ``kafka`` and ``clickhouse``. + Compose and Swarm set ``kafka`` for pipeline modules and ``kafka,clickhouse`` for the + monitoring agent. + * - ``HAMSTRING_WAIT_INITIAL_DELAY_SECONDS`` + - ``30`` + - Delay before dependency checks start. + * - ``HAMSTRING_WAIT_TIMEOUT_SECONDS`` + - ``180`` + - Maximum wait time per dependency endpoint. + * - ``HAMSTRING_WAIT_INTERVAL_SECONDS`` + - ``2`` + - Delay between readiness attempts. + * - ``HAMSTRING_KAFKA_WAIT_ENDPOINTS`` + - from ``environment.kafka_brokers`` + - Optional comma-separated ``host:port`` list overriding the Kafka endpoints used by the + readiness check. + * - ``HAMSTRING_CLICKHOUSE_WAIT_ENDPOINT`` + - from ``environment.monitoring.clickhouse_server`` + - Optional ``host:port`` endpoint overriding the ClickHouse readiness check. + +Application service variables +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: Application service variables + :header-rows: 1 + :widths: 30 25 45 + + * - Variable + - Default + - Description + * - ``GROUP_ID`` + - service-specific + - Kafka consumer group for the service. Compose and Swarm set service defaults such as + ``log_storage``, ``log_collection``, ``log_filtering``, ``data_inspection``, + ``data_analysis``, ``data_alerting``, and ``monitoring_agent``. + * - ``MONITORING_AGENT_GROUP_ID`` + - ``monitoring_agent`` + - Swarm override for the monitoring agent ``GROUP_ID``. + * - ``LOGSERVER_GROUP_ID`` + - ``log_storage`` + - Swarm override for the logserver ``GROUP_ID``. + * - ``LOGCOLLECTOR_GROUP_ID`` + - ``log_collection`` + - Swarm override for the logcollector ``GROUP_ID``. + * - ``PREFILTER_GROUP_ID`` + - ``log_filtering`` + - Swarm override for the prefilter ``GROUP_ID``. + * - ``INSPECTOR_GROUP_ID`` + - ``data_inspection`` + - Swarm override for the inspector ``GROUP_ID``. + * - ``DETECTOR_GROUP_ID`` + - ``data_analysis`` + - Swarm override for the detector ``GROUP_ID``. + * - ``ALERTER_GROUP_ID`` + - ``data_alerting`` + - Swarm override for the alerter ``GROUP_ID``. + * - ``NUMBER_OF_INSTANCES`` + - ``1`` + - Replica count hint used by Kafka topic creation. Set this to the Docker service replica count + when scaling a consumer service. + * - ``INSPECTOR_NUMBER_OF_INSTANCES`` + - ``1`` + - Swarm override for inspector ``NUMBER_OF_INSTANCES``. + * - ``KAFKA_TOPIC_PARTITIONS`` + - ``12`` + - Default partition count requested for new HAMSTRING Kafka topics. + * - ``KAFKA_TOPIC_REPLICATION_FACTOR`` + - from ``environment.kafka_topics.replication_factor`` + - Replication factor requested for new HAMSTRING Kafka topics. At runtime this is capped to the + configured broker count. + * - ``KAFKA_TOPIC_MIN_PARTITIONS`` + - ``1`` + - Runtime lower bound for topic partition creation and expansion. + * - ``HAMSTRING_CONFIG_CHECKSUM`` + - current compose value + - Optional deployment marker used to force Swarm service updates when ``config.yaml`` changes. + * - ``NVIDIA_VISIBLE_DEVICES`` + - ``all`` + - GPU device selection for detector services that use GPU acceleration. + * - ``NVIDIA_DRIVER_CAPABILITIES`` + - ``compute,utility`` + - NVIDIA runtime capabilities for GPU detector services. + * - ``ZEEK_CONTAINER_NAME`` + - ``zeek`` + - Container name passed to the Zeek image. + +Image variables +^^^^^^^^^^^^^^^ + +.. list-table:: Image variables + :header-rows: 1 + :widths: 35 25 40 + + * - Variable + - Default + - Description + * - ``HAMSTRING_IMAGE_REGISTRY`` + - ``ghcr.io/astraos-de`` + - Registry used for HAMSTRING application images. + * - ``HAMSTRING_MONITORING_IMAGE_TAG`` + - ``v2.2.0`` + - Tag for ``hamstring-monitoring``. + * - ``HAMSTRING_LOGSERVER_IMAGE_TAG`` + - ``v2.2.0`` + - Tag for ``hamstring-logserver``. + * - ``HAMSTRING_LOGCOLLECTOR_IMAGE_TAG`` + - ``v2.2.0`` + - Tag for ``hamstring-logcollector``. + * - ``HAMSTRING_PREFILTER_IMAGE_TAG`` + - ``v2.2.0`` + - Tag for ``hamstring-prefilter``. + * - ``HAMSTRING_INSPECTOR_IMAGE_TAG`` + - ``v2.2.0`` + - Tag for ``hamstring-inspector``. + * - ``HAMSTRING_DETECTOR_IMAGE_TAG`` + - ``v2.2.0`` + - Tag for ``hamstring-detector``. + * - ``HAMSTRING_ALERTER_IMAGE_TAG`` + - ``v2.2.0`` + - Tag for ``hamstring-alerter``. + * - ``HAMSTRING_ZEEK_IMAGE_TAG`` + - ``2.0.1`` + - Tag for ``hamstring-zeek``. + * - ``KAFKA_IMAGE_REPOSITORY`` + - ``confluentinc/cp-kafka`` + - Kafka image repository. + * - ``KAFKA_IMAGE_TAG`` + - ``8.2.2`` + - Kafka image tag. + * - ``CLICKHOUSE_IMAGE_REPOSITORY`` + - ``clickhouse/clickhouse-server`` + - ClickHouse image repository. + * - ``CLICKHOUSE_IMAGE_TAG`` + - ``26.5-alpine`` + - ClickHouse image tag. + * - ``GRAFANA_IMAGE_REPOSITORY`` + - ``grafana/grafana`` + - Grafana image repository. + * - ``GRAFANA_IMAGE_TAG`` + - ``13.0.3-slim`` + - Grafana image tag. + * - ``PROMETHEUS_IMAGE_REPOSITORY`` + - ``prom/prometheus`` + - Prometheus image repository. + * - ``PROMETHEUS_IMAGE_TAG`` + - ``latest`` + - Prometheus image tag. + * - ``KAFKA_EXPORTER_IMAGE_REPOSITORY`` + - ``danielqsj/kafka-exporter`` + - Kafka exporter image repository. + * - ``KAFKA_EXPORTER_IMAGE_TAG`` + - ``latest`` + - Kafka exporter image tag. + +Infrastructure variables +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: Infrastructure variables + :header-rows: 1 + :widths: 35 25 40 + + * - Variable + - Default + - Description + * - ``HOST_IP`` + - ``localhost`` + - Host advertised by Kafka for external listeners in Docker Compose. + * - ``KAFKA_EXTERNAL_HOST`` + - ``localhost`` + - Host advertised by Kafka for external listeners in Docker Swarm. + * - ``KAFKA1_EXTERNAL_PORT`` + - ``8097`` + - Published external port for Kafka broker 1. + * - ``KAFKA2_EXTERNAL_PORT`` + - ``8098`` + - Published external port for Kafka broker 2. + * - ``KAFKA3_EXTERNAL_PORT`` + - ``8099`` + - Published external port for Kafka broker 3. + * - ``KAFKA_CLUSTER_ID`` + - ``MkU3OEVBNTcwNTJENDM2Qk`` + - Kafka KRaft cluster id. + * - ``KAFKA_CONTROLLER_QUORUM_VOTERS`` + - ``1@kafka1:29093,2@kafka2:29093,3@kafka3:29093`` + - Kafka KRaft controller voter list. + * - ``KAFKA_AUTO_CREATE_TOPICS_ENABLE`` + - ``false`` + - Kafka broker auto-topic-creation setting. + * - ``KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR`` + - ``3`` + - Kafka transaction state topic replication factor. + * - ``KAFKA_TRANSACTION_STATE_LOG_MIN_ISR`` + - ``2`` + - Kafka transaction state topic minimum in-sync replicas. + * - ``KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR`` + - ``3`` + - Kafka offsets topic replication factor. + * - ``KAFKA_LOG_RETENTION_HOURS`` + - ``4`` + - Kafka log retention in hours. + * - ``KAFKA_LOG_RETENTION_BYTES`` + - ``10737418240`` + - Kafka log retention size. + * - ``KAFKA_LOG_SEGMENT_BYTES`` + - ``1073741824`` + - Kafka log segment size. + * - ``KAFKA_LOG_CLEANUP_POLICY`` + - ``delete`` + - Kafka log cleanup policy. + * - ``KAFKA_LOG4J_LOGGERS`` + - Kafka controller defaults + - Optional Kafka log category configuration for brokers 2 and 3. + * - ``CLICKHOUSE_USER`` + - ``default`` + - ClickHouse user used by ClickHouse, Grafana, and the monitoring agent. + * - ``CLICKHOUSE_PASSWORD`` + - ``hamstring`` + - ClickHouse password used by ClickHouse, Grafana, and the monitoring agent. + * - ``CLICKHOUSE_HTTP_PORT`` + - ``8123`` + - Published ClickHouse HTTP port in Swarm. + * - ``CLICKHOUSE_NATIVE_PORT`` + - ``9000`` + - Published ClickHouse native client port in Swarm. + * - ``GRAFANA_PORT`` + - ``3000`` + - Published Grafana port in Swarm. + * - ``GRAFANA_ADMIN_USER`` + - ``admin`` + - Grafana admin username. + * - ``GRAFANA_ADMIN_PASSWORD`` + - ``admin`` + - Grafana admin password. + * - ``GRAFANA_INSTALL_PLUGINS`` + - ``grafana-clickhouse-datasource`` + - Grafana plugins installed at startup. + * - ``PROMETHEUS_PORT`` + - ``9088`` + - Published Prometheus port in Swarm. + * - ``PROMETHEUS_CONFIG_FILE`` + - ``../../docker/prometheus/prometheus.yml`` + - Prometheus config file used for the Swarm config object. + * - ``KAFKA_EXPORTER_PORT`` + - ``9308`` + - Published Kafka exporter port in Swarm. + +Path and mount variables +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: Path variables + :header-rows: 1 + :widths: 35 25 40 + + * - Variable + - Default + - Description + * - ``HAMSTRING_ROOT`` + - ``../..`` + - Base path for Swarm bind mounts to ``config.yaml`` and provisioning files. + * - ``LOGSERVER_INPUT_PATH`` + - ``docker/default_input`` + - Host file mounted to ``/opt/file.txt`` for the logserver. The exact relative default differs + between Compose fragments and the Swarm file but resolves to ``docker/default_input`` from the + repository. + * - ``ALERTER_LOGS_PATH`` + - ``/opt/logs`` in prod, ``../../../logs`` in dev + - Host directory mounted to ``/opt/logs`` by Docker Compose. The Swarm stack uses a named volume + instead. + +Swarm scheduling variables +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Docker Swarm placement constraints default to ``node.platform.os == linux``. Set the corresponding +variable to pin a service to a labeled node, for example +``KAFKA1_PLACEMENT_CONSTRAINT='node.labels.kafka1 == true'``. + +.. list-table:: Swarm placement and replica variables + :header-rows: 1 + :widths: 35 25 40 + + * - Variable + - Default + - Description + * - ``KAFKA1_PLACEMENT_CONSTRAINT`` + - ``node.platform.os == linux`` + - Placement constraint for Kafka broker 1. + * - ``KAFKA2_PLACEMENT_CONSTRAINT`` + - ``node.platform.os == linux`` + - Placement constraint for Kafka broker 2. + * - ``KAFKA3_PLACEMENT_CONSTRAINT`` + - ``node.platform.os == linux`` + - Placement constraint for Kafka broker 3. + * - ``CLICKHOUSE_PLACEMENT_CONSTRAINT`` + - ``node.platform.os == linux`` + - Placement constraint for ClickHouse. + * - ``GRAFANA_PLACEMENT_CONSTRAINT`` + - ``node.platform.os == linux`` + - Placement constraint for Grafana. + * - ``PROMETHEUS_PLACEMENT_CONSTRAINT`` + - ``node.platform.os == linux`` + - Placement constraint for Prometheus. + * - ``KAFKA_EXPORTER_PLACEMENT_CONSTRAINT`` + - ``node.platform.os == linux`` + - Placement constraint for Kafka exporter. + * - ``MONITORING_AGENT_PLACEMENT_CONSTRAINT`` + - ``node.platform.os == linux`` + - Placement constraint for the monitoring agent. + * - ``LOGSERVER_PLACEMENT_CONSTRAINT`` + - ``node.platform.os == linux`` + - Placement constraint for the logserver. + * - ``LOGCOLLECTOR_PLACEMENT_CONSTRAINT`` + - ``node.platform.os == linux`` + - Placement constraint for the logcollector. + * - ``PREFILTER_PLACEMENT_CONSTRAINT`` + - ``node.platform.os == linux`` + - Placement constraint for the prefilter. + * - ``INSPECTOR_PLACEMENT_CONSTRAINT`` + - ``node.platform.os == linux`` + - Placement constraint for the inspector. + * - ``DETECTOR_PLACEMENT_CONSTRAINT`` + - ``node.platform.os == linux`` + - Placement constraint for the detector. + * - ``ALERTER_PLACEMENT_CONSTRAINT`` + - ``node.platform.os == linux`` + - Placement constraint for the alerter. + * - ``ZEEK_PLACEMENT_CONSTRAINT`` + - ``node.platform.os == linux`` + - Placement constraint for Zeek. + * - ``PREFILTER_REPLICAS`` + - ``1`` + - Swarm replica count for the prefilter service. + * - ``INSPECTOR_REPLICAS`` + - ``1`` + - Swarm replica count for the inspector service. + * - ``DETECTOR_REPLICAS`` + - ``1`` + - Swarm replica count for the detector service. + * - ``ALERTER_REPLICAS`` + - ``1`` + - Swarm replica count for the alerter service. + * - ``ZEEK_REPLICAS`` + - ``1`` + - Swarm replica count for the Zeek service. + * - ``INSPECTOR_CPU_LIMIT`` + - ``2`` + - Swarm CPU limit for the inspector. + * - ``INSPECTOR_MEMORY_LIMIT`` + - ``512M`` + - Swarm memory limit for the inspector. + * - ``INSPECTOR_CPU_RESERVATION`` + - ``1`` + - Swarm CPU reservation for the inspector. + * - ``INSPECTOR_MEMORY_RESERVATION`` + - ``256M`` + - Swarm memory reservation for the inspector. diff --git a/docs/usage.rst b/docs/usage.rst index a9c10dde..5d67d800 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -12,21 +12,20 @@ Usage Getting Started --------------- -To use HAMSTRING, just use the provided ``docker-compose.yml`` to quickly bootstrap your environment: +To use HAMSTRING, use the provided ``docker-compose.yml`` with either the production or development profile: .. code-block:: console - $ HOST_IP=127.0.0.1 docker compose -f docker/docker-compose.yml up + $ HOST_IP=127.0.0.1 docker compose -f docker/docker-compose.yml --profile prod up -If you want to run containers individually, use: +For local development builds, use the development profile: .. code-block:: console - $ HOST_IP=127.0.0.1 docker compose -f docker/docker-compose.kafka.yml up - $ docker run ... + $ HOST_IP=127.0.0.1 docker compose -f docker/docker-compose.yml --profile dev up - -Make sure you set the environment variable ``HOST_IP`` to your host's IP address, so that the services can communicate with each other. +Set ``HOST_IP`` to the host address that external Kafka clients should use. The default +``localhost`` works for single-host local runs. Scaling With Docker Compose --------------------------- diff --git a/src/base/service_entrypoint.py b/src/base/service_entrypoint.py new file mode 100644 index 00000000..1b9e5ab7 --- /dev/null +++ b/src/base/service_entrypoint.py @@ -0,0 +1,182 @@ +"""Container entrypoint that waits for service dependencies before startup.""" + +from __future__ import annotations + +import http.client +import logging +import os +import socket +import sys +import time +from collections.abc import Callable + +import yaml + + +logging.basicConfig( + format="[%(asctime)s, %(levelname)s] %(message)s", + level=logging.INFO, +) +logger = logging.getLogger(__name__) +CONFIG_FILEPATH = os.path.join(os.path.dirname(__file__), "../../config.yaml") +DEFAULT_CLICKHOUSE_HTTP_PORT = 8123 + + +def parse_dependency_names(raw_dependencies: str | None) -> list[str]: + if not raw_dependencies: + return [] + + return [ + dependency.strip().lower() + for dependency in raw_dependencies.split(",") + if dependency.strip() + ] + + +def parse_host_port_list(raw_endpoints: str | None) -> list[tuple[str, int]]: + if not raw_endpoints: + return [] + + endpoints: list[tuple[str, int]] = [] + for raw_endpoint in raw_endpoints.split(","): + endpoint = raw_endpoint.strip() + if not endpoint: + continue + + host, separator, raw_port = endpoint.rpartition(":") + if not separator or not host or not raw_port: + raise ValueError(f"Invalid endpoint {endpoint!r}; expected host:port") + + endpoints.append((host, int(raw_port))) + + return endpoints + + +def get_kafka_endpoints(config: dict) -> list[tuple[str, int]]: + env_endpoints = parse_host_port_list(os.getenv("HAMSTRING_KAFKA_WAIT_ENDPOINTS")) + if env_endpoints: + return env_endpoints + + endpoints = [] + for broker in config["environment"]["kafka_brokers"]: + endpoints.append((broker["hostname"], int(broker["internal_port"]))) + return endpoints + + +def get_clickhouse_endpoint(config: dict) -> tuple[str, int]: + env_endpoints = parse_host_port_list( + os.getenv("HAMSTRING_CLICKHOUSE_WAIT_ENDPOINT") + ) + if env_endpoints: + return env_endpoints[0] + + clickhouse_config = config["environment"]["monitoring"]["clickhouse_server"] + return ( + clickhouse_config["hostname"], + int(clickhouse_config.get("http_port", DEFAULT_CLICKHOUSE_HTTP_PORT)), + ) + + +def setup_config() -> dict: + with open(CONFIG_FILEPATH, "r", encoding="utf-8") as file: + return yaml.safe_load(file) + + +def can_connect(host: str, port: int, timeout_seconds: float) -> bool: + try: + with socket.create_connection((host, port), timeout=timeout_seconds): + return True + except OSError: + return False + + +def can_ping_clickhouse(host: str, port: int, timeout_seconds: float) -> bool: + connection = http.client.HTTPConnection(host, port, timeout=timeout_seconds) + try: + connection.request("GET", "/ping") + response = connection.getresponse() + response.read() + return response.status == 200 + except OSError: + return False + finally: + connection.close() + + +def wait_until_ready( + name: str, + check: Callable[[], bool], + timeout_seconds: int, + interval_seconds: float, +) -> None: + deadline = time.monotonic() + timeout_seconds + + while True: + if check(): + logger.info("%s is reachable.", name) + return + + if time.monotonic() >= deadline: + raise TimeoutError(f"Timed out waiting for {name}.") + + logger.info("Waiting for %s...", name) + time.sleep(interval_seconds) + + +def wait_for_dependencies(dependencies: list[str]) -> None: + if not dependencies: + return + + config = setup_config() + timeout_seconds = int(os.getenv("HAMSTRING_WAIT_TIMEOUT_SECONDS", "180")) + interval_seconds = float(os.getenv("HAMSTRING_WAIT_INTERVAL_SECONDS", "2")) + initial_delay_seconds = float( + os.getenv("HAMSTRING_WAIT_INITIAL_DELAY_SECONDS", "30") + ) + + if initial_delay_seconds > 0: + logger.info( + "Waiting %.1fs before dependency checks.", + initial_delay_seconds, + ) + time.sleep(initial_delay_seconds) + + for dependency in dependencies: + if dependency == "kafka": + endpoints = get_kafka_endpoints(config) + for host, port in endpoints: + wait_until_ready( + f"kafka at {host}:{port}", + lambda host=host, port=port: can_connect( + host, + port, + timeout_seconds=2, + ), + timeout_seconds, + interval_seconds, + ) + elif dependency == "clickhouse": + host, port = get_clickhouse_endpoint(config) + wait_until_ready( + f"clickhouse at {host}:{port}", + lambda: can_ping_clickhouse(host, port, timeout_seconds=2), + timeout_seconds, + interval_seconds, + ) + else: + raise ValueError( + f"Unsupported dependency {dependency!r}. " + "Supported dependencies: kafka, clickhouse." + ) + + +def main() -> None: + if len(sys.argv) < 2: + raise SystemExit("Usage: service_entrypoint.py [args...]") + + wait_for_dependencies(parse_dependency_names(os.getenv("HAMSTRING_WAIT_FOR"))) + os.execv(sys.executable, [sys.executable, *sys.argv[1:]]) + + +if __name__ == "__main__": + main() diff --git a/tests/miscellaneous/test_service_entrypoint.py b/tests/miscellaneous/test_service_entrypoint.py new file mode 100644 index 00000000..ba2a4ea8 --- /dev/null +++ b/tests/miscellaneous/test_service_entrypoint.py @@ -0,0 +1,118 @@ +import unittest +from unittest.mock import MagicMock, patch + +from src.base import service_entrypoint + + +class TestParsing(unittest.TestCase): + def test_parse_dependency_names(self): + self.assertEqual( + ["kafka", "clickhouse"], + service_entrypoint.parse_dependency_names(" kafka, ClickHouse ,, "), + ) + + def test_parse_host_port_list(self): + self.assertEqual( + [("kafka1", 19092), ("clickhouse-server", 8123)], + service_entrypoint.parse_host_port_list( + "kafka1:19092, clickhouse-server:8123" + ), + ) + + def test_parse_host_port_list_rejects_invalid_endpoint(self): + with self.assertRaises(ValueError): + service_entrypoint.parse_host_port_list("kafka1") + + +class TestEndpointDiscovery(unittest.TestCase): + def test_get_kafka_endpoints_from_config(self): + config = { + "environment": { + "kafka_brokers": [ + {"hostname": "kafka1", "internal_port": 19092}, + {"hostname": "kafka2", "internal_port": "19093"}, + ] + } + } + + self.assertEqual( + [("kafka1", 19092), ("kafka2", 19093)], + service_entrypoint.get_kafka_endpoints(config), + ) + + @patch.dict( + "os.environ", + {"HAMSTRING_KAFKA_WAIT_ENDPOINTS": "localhost:9092,localhost:9093"}, + ) + def test_get_kafka_endpoints_from_env(self): + self.assertEqual( + [("localhost", 9092), ("localhost", 9093)], + service_entrypoint.get_kafka_endpoints({}), + ) + + def test_get_clickhouse_endpoint_from_config(self): + config = { + "environment": { + "monitoring": { + "clickhouse_server": { + "hostname": "clickhouse-server", + "http_port": 8124, + } + } + } + } + + self.assertEqual( + ("clickhouse-server", 8124), + service_entrypoint.get_clickhouse_endpoint(config), + ) + + @patch.dict( + "os.environ", + {"HAMSTRING_CLICKHOUSE_WAIT_ENDPOINT": "localhost:18123"}, + ) + def test_get_clickhouse_endpoint_from_env(self): + self.assertEqual( + ("localhost", 18123), + service_entrypoint.get_clickhouse_endpoint({}), + ) + + +class TestReadinessChecks(unittest.TestCase): + @patch("src.base.service_entrypoint.socket.create_connection") + def test_can_connect_to_open_socket(self, mock_create_connection): + connection = MagicMock() + mock_create_connection.return_value.__enter__.return_value = connection + + self.assertTrue(service_entrypoint.can_connect("127.0.0.1", 19092, 1)) + mock_create_connection.assert_called_once_with(("127.0.0.1", 19092), timeout=1) + + @patch("src.base.service_entrypoint.http.client.HTTPConnection") + def test_can_ping_clickhouse(self, mock_http_connection): + response = MagicMock() + response.status = 200 + connection = mock_http_connection.return_value + connection.getresponse.return_value = response + + self.assertTrue(service_entrypoint.can_ping_clickhouse("127.0.0.1", 8123, 1)) + mock_http_connection.assert_called_once_with("127.0.0.1", 8123, timeout=1) + connection.request.assert_called_once_with("GET", "/ping") + response.read.assert_called_once() + connection.close.assert_called_once() + + +class TestMain(unittest.TestCase): + @patch.dict("os.environ", {"HAMSTRING_WAIT_FOR": ""}, clear=False) + @patch("src.base.service_entrypoint.os.execv") + @patch("src.base.service_entrypoint.sys.argv", ["service_entrypoint.py", "app.py"]) + def test_main_execs_target(self, mock_execv): + service_entrypoint.main() + + mock_execv.assert_called_once_with( + service_entrypoint.sys.executable, + [service_entrypoint.sys.executable, "app.py"], + ) + + +if __name__ == "__main__": + unittest.main()