diff --git a/.github/workflows/publish_dev.yml b/.github/workflows/publish_dev.yml index a717d818..81114a36 100644 --- a/.github/workflows/publish_dev.yml +++ b/.github/workflows/publish_dev.yml @@ -149,6 +149,4 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - file: ./docker/dockerfiles/Dockerfile.${{ matrix.container }} - cache-from: type=gha - cache-to: type=gha,mode=max + file: ./docker/dockerfiles/Dockerfile.${{ matrix.container }} \ No newline at end of file diff --git a/.github/workflows/publish_main.yml b/.github/workflows/publish_main.yml index 6040a85d..a2bb0234 100644 --- a/.github/workflows/publish_main.yml +++ b/.github/workflows/publish_main.yml @@ -177,5 +177,3 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} file: ./docker/dockerfiles/Dockerfile.${{ matrix.container }} - cache-from: type=gha - cache-to: type=gha,mode=max diff --git a/config.yaml b/config.yaml index 5aba2f3d..1e1a3431 100644 --- a/config.yaml +++ b/config.yaml @@ -288,7 +288,7 @@ pipeline: protocols: - dns interfaces: - - eth0 + - wlp3s0 environment: kafka_brokers: diff --git a/docker/data/test_pcaps/ctu-sample.pcap b/docker/data/test_pcaps/ctu-sample.pcap new file mode 100755 index 00000000..b9b8d19c Binary files /dev/null and b/docker/data/test_pcaps/ctu-sample.pcap differ diff --git a/docker/docker-compose/prod/docker-compose.monitoring.yml b/docker/docker-compose/prod/docker-compose.monitoring.yml index 7dbfa0d9..8d38f2c8 100644 --- a/docker/docker-compose/prod/docker-compose.monitoring.yml +++ b/docker/docker-compose/prod/docker-compose.monitoring.yml @@ -1,6 +1,6 @@ services: monitoring_agent: - image: ghcr.io/astraos-de/hamstring-monitoring:v2.2.0-dev + image: ghcr.io/astraos-de/hamstring-monitoring:v2.2.0 restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml diff --git a/docker/docker-compose/prod/docker-compose.pipeline.yml b/docker/docker-compose/prod/docker-compose.pipeline.yml index 2ea6803c..60fa083f 100644 --- a/docker/docker-compose/prod/docker-compose.pipeline.yml +++ b/docker/docker-compose/prod/docker-compose.pipeline.yml @@ -1,6 +1,6 @@ services: logserver: - image: ghcr.io/astraos-de/hamstring-logserver:v2.2.0-dev + image: ghcr.io/astraos-de/hamstring-logserver:v2.2.0 restart: "unless-stopped" networks: hamstring: @@ -10,7 +10,7 @@ services: environment: - GROUP_ID=log_storage logcollector: - image: ghcr.io/astraos-de/hamstring-logcollector:v2.2.0-dev + image: ghcr.io/astraos-de/hamstring-logcollector:v2.2.0 restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml @@ -20,7 +20,7 @@ services: - GROUP_ID=log_collection prefilter: - image: ghcr.io/astraos-de/hamstring-prefilter:v2.2.0-dev + image: ghcr.io/astraos-de/hamstring-prefilter:v2.2.0 restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml @@ -33,7 +33,7 @@ services: - GROUP_ID=log_filtering inspector: - image: ghcr.io/astraos-de/hamstring-inspector:v2.2.0-dev + image: ghcr.io/astraos-de/hamstring-inspector:v2.2.0 restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml @@ -54,7 +54,7 @@ services: - NUMBER_OF_INSTANCES=1 detector: - image: ghcr.io/astraos-de/hamstring-detector:v2.2.0-dev + image: ghcr.io/astraos-de/hamstring-detector:v2.2.0 restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml @@ -67,7 +67,7 @@ services: - GROUP_ID=data_analysis detector-gpu: - image: ghcr.io/astraos-de/hamstring-detector:v2.2.0-dev + image: ghcr.io/astraos-de/hamstring-detector:v2.2.0 restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml @@ -88,7 +88,7 @@ services: - NVIDIA_DRIVER_CAPABILITIES=compute,utility alerter: - image: ghcr.io/astraos-de/hamstring-alerter:v2.2.0-dev + image: ghcr.io/astraos-de/hamstring-alerter:v2.2.0 restart: "unless-stopped" volumes: - ../../../config.yaml:/app/config.yaml diff --git a/docker/docker_swarm/docker-compose.swarm.yml b/docker/docker_swarm/docker-compose.swarm.yml index 52edefe4..00d59a34 100644 --- a/docker/docker_swarm/docker-compose.swarm.yml +++ b/docker/docker_swarm/docker-compose.swarm.yml @@ -1,10 +1,8 @@ # Compose file to deploy directly with docker stack deploy. --- -version: "3.8" - services: zookeeper: - image: confluentinc/cp-zookeeper:7.9.3 + image: "${ZOOKEEPER_IMAGE:-confluentinc/cp-zookeeper:7.9.3}" networks: - hamstring environment: @@ -20,38 +18,35 @@ services: timeout: 5s retries: 3 deploy: - placement: - constraints: - - "node.hostname == workstation" restart_policy: condition: any kafka1: - image: confluentinc/cp-kafka:7.9.3 + image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:7.9.3}" networks: - hamstring ports: - target: 8097 - published: 8097 + published: "${KAFKA1_EXTERNAL_PORT:-8097}" protocol: tcp mode: ingress environment: KAFKA_BROKER_ID: "1" - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ZOOKEEPER_CONNECT: "${KAFKA_ZOOKEEPER_CONNECT:-zookeeper:2181}" KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:19092,EXTERNAL://workstation:8097 + KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka1:19092,EXTERNAL://${KAFKA_EXTERNAL_HOST:-localhost}:${KAFKA1_EXTERNAL_PORT:-8097}" KAFKA_LISTENERS: INTERNAL://0.0.0.0:19092,EXTERNAL://0.0.0.0:8097 KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "3" - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "2" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "3" + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "${KAFKA_AUTO_CREATE_TOPICS_ENABLE:-false}" + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "${KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR:-3}" + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "${KAFKA_TRANSACTION_STATE_LOG_MIN_ISR:-2}" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "${KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR:-3}" KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" - KAFKA_LOG_RETENTION_HOURS: 4 - KAFKA_LOG_RETENTION_BYTES: 10737418240 - KAFKA_LOG_SEGMENT_BYTES: 1073741824 - KAFKA_LOG_CLEANUP_POLICY: delete + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "${KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND:-true}" + KAFKA_LOG_RETENTION_HOURS: "${KAFKA_LOG_RETENTION_HOURS:-4}" + KAFKA_LOG_RETENTION_BYTES: "${KAFKA_LOG_RETENTION_BYTES:-10737418240}" + KAFKA_LOG_SEGMENT_BYTES: "${KAFKA_LOG_SEGMENT_BYTES:-1073741824}" + KAFKA_LOG_CLEANUP_POLICY: "${KAFKA_LOG_CLEANUP_POLICY:-delete}" volumes: - kafka-data1:/var/lib/kafka/data healthcheck: @@ -60,39 +55,36 @@ services: timeout: 10s retries: 10 deploy: - placement: - constraints: - - "node.hostname == workstation" restart_policy: condition: any kafka2: - image: confluentinc/cp-kafka:7.9.3 + image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:7.9.3}" networks: - hamstring ports: - target: 8098 - published: 8098 + published: "${KAFKA2_EXTERNAL_PORT:-8098}" protocol: tcp mode: ingress environment: KAFKA_BROKER_ID: "2" - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ZOOKEEPER_CONNECT: "${KAFKA_ZOOKEEPER_CONNECT:-zookeeper:2181}" KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:19093,EXTERNAL://workstation:8098 + KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka2:19093,EXTERNAL://${KAFKA_EXTERNAL_HOST:-localhost}:${KAFKA2_EXTERNAL_PORT:-8098}" KAFKA_LISTENERS: INTERNAL://0.0.0.0:19093,EXTERNAL://0.0.0.0:8098 KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" - KAFKA_LOG4J_LOGGERS: kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "3" - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "2" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "3" + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "${KAFKA_AUTO_CREATE_TOPICS_ENABLE:-false}" + KAFKA_LOG4J_LOGGERS: "${KAFKA_LOG4J_LOGGERS:-kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO}" + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "${KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR:-3}" + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "${KAFKA_TRANSACTION_STATE_LOG_MIN_ISR:-2}" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "${KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR:-3}" KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" - KAFKA_LOG_RETENTION_HOURS: 4 - KAFKA_LOG_RETENTION_BYTES: 10737418240 - KAFKA_LOG_SEGMENT_BYTES: 1073741824 - KAFKA_LOG_CLEANUP_POLICY: delete + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "${KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND:-true}" + KAFKA_LOG_RETENTION_HOURS: "${KAFKA_LOG_RETENTION_HOURS:-4}" + KAFKA_LOG_RETENTION_BYTES: "${KAFKA_LOG_RETENTION_BYTES:-10737418240}" + KAFKA_LOG_SEGMENT_BYTES: "${KAFKA_LOG_SEGMENT_BYTES:-1073741824}" + KAFKA_LOG_CLEANUP_POLICY: "${KAFKA_LOG_CLEANUP_POLICY:-delete}" volumes: - kafka-data2:/var/lib/kafka/data healthcheck: @@ -101,39 +93,36 @@ services: timeout: 10s retries: 10 deploy: - placement: - constraints: - - "node.hostname == workstation" restart_policy: condition: any kafka3: - image: confluentinc/cp-kafka:7.9.3 + image: "${KAFKA_IMAGE:-confluentinc/cp-kafka:7.9.3}" networks: - hamstring ports: - target: 8099 - published: 8099 + published: "${KAFKA3_EXTERNAL_PORT:-8099}" protocol: tcp mode: ingress environment: KAFKA_BROKER_ID: "3" - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ZOOKEEPER_CONNECT: "${KAFKA_ZOOKEEPER_CONNECT:-zookeeper:2181}" KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:19094,EXTERNAL://workstation:8099 + KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka3:19094,EXTERNAL://${KAFKA_EXTERNAL_HOST:-localhost}:${KAFKA3_EXTERNAL_PORT:-8099}" KAFKA_LISTENERS: INTERNAL://0.0.0.0:19094,EXTERNAL://0.0.0.0:8099 KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" - KAFKA_LOG4J_LOGGERS: kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "3" - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "2" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "3" + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "${KAFKA_AUTO_CREATE_TOPICS_ENABLE:-false}" + KAFKA_LOG4J_LOGGERS: "${KAFKA_LOG4J_LOGGERS:-kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO}" + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "${KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR:-3}" + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "${KAFKA_TRANSACTION_STATE_LOG_MIN_ISR:-2}" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "${KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR:-3}" KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" - KAFKA_LOG_RETENTION_HOURS: 4 - KAFKA_LOG_RETENTION_BYTES: 10737418240 - KAFKA_LOG_SEGMENT_BYTES: 1073741824 - KAFKA_LOG_CLEANUP_POLICY: delete + KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "${KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND:-true}" + KAFKA_LOG_RETENTION_HOURS: "${KAFKA_LOG_RETENTION_HOURS:-4}" + KAFKA_LOG_RETENTION_BYTES: "${KAFKA_LOG_RETENTION_BYTES:-10737418240}" + KAFKA_LOG_SEGMENT_BYTES: "${KAFKA_LOG_SEGMENT_BYTES:-1073741824}" + KAFKA_LOG_CLEANUP_POLICY: "${KAFKA_LOG_CLEANUP_POLICY:-delete}" volumes: - kafka-data3:/var/lib/kafka/data healthcheck: @@ -142,27 +131,24 @@ services: timeout: 10s retries: 10 deploy: - placement: - constraints: - - "node.hostname == workstation" restart_policy: condition: any clickhouse-server: - image: clickhouse/clickhouse-server:24.3.12.75-alpine + image: "${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:24.3.12.75-alpine}" networks: - hamstring ports: - target: 8123 - published: 8123 + published: "${CLICKHOUSE_HTTP_PORT:-8123}" protocol: tcp mode: ingress - target: 9000 - published: 9000 + published: "${CLICKHOUSE_NATIVE_PORT:-9000}" protocol: tcp mode: ingress volumes: - - "/home/max/HAMSTRING/hamstring/docker/create_tables:/docker-entrypoint-initdb.d" + - "${HAMSTRING_ROOT:-../..}/docker/create_tables:/docker-entrypoint-initdb.d" - ch_data:/var/lib/clickhouse/ - ch_logs:/var/log/clickhouse-server/ healthcheck: @@ -171,61 +157,52 @@ services: timeout: 5s retries: 3 deploy: - placement: - constraints: - - "node.hostname == workstation" restart_policy: condition: any grafana: - image: grafana/grafana:11.2.2-security-01 + image: "${GRAFANA_IMAGE:-grafana/grafana:11.2.2-security-01}" networks: - hamstring ports: - target: 3000 - published: 3000 + published: "${GRAFANA_PORT:-3000}" protocol: tcp mode: ingress volumes: - - "/home/max/HAMSTRING/hamstring/docker/grafana-provisioning/dashboards:/etc/grafana/provisioning/dashboards" - - "/home/max/HAMSTRING/hamstring/docker/grafana-provisioning/dashboards/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml" - - "/home/max/HAMSTRING/hamstring/docker/grafana-provisioning/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml" + - "${HAMSTRING_ROOT:-../..}/docker/grafana-provisioning/dashboards:/etc/grafana/provisioning/dashboards" + - "${HAMSTRING_ROOT:-../..}/docker/grafana-provisioning/dashboards/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml" + - "${HAMSTRING_ROOT:-../..}/docker/grafana-provisioning/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml" environment: - GF_SECURITY_ADMIN_USER: admin - GF_SECURITY_ADMIN_PASSWORD: admin - GF_INSTALL_PLUGINS: grafana-clickhouse-datasource + GF_SECURITY_ADMIN_USER: "${GRAFANA_ADMIN_USER:-admin}" + GF_SECURITY_ADMIN_PASSWORD: "${GRAFANA_ADMIN_PASSWORD:-admin}" + GF_INSTALL_PLUGINS: "${GRAFANA_INSTALL_PLUGINS:-grafana-clickhouse-datasource}" healthcheck: test: ["CMD-SHELL", "nc -z localhost 3000"] interval: 10s timeout: 5s retries: 3 deploy: - placement: - constraints: - - "node.hostname == workstation" restart_policy: condition: any prometheus: - image: prom/prometheus + image: "${PROMETHEUS_IMAGE:-prom/prometheus}" networks: - hamstring ports: - target: 9090 - published: 9088 + published: "${PROMETHEUS_PORT:-9088}" protocol: tcp mode: ingress volumes: - - "/home/max/HAMSTRING/hamstring/docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml" + - "${HAMSTRING_ROOT:-../..}/docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml" deploy: - placement: - constraints: - - "node.hostname == workstation" restart_policy: condition: any kafka-exporter: - image: danielqsj/kafka-exporter + image: "${KAFKA_EXPORTER_IMAGE:-danielqsj/kafka-exporter}" command: - --kafka.server=kafka1:19092 - --kafka.server=kafka2:19093 @@ -234,7 +211,7 @@ services: - hamstring ports: - target: 9308 - published: 9308 + published: "${KAFKA_EXPORTER_PORT:-9308}" protocol: tcp mode: ingress deploy: @@ -242,97 +219,103 @@ services: condition: any monitoring_agent: - image: ghcr.io/astraos-de/hamstring-monitoring:v2.2.0-dev + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-monitoring:${HAMSTRING_IMAGE_TAG:-v2.2.0}" + command: ["src/monitoring/monitoring_agent.py"] networks: - hamstring volumes: - - "/home/max/HAMSTRING/hamstring/config.yaml:/app/config.yaml" + - "${HAMSTRING_ROOT:-../..}/config.yaml:/app/config.yaml" environment: - GROUP_ID: monitoring_agent - HAMSTRING_CONFIG_CHECKSUM: "d9bc812b76c2b58c633b412a6c07f981f3c4d787" + GROUP_ID: "${MONITORING_AGENT_GROUP_ID:-monitoring_agent}" + HAMSTRING_CONFIG_CHECKSUM: "${HAMSTRING_CONFIG_CHECKSUM:-d9bc812b76c2b58c633b412a6c07f981f3c4d787}" deploy: restart_policy: condition: any logserver: - image: ghcr.io/astraos-de/hamstring-logserver:v2.2.0-dev + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-logserver:${HAMSTRING_IMAGE_TAG:-v2.2.0}" + command: ["src/logserver/server.py"] networks: - hamstring volumes: - - "/home/max/HAMSTRING/hamstring/docker/default.txt:/opt/file.txt" - - "/home/max/HAMSTRING/hamstring/config.yaml:/app/config.yaml" + - "${HAMSTRING_ROOT:-../..}/docker/default.txt:/opt/file.txt" + - "${HAMSTRING_ROOT:-../..}/config.yaml:/app/config.yaml" environment: - GROUP_ID: log_storage - HAMSTRING_CONFIG_CHECKSUM: "d9bc812b76c2b58c633b412a6c07f981f3c4d787" + GROUP_ID: "${LOGSERVER_GROUP_ID:-log_storage}" + HAMSTRING_CONFIG_CHECKSUM: "${HAMSTRING_CONFIG_CHECKSUM:-d9bc812b76c2b58c633b412a6c07f981f3c4d787}" deploy: restart_policy: condition: any logcollector: - image: ghcr.io/astraos-de/hamstring-logcollector:v2.2.0-dev + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-logcollector:${HAMSTRING_IMAGE_TAG:-v2.2.0}" + command: ["src/logcollector/collector.py"] networks: - hamstring volumes: - - "/home/max/HAMSTRING/hamstring/config.yaml:/app/config.yaml" + - "${HAMSTRING_ROOT:-../..}/config.yaml:/app/config.yaml" environment: - GROUP_ID: log_collection - HAMSTRING_CONFIG_CHECKSUM: "d9bc812b76c2b58c633b412a6c07f981f3c4d787" + GROUP_ID: "${LOGCOLLECTOR_GROUP_ID:-log_collection}" + HAMSTRING_CONFIG_CHECKSUM: "${HAMSTRING_CONFIG_CHECKSUM:-d9bc812b76c2b58c633b412a6c07f981f3c4d787}" deploy: restart_policy: condition: any prefilter: - image: ghcr.io/astraos-de/hamstring-prefilter:v2.2.0-dev + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-prefilter:${HAMSTRING_IMAGE_TAG:-v2.2.0}" + command: ["src/prefilter/prefilter.py"] networks: - hamstring volumes: - - "/home/max/HAMSTRING/hamstring/config.yaml:/app/config.yaml" + - "${HAMSTRING_ROOT:-../..}/config.yaml:/app/config.yaml" environment: - GROUP_ID: log_filtering - HAMSTRING_CONFIG_CHECKSUM: "d9bc812b76c2b58c633b412a6c07f981f3c4d787" + GROUP_ID: "${PREFILTER_GROUP_ID:-log_filtering}" + HAMSTRING_CONFIG_CHECKSUM: "${HAMSTRING_CONFIG_CHECKSUM:-d9bc812b76c2b58c633b412a6c07f981f3c4d787}" deploy: mode: replicated - replicas: 1 + replicas: ${PREFILTER_REPLICAS:-1} restart_policy: condition: any inspector: - image: ghcr.io/astraos-de/hamstring-inspector:v2.2.0-dev + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-inspector:${HAMSTRING_IMAGE_TAG:-v2.2.0}" + command: ["src/inspector/inspector.py"] networks: - hamstring volumes: - - "/home/max/HAMSTRING/hamstring/config.yaml:/app/config.yaml" + - "${HAMSTRING_ROOT:-../..}/config.yaml:/app/config.yaml" environment: - GROUP_ID: data_inspection - NUMBER_OF_INSTANCES: "1" - HAMSTRING_CONFIG_CHECKSUM: "d9bc812b76c2b58c633b412a6c07f981f3c4d787" + GROUP_ID: "${INSPECTOR_GROUP_ID:-data_inspection}" + NUMBER_OF_INSTANCES: "${INSPECTOR_NUMBER_OF_INSTANCES:-1}" + HAMSTRING_CONFIG_CHECKSUM: "${HAMSTRING_CONFIG_CHECKSUM:-d9bc812b76c2b58c633b412a6c07f981f3c4d787}" deploy: mode: replicated - replicas: 1 + replicas: ${INSPECTOR_REPLICAS:-1} resources: limits: - cpus: "2" - memory: 512M + cpus: "${INSPECTOR_CPU_LIMIT:-2}" + memory: "${INSPECTOR_MEMORY_LIMIT:-512M}" reservations: - cpus: "1" - memory: 256M + cpus: "${INSPECTOR_CPU_RESERVATION:-1}" + memory: "${INSPECTOR_MEMORY_RESERVATION:-256M}" restart_policy: condition: any detector: - image: ghcr.io/astraos-de/hamstring-detector:v2.2.0-dev + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-detector:${HAMSTRING_IMAGE_TAG:-v2.2.0}" + command: ["src/detector/detector.py"] networks: - hamstring volumes: - - "/home/max/HAMSTRING/hamstring/config.yaml:/app/config.yaml" + - "${HAMSTRING_ROOT:-../..}/config.yaml:/app/config.yaml" environment: - GROUP_ID: data_analysis - NVIDIA_DRIVER_CAPABILITIES: compute,utility - NVIDIA_VISIBLE_DEVICES: all - HAMSTRING_CONFIG_CHECKSUM: "d9bc812b76c2b58c633b412a6c07f981f3c4d787" + GROUP_ID: "${DETECTOR_GROUP_ID:-data_analysis}" + NVIDIA_DRIVER_CAPABILITIES: "${NVIDIA_DRIVER_CAPABILITIES:-compute,utility}" + NVIDIA_VISIBLE_DEVICES: "${NVIDIA_VISIBLE_DEVICES:-all}" + HAMSTRING_CONFIG_CHECKSUM: "${HAMSTRING_CONFIG_CHECKSUM:-d9bc812b76c2b58c633b412a6c07f981f3c4d787}" deploy: mode: replicated - replicas: 1 + replicas: ${DETECTOR_REPLICAS:-1} # Use when nvidia gpu is available to you # resources: # reservations: @@ -344,43 +327,52 @@ services: condition: any alerter: - image: ghcr.io/astraos-de/hamstring-alerter:v2.2.0-dev + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-alerter:${HAMSTRING_IMAGE_TAG:-v2.2.0}" + command: ["src/alerter/alerter.py"] networks: - hamstring volumes: - - "/home/max/HAMSTRING/hamstring/config.yaml:/app/config.yaml" - - "/home/max/HAMSTRING/hamstring/docker/logs:/opt/logs" + - "${HAMSTRING_ROOT:-../..}/config.yaml:/app/config.yaml" + - "${HAMSTRING_ROOT:-../..}/docker/logs:/opt/logs" environment: - GROUP_ID: data_alerting - HAMSTRING_CONFIG_CHECKSUM: "d9bc812b76c2b58c633b412a6c07f981f3c4d787" + GROUP_ID: "${ALERTER_GROUP_ID:-data_alerting}" + HAMSTRING_CONFIG_CHECKSUM: "${HAMSTRING_CONFIG_CHECKSUM:-d9bc812b76c2b58c633b412a6c07f981f3c4d787}" deploy: mode: replicated - replicas: 1 + replicas: ${ALERTER_REPLICAS:-1} restart_policy: condition: any zeek: - image: ghcr.io/astraos-de/hamstring-zeek:v2.2.0-dev + image: "${HAMSTRING_IMAGE_REGISTRY:-ghcr.io/astraos-de}/hamstring-zeek:${HAMSTRING_IMAGE_TAG:-2.0.1}" + cap_add: + - NET_ADMIN networks: - - hamstring + - host volumes: - - "/home/max/HAMSTRING/hamstring/config.yaml:/opt/config.yaml" - - "/home/max/HAMSTRING/hamstring/docker/data/test_pcaps:/opt/static_files" + - "${HAMSTRING_ROOT:-../..}/config.yaml:/opt/config.yaml" + - "${HAMSTRING_ROOT:-../..}/docker/data/test_pcaps:/opt/static_files" environment: - CONTAINER_NAME: zeek - HAMSTRING_CONFIG_CHECKSUM: "d9bc812b76c2b58c633b412a6c07f981f3c4d787" + # Kafka endpoints for host-networked Zeek come from config.yaml: + # environment.kafka_brokers[*].node_ip/external_port. + CONTAINER_NAME: "${ZEEK_CONTAINER_NAME:-zeek}" deploy: + mode: replicated + replicas: ${ZEEK_REPLICAS:-1} restart_policy: condition: any networks: + host: + external: true + name: host hamstring: driver: overlay attachable: true ipam: driver: default config: - - subnet: 172.27.0.0/16 + - subnet: "${HAMSTRING_SUBNET:-172.27.0.0/16}" volumes: ch_data: diff --git a/src/prefilter/prefilter.py b/src/prefilter/prefilter.py index 4abc2fd1..4e3349de 100644 --- a/src/prefilter/prefilter.py +++ b/src/prefilter/prefilter.py @@ -305,7 +305,6 @@ def bootstrap_prefiltering_process(self): 3. Sends the filtered data to inspectors """ - logger.info(f"I am {self.consume_topic}") while True: self.get_and_fill_data() self.check_data_relevance_using_rules() diff --git a/tests/prefilter/test_prefilter.py b/tests/prefilter/test_prefilter.py index 2b648de9..71206544 100644 --- a/tests/prefilter/test_prefilter.py +++ b/tests/prefilter/test_prefilter.py @@ -72,7 +72,6 @@ def mock_send_filtered_data(): # Verify logger calls expected_log_calls = [ - call("I am test_topic"), call( "test_topic Received message:\n ⤷ Contains data field of 2 message(s) with subnet_id: '127.0.0.0_24'." ),