diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/amd64/cmp-agent_install_common b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/amd64/cmp-agent_install_common index 42d699c1..52a4e95f 100644 --- a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/amd64/cmp-agent_install_common +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/amd64/cmp-agent_install_common @@ -4,11 +4,35 @@ mkdir -p /cmp-agent/ +# Another agent's install on this VM may be running podman (e.g. loading its image) while this +# runs, and the log agent's service runs podman too. Extracting straight over the binaries leaves +# them missing or half-written for a moment, so exec fails with "Text file busy". Extract to a +# staging directory on the same filesystem and rename each file into place: rename swaps the +# file in one step, so podman is always either the old or the new complete binary. +STAGE=/cmp-agent/.common-stage_"$1" +rm -rf "$STAGE" +mkdir -p "$STAGE" + echo "[*] Extracting agent common files..." -/tmp/busybox_"$1" tar xvf /tmp/cmp-agent-common_"$1.tar.gz" -C /cmp-agent/ +/tmp/busybox_"$1" tar xvf /tmp/cmp-agent-common_"$1.tar.gz" -C "$STAGE" STATUS=`echo $?` if [ "$STATUS" != 0 ]; then echo "[!] ERROR: Failed to extract agent common files!" + rm -rf "$STAGE" + exit 1; +fi + +echo "[*] Moving agent common files into place..." +(cd "$STAGE" && find . -mindepth 1 -type d) | while read -r d; do + mkdir -p "/cmp-agent/$d" && chmod --reference="$STAGE/$d" "/cmp-agent/$d" +done && +(cd "$STAGE" && find . ! -type d) | while read -r f; do + mv -f "$STAGE/$f" "/cmp-agent/$f" || exit 1 +done +STATUS=`echo $?` +rm -rf "$STAGE" +if [ "$STATUS" != 0 ]; then + echo "[!] ERROR: Failed to move agent common files into place!" exit 1; fi @@ -21,47 +45,76 @@ chcon -Rt bin_t /cmp-agent/common/iptables/bin/ echo "[*] Fixing permission for '/cmp-agent/common/iptables/sbin/'..." chcon -Rt bin_t /cmp-agent/common/iptables/sbin/ +# fluent-bit tails /var/log/syslog, which rsyslog writes. Some CSP images (e.g. minimal +# Ubuntu/Rocky) ship without rsyslog, so install it first. If the package repo is unreachable +# the install fails; skip the rsyslog steps then so the metric agent still installs - the +# fluent-bit playbook checks rsyslog itself and fails the log agent with a clear message. +install_rsyslog() { + if command -v apt-get >/dev/null 2>&1; then + # A freshly created VM may still hold the apt lock (cloud-init, unattended-upgrades). + DEBIAN_FRONTEND=noninteractive timeout 900 apt-get -o DPkg::Lock::Timeout=300 update -q < /dev/null && + DEBIAN_FRONTEND=noninteractive timeout 900 apt-get -o DPkg::Lock::Timeout=300 install -y -q rsyslog < /dev/null + elif command -v dnf >/dev/null 2>&1; then + timeout 900 dnf install -y -q rsyslog < /dev/null + elif command -v yum >/dev/null 2>&1; then + timeout 900 yum install -y -q rsyslog < /dev/null + elif command -v zypper >/dev/null 2>&1; then + timeout 900 zypper --non-interactive install rsyslog < /dev/null + else + echo "[!] No supported package manager found (apt-get/dnf/yum/zypper)." + return 1 + fi +} + echo "[*] Getting rsyslog service configuration path..." +# Without rsyslog, systemctl show prints an empty FragmentPath and still exits 0, so check the +# value itself. Running the steps below with an empty path makes grep wait on stdin forever. RSYSLOG_PATH=`systemctl show rsyslog -p FragmentPath | cut -d= -f2` -STATUS=`echo $?` -if [ "$STATUS" != 0 ]; then - echo "[!] ERROR: Failed to get rsyslog service configuration path." - exit 1; +if [ -z "$RSYSLOG_PATH" ]; then + echo "[*] rsyslog service not found - installing rsyslog..." + if install_rsyslog; then + systemctl enable --now rsyslog + RSYSLOG_PATH=`systemctl show rsyslog -p FragmentPath | cut -d= -f2` + fi fi -echo "[*] Removing existing TZ environment variable if exists in rsyslog.service..." -TZ_EXISTS=`grep -c "Environment=TZ" $RSYSLOG_PATH` -if [ "$TZ_EXISTS" != "" ]; then - sed -i '/^Environment=TZ.*/d' $RSYSLOG_PATH | true -fi -STATUS=`echo $?` -if [ "$STATUS" != 0 ]; then - echo "[!] ERROR: Failed to remove existing TZ environment variable if exists in rsyslog.service." - exit 1; -fi +if [ -z "$RSYSLOG_PATH" ]; then + echo "[!] rsyslog install failed - skipping rsyslog TZ configuration. Host logs will not be collected." +else + echo "[*] Removing existing TZ environment variable if exists in rsyslog.service..." + TZ_EXISTS=`grep -c "Environment=TZ" "$RSYSLOG_PATH"` + if [ "$TZ_EXISTS" != "" ]; then + sed -i '/^Environment=TZ.*/d' "$RSYSLOG_PATH" | true + fi + STATUS=`echo $?` + if [ "$STATUS" != 0 ]; then + echo "[!] ERROR: Failed to remove existing TZ environment variable if exists in rsyslog.service." + exit 1; + fi -echo "[*] Adding new TZ environment variable to rsyslog.service..." -sed -i '/^\[Service\]/a Environment=TZ=Asia/Seoul' $RSYSLOG_PATH -STATUS=`echo $?` -if [ "$STATUS" != 0 ]; then - echo "[!] ERROR: Failed to add new TZ environment variable to rsyslog.service." - exit 1; -fi + echo "[*] Adding new TZ environment variable to rsyslog.service..." + sed -i '/^\[Service\]/a Environment=TZ=Asia/Seoul' "$RSYSLOG_PATH" + STATUS=`echo $?` + if [ "$STATUS" != 0 ]; then + echo "[!] ERROR: Failed to add new TZ environment variable to rsyslog.service." + exit 1; + fi -echo "[*] Reloading systemctl daemon..." -systemctl daemon-reload -STATUS=`echo $?` -if [ "$STATUS" != 0 ]; then - echo "[!] ERROR: Failed to reload systemctl daemon." - exit 1; -fi + echo "[*] Reloading systemctl daemon..." + systemctl daemon-reload + STATUS=`echo $?` + if [ "$STATUS" != 0 ]; then + echo "[!] ERROR: Failed to reload systemctl daemon." + exit 1; + fi -echo "[*] Restarting rsyslog service..." -systemctl restart rsyslog -STATUS=`echo $?` -if [ "$STATUS" != 0 ]; then - echo "[!] ERROR: Failed to restart rsyslog service." - exit 1; + echo "[*] Restarting rsyslog service..." + systemctl restart rsyslog + STATUS=`echo $?` + if [ "$STATUS" != 0 ]; then + echo "[!] ERROR: Failed to restart rsyslog service." + exit 1; + fi fi echo "[*] Creating syslog symbolic link file for RHEL based OS..." diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/arm64/cmp-agent_install_common b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/arm64/cmp-agent_install_common index 42d699c1..52a4e95f 100644 --- a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/arm64/cmp-agent_install_common +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/agent-install-files/arm64/cmp-agent_install_common @@ -4,11 +4,35 @@ mkdir -p /cmp-agent/ +# Another agent's install on this VM may be running podman (e.g. loading its image) while this +# runs, and the log agent's service runs podman too. Extracting straight over the binaries leaves +# them missing or half-written for a moment, so exec fails with "Text file busy". Extract to a +# staging directory on the same filesystem and rename each file into place: rename swaps the +# file in one step, so podman is always either the old or the new complete binary. +STAGE=/cmp-agent/.common-stage_"$1" +rm -rf "$STAGE" +mkdir -p "$STAGE" + echo "[*] Extracting agent common files..." -/tmp/busybox_"$1" tar xvf /tmp/cmp-agent-common_"$1.tar.gz" -C /cmp-agent/ +/tmp/busybox_"$1" tar xvf /tmp/cmp-agent-common_"$1.tar.gz" -C "$STAGE" STATUS=`echo $?` if [ "$STATUS" != 0 ]; then echo "[!] ERROR: Failed to extract agent common files!" + rm -rf "$STAGE" + exit 1; +fi + +echo "[*] Moving agent common files into place..." +(cd "$STAGE" && find . -mindepth 1 -type d) | while read -r d; do + mkdir -p "/cmp-agent/$d" && chmod --reference="$STAGE/$d" "/cmp-agent/$d" +done && +(cd "$STAGE" && find . ! -type d) | while read -r f; do + mv -f "$STAGE/$f" "/cmp-agent/$f" || exit 1 +done +STATUS=`echo $?` +rm -rf "$STAGE" +if [ "$STATUS" != 0 ]; then + echo "[!] ERROR: Failed to move agent common files into place!" exit 1; fi @@ -21,47 +45,76 @@ chcon -Rt bin_t /cmp-agent/common/iptables/bin/ echo "[*] Fixing permission for '/cmp-agent/common/iptables/sbin/'..." chcon -Rt bin_t /cmp-agent/common/iptables/sbin/ +# fluent-bit tails /var/log/syslog, which rsyslog writes. Some CSP images (e.g. minimal +# Ubuntu/Rocky) ship without rsyslog, so install it first. If the package repo is unreachable +# the install fails; skip the rsyslog steps then so the metric agent still installs - the +# fluent-bit playbook checks rsyslog itself and fails the log agent with a clear message. +install_rsyslog() { + if command -v apt-get >/dev/null 2>&1; then + # A freshly created VM may still hold the apt lock (cloud-init, unattended-upgrades). + DEBIAN_FRONTEND=noninteractive timeout 900 apt-get -o DPkg::Lock::Timeout=300 update -q < /dev/null && + DEBIAN_FRONTEND=noninteractive timeout 900 apt-get -o DPkg::Lock::Timeout=300 install -y -q rsyslog < /dev/null + elif command -v dnf >/dev/null 2>&1; then + timeout 900 dnf install -y -q rsyslog < /dev/null + elif command -v yum >/dev/null 2>&1; then + timeout 900 yum install -y -q rsyslog < /dev/null + elif command -v zypper >/dev/null 2>&1; then + timeout 900 zypper --non-interactive install rsyslog < /dev/null + else + echo "[!] No supported package manager found (apt-get/dnf/yum/zypper)." + return 1 + fi +} + echo "[*] Getting rsyslog service configuration path..." +# Without rsyslog, systemctl show prints an empty FragmentPath and still exits 0, so check the +# value itself. Running the steps below with an empty path makes grep wait on stdin forever. RSYSLOG_PATH=`systemctl show rsyslog -p FragmentPath | cut -d= -f2` -STATUS=`echo $?` -if [ "$STATUS" != 0 ]; then - echo "[!] ERROR: Failed to get rsyslog service configuration path." - exit 1; +if [ -z "$RSYSLOG_PATH" ]; then + echo "[*] rsyslog service not found - installing rsyslog..." + if install_rsyslog; then + systemctl enable --now rsyslog + RSYSLOG_PATH=`systemctl show rsyslog -p FragmentPath | cut -d= -f2` + fi fi -echo "[*] Removing existing TZ environment variable if exists in rsyslog.service..." -TZ_EXISTS=`grep -c "Environment=TZ" $RSYSLOG_PATH` -if [ "$TZ_EXISTS" != "" ]; then - sed -i '/^Environment=TZ.*/d' $RSYSLOG_PATH | true -fi -STATUS=`echo $?` -if [ "$STATUS" != 0 ]; then - echo "[!] ERROR: Failed to remove existing TZ environment variable if exists in rsyslog.service." - exit 1; -fi +if [ -z "$RSYSLOG_PATH" ]; then + echo "[!] rsyslog install failed - skipping rsyslog TZ configuration. Host logs will not be collected." +else + echo "[*] Removing existing TZ environment variable if exists in rsyslog.service..." + TZ_EXISTS=`grep -c "Environment=TZ" "$RSYSLOG_PATH"` + if [ "$TZ_EXISTS" != "" ]; then + sed -i '/^Environment=TZ.*/d' "$RSYSLOG_PATH" | true + fi + STATUS=`echo $?` + if [ "$STATUS" != 0 ]; then + echo "[!] ERROR: Failed to remove existing TZ environment variable if exists in rsyslog.service." + exit 1; + fi -echo "[*] Adding new TZ environment variable to rsyslog.service..." -sed -i '/^\[Service\]/a Environment=TZ=Asia/Seoul' $RSYSLOG_PATH -STATUS=`echo $?` -if [ "$STATUS" != 0 ]; then - echo "[!] ERROR: Failed to add new TZ environment variable to rsyslog.service." - exit 1; -fi + echo "[*] Adding new TZ environment variable to rsyslog.service..." + sed -i '/^\[Service\]/a Environment=TZ=Asia/Seoul' "$RSYSLOG_PATH" + STATUS=`echo $?` + if [ "$STATUS" != 0 ]; then + echo "[!] ERROR: Failed to add new TZ environment variable to rsyslog.service." + exit 1; + fi -echo "[*] Reloading systemctl daemon..." -systemctl daemon-reload -STATUS=`echo $?` -if [ "$STATUS" != 0 ]; then - echo "[!] ERROR: Failed to reload systemctl daemon." - exit 1; -fi + echo "[*] Reloading systemctl daemon..." + systemctl daemon-reload + STATUS=`echo $?` + if [ "$STATUS" != 0 ]; then + echo "[!] ERROR: Failed to reload systemctl daemon." + exit 1; + fi -echo "[*] Restarting rsyslog service..." -systemctl restart rsyslog -STATUS=`echo $?` -if [ "$STATUS" != 0 ]; then - echo "[!] ERROR: Failed to restart rsyslog service." - exit 1; + echo "[*] Restarting rsyslog service..." + systemctl restart rsyslog + STATUS=`echo $?` + if [ "$STATUS" != 0 ]; then + echo "[!] ERROR: Failed to restart rsyslog service." + exit 1; + fi fi echo "[*] Creating syslog symbolic link file for RHEL based OS..." diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/tasks/agent_common_begin.yaml b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/tasks/agent_common_begin.yaml index a82118ac..fb2348b5 100644 --- a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/tasks/agent_common_begin.yaml +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/tasks/agent_common_begin.yaml @@ -61,7 +61,9 @@ ansible_ssh_private_key_file: "{{ target_sshkey_path }}" raw: "systemctl status --no-pager rsyslog" register: rsyslog_status - failed_when: rsyslog_status.rc not in [0, 3] + # rc 4 = no rsyslog unit (the install script could not install it). The log agent install + # checks for rsyslog itself; don't fail the metric agent install over it. + failed_when: rsyslog_status.rc not in [0, 3, 4] - name: Display rsyslog status debug: diff --git a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/tasks/agent_fluent-bit_install.yaml b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/tasks/agent_fluent-bit_install.yaml index 070b6dbe..cad34ebf 100644 --- a/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/tasks/agent_fluent-bit_install.yaml +++ b/conf/docker/conf/mc-observability/mc-observability-infra/ansible/playbooks/install-agent/roles/agent/tasks/agent_fluent-bit_install.yaml @@ -8,6 +8,29 @@ - name: Begin Agent Common Tasks include_tasks: agent_common_begin.yaml +# fluent-bit reads host logs from /var/log/syslog, which rsyslog writes. The common install +# script tries to install rsyslog; if that failed (e.g. no access to the package repo), fail +# the log agent here so it shows as FAILED instead of installing and collecting nothing. +- name: Check rsyslog is running for Fluent Bit + become: true + become_method: sudo + become_user: root + vars: + ansible_connection: ssh + ansible_host: "{{ target_host }}" + ansible_port: "{{ target_port }}" + ansible_user: "{{ target_user }}" + ansible_ssh_private_key_file: "{{ target_sshkey_path }}" + raw: "systemctl is-active rsyslog" + register: rsyslog_active + changed_when: false + failed_when: false + +- name: Fail when rsyslog is not running + fail: + msg: "rsyslog is not running on this VM and could not be installed (check that the VM can reach its OS package repository). Fluent Bit collects host logs from /var/log/syslog written by rsyslog." + when: rsyslog_active.stdout | trim != 'active' + - name: Compress agent files raw: "cd {{ role_path }}/agent-install-files/{{ arch }} && sudo rm -f cmp-agent/cmp-fluent-bit_{{ request_id }}.tar.gz && sudo chown -R root:root fluent-bit/ && sudo ../amd64/busybox tar cvfz cmp-agent/cmp-fluent-bit_{{ request_id }}.tar.gz fluent-bit/ && sudo chmod 644 cmp-agent/cmp-fluent-bit_{{ request_id }}.tar.gz" delegate_to: 127.0.0.1 diff --git a/conf/docker/docker-compose.yaml b/conf/docker/docker-compose.yaml index 567b814a..68d129e0 100644 --- a/conf/docker/docker-compose.yaml +++ b/conf/docker/docker-compose.yaml @@ -1254,7 +1254,7 @@ services: <<: *default-health-check ##### MC-OBSERVABILITY ######################################################################################################################### mc-observability-manager: - image: cloudbaristaorg/mc-observability:0.7.0 + image: cloudbaristaorg/mc-observability:0.7.1 container_name: mc-observability-manager restart: unless-stopped networks: @@ -1263,6 +1263,10 @@ services: - mc-infra-manager-network - mc-infra-connector-network - mc-web-console-network + ports: + - target: 18080 + published: ${MC_OBSERVABILITY_MANAGER_PORT} + protocol: tcp depends_on: mc-observability-grafana: condition: service_healthy @@ -1294,7 +1298,7 @@ services: start_period: 5m mc-observability-front: - image: cloudbaristaorg/mc-observability-front:0.7.0 + image: cloudbaristaorg/mc-observability-front:0.7.1 container_name: mc-observability-front restart: unless-stopped networks: @@ -1306,7 +1310,7 @@ services: condition: service_started mc-observability-infra: - image: cloudbaristaorg/mc-observability-infra:0.7.0 + image: cloudbaristaorg/mc-observability-infra:0.7.1 container_name: mc-observability-infra restart: unless-stopped networks: @@ -1431,7 +1435,7 @@ services: init: true mc-observability-influx: - image: cloudbaristaorg/mc-observability-influx:0.7.0 + image: cloudbaristaorg/mc-observability-influx:0.7.1 container_name: mc-observability-influx restart: unless-stopped networks: @@ -1473,7 +1477,7 @@ services: - mc-observability-network mc-observability-influx-2: - image: cloudbaristaorg/mc-observability-influx:0.7.0 + image: cloudbaristaorg/mc-observability-influx:0.7.1 container_name: mc-observability-influx-2 restart: unless-stopped networks: @@ -1649,7 +1653,7 @@ services: init: true mc-observability-grafana: - image: cloudbaristaorg/mc-observability-grafana:0.7.0 + image: cloudbaristaorg/mc-observability-grafana:0.7.1 container_name: mc-observability-grafana restart: unless-stopped networks: @@ -1684,7 +1688,7 @@ services: fi mc-observability-insight: - image: cloudbaristaorg/mc-observability-insight:0.7.0 + image: cloudbaristaorg/mc-observability-insight:0.7.1 container_name: mc-observability-insight restart: always networks: @@ -1713,7 +1717,7 @@ services: start_period: 10s mc-observability-insight-scheduler: - image: cloudbaristaorg/mc-observability-insight-scheduler:0.7.0 + image: cloudbaristaorg/mc-observability-insight-scheduler:0.7.1 container_name: mc-observability-insight-scheduler restart: always ports: @@ -1801,7 +1805,7 @@ services: - mc-observability-network mc-observability-mcp-maria: - image: cloudbaristaorg/mc-observability-mcp-mariadb:0.7.0 + image: cloudbaristaorg/mc-observability-mcp-mariadb:0.7.1 container_name: mc-observability-mcp-maria restart: always networks: @@ -1825,7 +1829,7 @@ services: - OPENAI_API_KEY=${MC_OBSERVABILITY_OPENAI_API_KEY:-} mc-observability-mcp-influx: - image: cloudbaristaorg/mc-observability-mcp-influxdb:0.7.0 + image: cloudbaristaorg/mc-observability-mcp-influxdb:0.7.1 container_name: mc-observability-mcp-influx restart: always ports: