Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions content/validators/alloy-config.river
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,65 @@ loki.source.file "genlayer" {
}

// Write logs to central Loki instance
// Read opted-in WebDriver/Alloy recovery events through the existing read-only
// Docker proxy. Containers log their own restart attempts before exiting.
discovery.docker "health_recovery" {
host = "http://docker-socket-proxy:2375"
refresh_interval = "5s"

filter {
name = "label"
values = [
"com.genlayer.health-recovery.logs=true",
]
}
}

discovery.relabel "health_recovery" {
targets = discovery.docker.health_recovery.targets

rule {
source_labels = ["__meta_docker_container_label_com_docker_compose_project"]
target_label = "compose_project"
}

rule {
source_labels = ["__meta_docker_container_label_com_genlayer_network"]
target_label = "network"
}

rule {
source_labels = ["network"]
regex = ""
target_label = "network"
replacement = "unknown"
}
}

loki.source.docker "health_recovery" {
// Docker's TCP transport keeps the log stream open. With an HTTP(S) host,
// Alloy bounds the entire stream by refresh_interval and repeatedly reopens it.
host = "tcp://docker-socket-proxy:2375"
targets = discovery.relabel.health_recovery.output
refresh_interval = "5s"
labels = {
job = "genlayer-node",
component = "health-recovery",
instance = sys.env("NODE_ID"),
validator_name = sys.env("VALIDATOR_NAME"),
}
forward_to = [loki.process.health_recovery.receiver]
}

// Collect only restart events, not Alloy's own operational log stream.
loki.process "health_recovery" {
stage.match {
selector = "{component=\"health-recovery\"} != \"found to be unhealthy - Restarting container now\""
action = "drop"
}
forward_to = [loki.write.central.receiver]
}

loki.write "central" {
endpoint {
url = sys.env("CENTRAL_LOKI_URL")
Expand Down
31 changes: 31 additions & 0 deletions content/validators/changelog/v0.6.0-rc.2.mdx
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
## v0.6.0-rc.2

### New features

- Check WebDriver rendering in `/health` and `doctor` with the same fresh render of a test page (10 s limit); a failure sets the `webdriver.render` condition to `degraded` with reason `webdriver_check_failed`
- The release compose file restarts a WebDriver that stops rendering (after 4 failed health checks) and an Alloy that stays unhealthy; set `HEALTHCHECK_AUTO_RESTART=false` to keep the health checks and turn off the restarts
- The release monitoring profile sends the restart events of WebDriver and Alloy to central monitoring, with the `NETWORK_NAME` label from `.env`

### Misc

- Docker: mount custom GenVM configs (for example `genvm-module-llm.yaml` for OpenRouter/greybox) at `/app/opt/genvm/current/config/`; v0.6 no longer reads `/app/third_party/genvm/config/`, so a v0.5 `docker-compose.override.yml` mount there is ignored and the image default loads
- `/health` no longer returns `severity` in `conditions`; use `state` for the current health
- The Bradbury example config (`configs/node/bradbury.yaml.example`) uses the GenLayer RPC host `rpc.testnet-chain.genlayer.com` (HTTPS and WSS); RPC keys still come from Conduit
- The release compose file mounts the protocol manifest from `NODE_MANIFEST_PATH` (default `./configs/node/protocol-manifest.json`) at the path that `consensus.manifestpath` names
- The release package links `opt/genvm/current` to its GenVM tree, and its `config.yaml` sets `genvm.root_dir: ./opt/genvm/current`
- The example `.env` lists all 11 LLM provider keys of the release LLM config, and sets the monitoring defaults `METRICS_SCRAPE_INTERVAL=60s` and `LOKI_BATCH_WAIT=60s`
- The release README shows how to run `doctor` under Docker
- Update `tecnativa/docker-socket-proxy` to v0.5.0; the proxy also gives Alloy read-only network data
- Update `github.com/ethereum/go-ethereum` to v1.17.8, `labstack/echo` to v4.16.0, `golang.org/x/crypto` to v0.58.0 and `golang.org/x/net` to v0.61.0

### Bug fixes

- The release compose file no longer requires `NODE_PASSWORD` for every service: `docker compose up -d webdriver-container` (the binary install) and a full node work with it empty, and a validator still refuses to start without it
- The release compose file has a fixed project name (`genlayer-node`), so starting the stack from a new release directory replaces the old one's containers instead of failing with `container name ... is already in use`
- `install.sh` keeps `genvm.root_dir: ./opt/genvm/current` in a package that has that link, as the docs say
- A synced node no longer logs a false `ERR operator condition requires attention` for `chain.sync` at start: the condition stays unknown until the sync loop has measured the lag
- Fix web access from contracts under the release compose file: the GenVM web config is mounted at the path GenVM reads, so GenVM uses the WebDriver container instead of `localhost`
- The release compose file no longer replaces the image entrypoint, so the first start precompiles GenVM for the host CPU (the first start takes longer)
- The example config sets `genvm.root_dir` to the shipped GenVM (`./opt/genvm/v0.6.0-rc10`), not `v0.6.0-rc1`
- Update the GenVM web config of the release to the schema and keys of the shipped GenVM
- Correct the API reference: `LeaderRevealing` is status `13`, and the vote values include `5` (`DeterministicViolation`)
6 changes: 3 additions & 3 deletions content/validators/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ rollup:
consensus:
# The deployment-specific fields are empty: take them from your network's documentation.
consensusaddress: "" # AddressManager contract address for this deployment.
manifestpath: "" # Trusted producer protocol-manifest.json for this deployment. The deploy places it at configs/node/protocol-manifest.json; relative paths resolve against the node root. A mismatch with consensusaddress is refused at startup.
manifestpath: "configs/node/protocol-manifest.json" # Trusted producer protocol manifest for this deployment: save your network's manifest (from its documentation) at this path; a dev-env deploy places its own here too. Relative paths resolve against the node root. A mismatch with consensusaddress is refused at startup.
chainid: "" # DECIMAL chain id this deployment was made on (not hex, no leading zeros). Held independently of the manifest so a manifest describing another deployment is caught.
# genesis: 17326 # (Optional) Genesis block number for this consensus deployment. If not provided, it will be auto-detected by searching for the first log from the ConsensusMain contract.
# genesis: 47930 # (Optional) Genesis block number for this consensus deployment. If not provided, it will be auto-detected by searching for the first log from the ConsensusMain contract.
# chain_submit_buffer: "30s" # (Optional) Slack reserved at the end of each consensus step to submit the resulting on-chain tx (commit vote, reveal vote, proposed receipt) before the slot closes. Tune to underlying settlement chain latency. Default 30s.
# admissiontimeout: "15s" # (Optional) Bound on the startup deployment admission check. Never unbounded. Default 15s.
# data directory
Expand Down Expand Up @@ -195,7 +195,7 @@ node:
# max_debug_field_bytes: 262144 # 256 KB per field
# genvm configuration
genvm:
root_dir: ./opt/genvm/v0.6.0-rc1
root_dir: ./opt/genvm/v0.6.0-rc10
start_manager: true # if true node will start genvm manager itself
manager_url: http://127.0.0.1:3999
# Permits are charged PER EXECUTION by the genvm manager, and one
Expand Down
113 changes: 90 additions & 23 deletions content/validators/docker-compose-monitoring.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
# A fixed project name: the container names below are fixed too, so a stack
# started from a new release directory must be the SAME Compose project, or
# Docker refuses with "container name ... is already in use". With it, `up -d`
# from the new version's directory recreates the old version's containers.
name: genlayer-node

services:
webdriver-container:
container_name: genlayer-node-webdriver
Expand All @@ -7,25 +13,55 @@ services:
- no-new-privileges:true
environment:
PORT: 4444
HEALTHCHECK_AUTO_RESTART: "${HEALTHCHECK_AUTO_RESTART:-true}"
ports:
- "${WEBDRIVER_PORT:-4444}:4444"
restart: unless-stopped
labels:
com.genlayer.health-recovery.logs: "true"
com.genlayer.network: "${NETWORK_NAME:-unknown}"
healthcheck:
test:
- CMD-SHELL
- |
set -eu
boot=$$(awk '{print $$22}' /proc/1/stat)
if curl -fsS --max-time 10 "http://localhost:$${PORT:-4444}/healthcheck?mode=text&url=https%3A%2F%2Ftest-server.genlayer.com%2Fstatic%2Fgenvm%2Fhello.html"; then
printf '%s 0\n' "$$boot" > /tmp/webdriver-health-failures
exit 0
fi
age=$$(awk -v ticks="$$boot" -v hz="$$(getconf CLK_TCK)" '{print int($$1-ticks/hz)}' /proc/uptime)
[ "$$age" -ge 10 ] || exit 1
read -r previous failures 2>/dev/null < /tmp/webdriver-health-failures || failures=0
[ "$${previous:-}" = "$$boot" ] || failures=0
failures=$$((failures + 1))
printf '%s %s\n' "$$boot" "$$failures" > /tmp/webdriver-health-failures
if [ "$$failures" -ge 4 ] && [ "$${HEALTHCHECK_AUTO_RESTART:-true}" != false ]; then
printf 'Container webdriver (%s) found to be unhealthy - Restarting container now\n' "$$(cat /etc/hostname)" > /proc/1/fd/1
children=$$(cat /proc/1/task/1/children)
for child in $$children; do kill -KILL "$$child"; done
fi
exit 1
interval: 30s
timeout: 12s
retries: 4
start_period: 10s

# docker-socket-proxy β€” hardened, read-only Docker API gateway used ONLY so the
# node can read the WebDriver container's resource stats (CPU/memory/traffic) for
# metrics. Runs with the `node` OR `monitoring` profile, so it comes up alongside
# docker-socket-proxy β€” read-only Docker API gateway for the node's WebDriver
# resource stats and Alloy's recovery-log reader. Runs with the `node` OR
# `monitoring` profile, so it comes up alongside
# a node container AND alongside a monitoring/telemetry bring-up β€” the latter is
# how a binary/systemd node (node not containerized) gets its webdriver metrics.
#
# WHY THIS EXISTS β€” and why it is NOT a new exposure or a "leak":
# The node and Alloy use this read-only gateway instead of the host socket.
# Reading container stats requires the Docker API. Previously the node reached
# the host /var/run/docker.sock directly (bind-mounted into the container, or via
# docker-group membership for a binary node) β€” but that grants the FULL Docker
# API as root (a `:ro` flag does NOT restrict API calls), so a node compromise
# could escalate to host root. This proxy REPLACES that: it is the only thing
# touching the socket, restricted to read-only container endpoints (CONTAINERS=1;
# POST=0 => it returns 403 for create / exec / images / everything else). Net
# exposure goes DOWN, not up.
# could escalate to host root. This proxy REPLACES the node's socket access,
# restricted to read-only container endpoints (CONTAINERS=1;
# POST=0 => it returns 403 for create / exec / image operations).
# Health checks stop only their own service process; recovery has no Docker API access.
# - Published ONLY on 127.0.0.1:2375 (host loopback), never 0.0.0.0: 2375 is
# the unauthenticated Docker API, so it must not leave the host. A container
# node reaches it over the compose network (tcp://docker-socket-proxy:2375);
Expand All @@ -36,14 +72,15 @@ services:
# socket: if DOCKER_HOST is unset the node skips webdriver metrics entirely
# (non-fatal). Note those are the webdriver container's CPU/memory/traffic series
# (genlayer_node_*{component="webdriver"}) that feed validator resource dashboards β€”
# so keep DOCKER_HOST set to keep those panels populated. To disable metrics
# instead: set metrics.collectors.webdriver.enabled: false and delete this service.
# so keep DOCKER_HOST set to keep those panels populated. Disabling metrics
# permits removing the node's DOCKER_HOST; Alloy still needs this gateway.
docker-socket-proxy:
image: tecnativa/docker-socket-proxy:0.3.0
image: tecnativa/docker-socket-proxy:v0.5.0
container_name: genlayer-node-docker-proxy
environment:
CONTAINERS: 1 # GET /containers/{json,inspect,stats} β€” read-only via POST=0
VERSION: 1 # required by the Docker client's API-version negotiation
NETWORKS: 1 # read-only network metadata required by Alloy Docker discovery
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
ports:
Expand All @@ -66,23 +103,27 @@ services:

genlayer-node:
image: yeagerai/genlayer-node:${NODE_VERSION:-latest}
# The keystore password is NOT on this command line. It reaches the node as
# No entrypoint here: the image's own entrypoint precompiles GenVM for the
# host CPU on the first start, then runs `genlayernode run`. The keystore
# password is NOT on a command line. It reaches the node as
# GENLAYERNODE_PASSWORD (environment below), which the run command reads in
# place of --password. A command-line argument is visible in process
# listings on the host; an environment variable is not.
entrypoint: ["/app/bin/genlayernode", "run"]
container_name: genlayer-node
restart: unless-stopped
env_file:
- path: ./.env
required: false
environment:
# ":?" makes Compose refuse to start when NODE_PASSWORD is unset or empty,
# before any container runs. There is no fallback: this is the password
# that encrypts the validator's signing key, and a default shared by every
# operator who forgot to set one is a key anyone can open. The release
# binary carries no default either β€” it is compiled out of production.
GENLAYERNODE_PASSWORD: "${NODE_PASSWORD:?NODE_PASSWORD is not set. It is the keystore password that unlocks the validator key. Set it in .env; see README section 4.}"
# No default: this is the password that encrypts the validator's signing
# key, and a default shared by every operator who forgot to set one is a
# key anyone can open (the release binary carries none either). The node
# itself refuses validator mode with an empty password ("password is
# required for validator mode"); a full node needs none. Not a ":?" guard:
# Compose interpolates every service, also under another profile, so a
# guard here would stop `docker compose up -d webdriver-container` (the
# binary install path) and a full node, which never read it.
GENLAYERNODE_PASSWORD: "${NODE_PASSWORD:-}"
# WebDriver metrics collector reaches Docker via the read-only proxy
# (see docker-socket-proxy above), never the host socket directly.
DOCKER_HOST: tcp://docker-socket-proxy:2375
Expand All @@ -91,8 +132,11 @@ services:
- "${NODE_OPS_PORT:-9153}:9153"
volumes:
- ${NODE_CONFIG_PATH:-./configs/node/config.yaml}:/app/configs/node/config.yaml:ro
# The trusted protocol manifest of the network; consensus.manifestpath names this path.
- ${NODE_MANIFEST_PATH:-./configs/node/protocol-manifest.json}:/app/configs/node/protocol-manifest.json:ro
- ${NODE_DATA_PATH:-./data}:/app/data
- ./genvm-module-web-docker.yaml:/app/opt/genvm/${GENVM_VERSION}/config/genvm-module-web.yaml
# opt/genvm/current links to the image's GenVM tree, so no version variable is needed.
- ./genvm-module-web-docker.yaml:/app/opt/genvm/current/config/genvm-module-web.yaml
security_opt:
- no-new-privileges:true
cap_drop:
Expand Down Expand Up @@ -121,14 +165,20 @@ services:
# docker compose --profile monitoring up -d
alloy:
image: grafana/alloy:v1.20.1
init: true
container_name: genlayer-node-alloy
labels:
com.genlayer.health-recovery.logs: "true"
com.genlayer.network: "${NETWORK_NAME:-unknown}"
depends_on:
- docker-socket-proxy
# The guard runs before Alloy: with the monitoring profile up but the
# endpoints or credentials unset, the container stops with a message
# naming what to set, instead of half-running. Half-running is worse than
# stopping here in both directions: pushing to a shared endpoint with a
# shared default credential pollutes it for everyone, and pushing to an
# unreachable one grows the WAL until the healthcheck below reads the
# growth as a stall and deletes it. The $$ keeps compose from
# unreachable one leaves telemetry unavailable and can grow the WAL. The
# healthcheck preserves that buffer. The $$ keeps compose from
# interpolating; the shell only tests emptiness β€” no value ever lands on
# a command line.
entrypoint: ["/bin/sh", "-c"]
Expand All @@ -155,12 +205,29 @@ services:
- ${NODE_LOGS_PATH:-./data/node/logs}:/var/log/genlayer:ro
- alloy_data:/var/lib/alloy
healthcheck:
test: ["CMD", "sh", "/etc/alloy/healthcheck.sh"]
test:
- CMD-SHELL
- |
set -eu
sh /etc/alloy/healthcheck.sh && exit 0
[ "$${HEALTHCHECK_AUTO_RESTART:-true}" != false ] || exit 1
[ "$$(cat /proc/1/comm)" = docker-init ] || exit 1
boot=$$(awk '{print $$22}' /proc/1/stat)
age=$$(awk -v ticks="$$boot" -v hz="$$(getconf CLK_TCK)" '{print int($$1-ticks/hz)}' /proc/uptime)
[ "$$age" -ge 120 ] || exit 1
children=$$(cat /proc/1/task/1/children)
printf 'Container alloy (%s) found to be unhealthy - Restarting container now\n' "$$(cat /etc/hostname)" > /proc/1/fd/1
kill -TERM 1
sleep 5
# A stopped service cannot handle TERM; init's child can still receive KILL.
for child in $$children; do kill -KILL "$$child"; done
exit 1
interval: 120s
timeout: 15s
retries: 1
start_period: 120s
environment:
- HEALTHCHECK_AUTO_RESTART=${HEALTHCHECK_AUTO_RESTART:-true}
# Central monitoring endpoints and credentials: NO defaults, on purpose.
# An endpoint may be public knowledge; a credential never is, and a
# default one ships the same key to everyone holding this file. The
Expand Down
Loading
Loading