diff --git a/content/validators/alloy-config.river b/content/validators/alloy-config.river index db983230..405c203b 100644 --- a/content/validators/alloy-config.river +++ b/content/validators/alloy-config.river @@ -55,6 +55,65 @@ loki.source.file "genlayer" { } // Write logs to central Loki instance +// Read opted-in WebDriver/Alloy recovery events through the existing read-only +// Docker proxy. Containers log their own restart attempts before exiting. +discovery.docker "health_recovery" { + host = "http://docker-socket-proxy:2375" + refresh_interval = "5s" + + filter { + name = "label" + values = [ + "com.genlayer.health-recovery.logs=true", + ] + } +} + +discovery.relabel "health_recovery" { + targets = discovery.docker.health_recovery.targets + + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_project"] + target_label = "compose_project" + } + + rule { + source_labels = ["__meta_docker_container_label_com_genlayer_network"] + target_label = "network" + } + + rule { + source_labels = ["network"] + regex = "" + target_label = "network" + replacement = "unknown" + } +} + +loki.source.docker "health_recovery" { + // Docker's TCP transport keeps the log stream open. With an HTTP(S) host, + // Alloy bounds the entire stream by refresh_interval and repeatedly reopens it. + host = "tcp://docker-socket-proxy:2375" + targets = discovery.relabel.health_recovery.output + refresh_interval = "5s" + labels = { + job = "genlayer-node", + component = "health-recovery", + instance = sys.env("NODE_ID"), + validator_name = sys.env("VALIDATOR_NAME"), + } + forward_to = [loki.process.health_recovery.receiver] +} + +// Collect only restart events, not Alloy's own operational log stream. +loki.process "health_recovery" { + stage.match { + selector = "{component=\"health-recovery\"} != \"found to be unhealthy - Restarting container now\"" + action = "drop" + } + forward_to = [loki.write.central.receiver] +} + loki.write "central" { endpoint { url = sys.env("CENTRAL_LOKI_URL") diff --git a/content/validators/changelog/v0.6.0-rc.2.mdx b/content/validators/changelog/v0.6.0-rc.2.mdx new file mode 100644 index 00000000..9cb39afd --- /dev/null +++ b/content/validators/changelog/v0.6.0-rc.2.mdx @@ -0,0 +1,31 @@ +## v0.6.0-rc.2 + +### New features + +- Check WebDriver rendering in `/health` and `doctor` with the same fresh render of a test page (10 s limit); a failure sets the `webdriver.render` condition to `degraded` with reason `webdriver_check_failed` +- The release compose file restarts a WebDriver that stops rendering (after 4 failed health checks) and an Alloy that stays unhealthy; set `HEALTHCHECK_AUTO_RESTART=false` to keep the health checks and turn off the restarts +- The release monitoring profile sends the restart events of WebDriver and Alloy to central monitoring, with the `NETWORK_NAME` label from `.env` + +### Misc + +- Docker: mount custom GenVM configs (for example `genvm-module-llm.yaml` for OpenRouter/greybox) at `/app/opt/genvm/current/config/`; v0.6 no longer reads `/app/third_party/genvm/config/`, so a v0.5 `docker-compose.override.yml` mount there is ignored and the image default loads +- `/health` no longer returns `severity` in `conditions`; use `state` for the current health +- The Bradbury example config (`configs/node/bradbury.yaml.example`) uses the GenLayer RPC host `rpc.testnet-chain.genlayer.com` (HTTPS and WSS); RPC keys still come from Conduit +- The release compose file mounts the protocol manifest from `NODE_MANIFEST_PATH` (default `./configs/node/protocol-manifest.json`) at the path that `consensus.manifestpath` names +- The release package links `opt/genvm/current` to its GenVM tree, and its `config.yaml` sets `genvm.root_dir: ./opt/genvm/current` +- The example `.env` lists all 11 LLM provider keys of the release LLM config, and sets the monitoring defaults `METRICS_SCRAPE_INTERVAL=60s` and `LOKI_BATCH_WAIT=60s` +- The release README shows how to run `doctor` under Docker +- Update `tecnativa/docker-socket-proxy` to v0.5.0; the proxy also gives Alloy read-only network data +- Update `github.com/ethereum/go-ethereum` to v1.17.8, `labstack/echo` to v4.16.0, `golang.org/x/crypto` to v0.58.0 and `golang.org/x/net` to v0.61.0 + +### Bug fixes + +- The release compose file no longer requires `NODE_PASSWORD` for every service: `docker compose up -d webdriver-container` (the binary install) and a full node work with it empty, and a validator still refuses to start without it +- The release compose file has a fixed project name (`genlayer-node`), so starting the stack from a new release directory replaces the old one's containers instead of failing with `container name ... is already in use` +- `install.sh` keeps `genvm.root_dir: ./opt/genvm/current` in a package that has that link, as the docs say +- A synced node no longer logs a false `ERR operator condition requires attention` for `chain.sync` at start: the condition stays unknown until the sync loop has measured the lag +- Fix web access from contracts under the release compose file: the GenVM web config is mounted at the path GenVM reads, so GenVM uses the WebDriver container instead of `localhost` +- The release compose file no longer replaces the image entrypoint, so the first start precompiles GenVM for the host CPU (the first start takes longer) +- The example config sets `genvm.root_dir` to the shipped GenVM (`./opt/genvm/v0.6.0-rc10`), not `v0.6.0-rc1` +- Update the GenVM web config of the release to the schema and keys of the shipped GenVM +- Correct the API reference: `LeaderRevealing` is status `13`, and the vote values include `5` (`DeterministicViolation`) diff --git a/content/validators/config.yaml b/content/validators/config.yaml index b8c42475..0ff0edec 100644 --- a/content/validators/config.yaml +++ b/content/validators/config.yaml @@ -11,9 +11,9 @@ rollup: consensus: # The deployment-specific fields are empty: take them from your network's documentation. consensusaddress: "" # AddressManager contract address for this deployment. - manifestpath: "" # Trusted producer protocol-manifest.json for this deployment. The deploy places it at configs/node/protocol-manifest.json; relative paths resolve against the node root. A mismatch with consensusaddress is refused at startup. + manifestpath: "configs/node/protocol-manifest.json" # Trusted producer protocol manifest for this deployment: save your network's manifest (from its documentation) at this path; a dev-env deploy places its own here too. Relative paths resolve against the node root. A mismatch with consensusaddress is refused at startup. chainid: "" # DECIMAL chain id this deployment was made on (not hex, no leading zeros). Held independently of the manifest so a manifest describing another deployment is caught. - # genesis: 17326 # (Optional) Genesis block number for this consensus deployment. If not provided, it will be auto-detected by searching for the first log from the ConsensusMain contract. + # genesis: 47930 # (Optional) Genesis block number for this consensus deployment. If not provided, it will be auto-detected by searching for the first log from the ConsensusMain contract. # chain_submit_buffer: "30s" # (Optional) Slack reserved at the end of each consensus step to submit the resulting on-chain tx (commit vote, reveal vote, proposed receipt) before the slot closes. Tune to underlying settlement chain latency. Default 30s. # admissiontimeout: "15s" # (Optional) Bound on the startup deployment admission check. Never unbounded. Default 15s. # data directory @@ -195,7 +195,7 @@ node: # max_debug_field_bytes: 262144 # 256 KB per field # genvm configuration genvm: - root_dir: ./opt/genvm/v0.6.0-rc1 + root_dir: ./opt/genvm/v0.6.0-rc10 start_manager: true # if true node will start genvm manager itself manager_url: http://127.0.0.1:3999 # Permits are charged PER EXECUTION by the genvm manager, and one diff --git a/content/validators/docker-compose-monitoring.yaml b/content/validators/docker-compose-monitoring.yaml index 0015e84a..89ed4db9 100644 --- a/content/validators/docker-compose-monitoring.yaml +++ b/content/validators/docker-compose-monitoring.yaml @@ -1,3 +1,9 @@ +# A fixed project name: the container names below are fixed too, so a stack +# started from a new release directory must be the SAME Compose project, or +# Docker refuses with "container name ... is already in use". With it, `up -d` +# from the new version's directory recreates the old version's containers. +name: genlayer-node + services: webdriver-container: container_name: genlayer-node-webdriver @@ -7,25 +13,55 @@ services: - no-new-privileges:true environment: PORT: 4444 + HEALTHCHECK_AUTO_RESTART: "${HEALTHCHECK_AUTO_RESTART:-true}" ports: - "${WEBDRIVER_PORT:-4444}:4444" restart: unless-stopped + labels: + com.genlayer.health-recovery.logs: "true" + com.genlayer.network: "${NETWORK_NAME:-unknown}" + healthcheck: + test: + - CMD-SHELL + - | + set -eu + boot=$$(awk '{print $$22}' /proc/1/stat) + if curl -fsS --max-time 10 "http://localhost:$${PORT:-4444}/healthcheck?mode=text&url=https%3A%2F%2Ftest-server.genlayer.com%2Fstatic%2Fgenvm%2Fhello.html"; then + printf '%s 0\n' "$$boot" > /tmp/webdriver-health-failures + exit 0 + fi + age=$$(awk -v ticks="$$boot" -v hz="$$(getconf CLK_TCK)" '{print int($$1-ticks/hz)}' /proc/uptime) + [ "$$age" -ge 10 ] || exit 1 + read -r previous failures 2>/dev/null < /tmp/webdriver-health-failures || failures=0 + [ "$${previous:-}" = "$$boot" ] || failures=0 + failures=$$((failures + 1)) + printf '%s %s\n' "$$boot" "$$failures" > /tmp/webdriver-health-failures + if [ "$$failures" -ge 4 ] && [ "$${HEALTHCHECK_AUTO_RESTART:-true}" != false ]; then + printf 'Container webdriver (%s) found to be unhealthy - Restarting container now\n' "$$(cat /etc/hostname)" > /proc/1/fd/1 + children=$$(cat /proc/1/task/1/children) + for child in $$children; do kill -KILL "$$child"; done + fi + exit 1 + interval: 30s + timeout: 12s + retries: 4 + start_period: 10s - # docker-socket-proxy — hardened, read-only Docker API gateway used ONLY so the - # node can read the WebDriver container's resource stats (CPU/memory/traffic) for - # metrics. Runs with the `node` OR `monitoring` profile, so it comes up alongside + # docker-socket-proxy — read-only Docker API gateway for the node's WebDriver + # resource stats and Alloy's recovery-log reader. Runs with the `node` OR + # `monitoring` profile, so it comes up alongside # a node container AND alongside a monitoring/telemetry bring-up — the latter is # how a binary/systemd node (node not containerized) gets its webdriver metrics. # - # WHY THIS EXISTS — and why it is NOT a new exposure or a "leak": + # The node and Alloy use this read-only gateway instead of the host socket. # Reading container stats requires the Docker API. Previously the node reached # the host /var/run/docker.sock directly (bind-mounted into the container, or via # docker-group membership for a binary node) — but that grants the FULL Docker # API as root (a `:ro` flag does NOT restrict API calls), so a node compromise - # could escalate to host root. This proxy REPLACES that: it is the only thing - # touching the socket, restricted to read-only container endpoints (CONTAINERS=1; - # POST=0 => it returns 403 for create / exec / images / everything else). Net - # exposure goes DOWN, not up. + # could escalate to host root. This proxy REPLACES the node's socket access, + # restricted to read-only container endpoints (CONTAINERS=1; + # POST=0 => it returns 403 for create / exec / image operations). + # Health checks stop only their own service process; recovery has no Docker API access. # - Published ONLY on 127.0.0.1:2375 (host loopback), never 0.0.0.0: 2375 is # the unauthenticated Docker API, so it must not leave the host. A container # node reaches it over the compose network (tcp://docker-socket-proxy:2375); @@ -36,14 +72,15 @@ services: # socket: if DOCKER_HOST is unset the node skips webdriver metrics entirely # (non-fatal). Note those are the webdriver container's CPU/memory/traffic series # (genlayer_node_*{component="webdriver"}) that feed validator resource dashboards — - # so keep DOCKER_HOST set to keep those panels populated. To disable metrics - # instead: set metrics.collectors.webdriver.enabled: false and delete this service. + # so keep DOCKER_HOST set to keep those panels populated. Disabling metrics + # permits removing the node's DOCKER_HOST; Alloy still needs this gateway. docker-socket-proxy: - image: tecnativa/docker-socket-proxy:0.3.0 + image: tecnativa/docker-socket-proxy:v0.5.0 container_name: genlayer-node-docker-proxy environment: CONTAINERS: 1 # GET /containers/{json,inspect,stats} — read-only via POST=0 VERSION: 1 # required by the Docker client's API-version negotiation + NETWORKS: 1 # read-only network metadata required by Alloy Docker discovery volumes: - /var/run/docker.sock:/var/run/docker.sock:ro ports: @@ -66,23 +103,27 @@ services: genlayer-node: image: yeagerai/genlayer-node:${NODE_VERSION:-latest} - # The keystore password is NOT on this command line. It reaches the node as + # No entrypoint here: the image's own entrypoint precompiles GenVM for the + # host CPU on the first start, then runs `genlayernode run`. The keystore + # password is NOT on a command line. It reaches the node as # GENLAYERNODE_PASSWORD (environment below), which the run command reads in # place of --password. A command-line argument is visible in process # listings on the host; an environment variable is not. - entrypoint: ["/app/bin/genlayernode", "run"] container_name: genlayer-node restart: unless-stopped env_file: - path: ./.env required: false environment: - # ":?" makes Compose refuse to start when NODE_PASSWORD is unset or empty, - # before any container runs. There is no fallback: this is the password - # that encrypts the validator's signing key, and a default shared by every - # operator who forgot to set one is a key anyone can open. The release - # binary carries no default either — it is compiled out of production. - GENLAYERNODE_PASSWORD: "${NODE_PASSWORD:?NODE_PASSWORD is not set. It is the keystore password that unlocks the validator key. Set it in .env; see README section 4.}" + # No default: this is the password that encrypts the validator's signing + # key, and a default shared by every operator who forgot to set one is a + # key anyone can open (the release binary carries none either). The node + # itself refuses validator mode with an empty password ("password is + # required for validator mode"); a full node needs none. Not a ":?" guard: + # Compose interpolates every service, also under another profile, so a + # guard here would stop `docker compose up -d webdriver-container` (the + # binary install path) and a full node, which never read it. + GENLAYERNODE_PASSWORD: "${NODE_PASSWORD:-}" # WebDriver metrics collector reaches Docker via the read-only proxy # (see docker-socket-proxy above), never the host socket directly. DOCKER_HOST: tcp://docker-socket-proxy:2375 @@ -91,8 +132,11 @@ services: - "${NODE_OPS_PORT:-9153}:9153" volumes: - ${NODE_CONFIG_PATH:-./configs/node/config.yaml}:/app/configs/node/config.yaml:ro + # The trusted protocol manifest of the network; consensus.manifestpath names this path. + - ${NODE_MANIFEST_PATH:-./configs/node/protocol-manifest.json}:/app/configs/node/protocol-manifest.json:ro - ${NODE_DATA_PATH:-./data}:/app/data - - ./genvm-module-web-docker.yaml:/app/opt/genvm/${GENVM_VERSION}/config/genvm-module-web.yaml + # opt/genvm/current links to the image's GenVM tree, so no version variable is needed. + - ./genvm-module-web-docker.yaml:/app/opt/genvm/current/config/genvm-module-web.yaml security_opt: - no-new-privileges:true cap_drop: @@ -121,14 +165,20 @@ services: # docker compose --profile monitoring up -d alloy: image: grafana/alloy:v1.20.1 + init: true container_name: genlayer-node-alloy + labels: + com.genlayer.health-recovery.logs: "true" + com.genlayer.network: "${NETWORK_NAME:-unknown}" + depends_on: + - docker-socket-proxy # The guard runs before Alloy: with the monitoring profile up but the # endpoints or credentials unset, the container stops with a message # naming what to set, instead of half-running. Half-running is worse than # stopping here in both directions: pushing to a shared endpoint with a # shared default credential pollutes it for everyone, and pushing to an - # unreachable one grows the WAL until the healthcheck below reads the - # growth as a stall and deletes it. The $$ keeps compose from + # unreachable one leaves telemetry unavailable and can grow the WAL. The + # healthcheck preserves that buffer. The $$ keeps compose from # interpolating; the shell only tests emptiness — no value ever lands on # a command line. entrypoint: ["/bin/sh", "-c"] @@ -155,12 +205,29 @@ services: - ${NODE_LOGS_PATH:-./data/node/logs}:/var/log/genlayer:ro - alloy_data:/var/lib/alloy healthcheck: - test: ["CMD", "sh", "/etc/alloy/healthcheck.sh"] + test: + - CMD-SHELL + - | + set -eu + sh /etc/alloy/healthcheck.sh && exit 0 + [ "$${HEALTHCHECK_AUTO_RESTART:-true}" != false ] || exit 1 + [ "$$(cat /proc/1/comm)" = docker-init ] || exit 1 + boot=$$(awk '{print $$22}' /proc/1/stat) + age=$$(awk -v ticks="$$boot" -v hz="$$(getconf CLK_TCK)" '{print int($$1-ticks/hz)}' /proc/uptime) + [ "$$age" -ge 120 ] || exit 1 + children=$$(cat /proc/1/task/1/children) + printf 'Container alloy (%s) found to be unhealthy - Restarting container now\n' "$$(cat /etc/hostname)" > /proc/1/fd/1 + kill -TERM 1 + sleep 5 + # A stopped service cannot handle TERM; init's child can still receive KILL. + for child in $$children; do kill -KILL "$$child"; done + exit 1 interval: 120s timeout: 15s retries: 1 start_period: 120s environment: + - HEALTHCHECK_AUTO_RESTART=${HEALTHCHECK_AUTO_RESTART:-true} # Central monitoring endpoints and credentials: NO defaults, on purpose. # An endpoint may be public knowledge; a credential never is, and a # default one ships the same key to everyone holding this file. The diff --git a/content/validators/docker-compose.yaml b/content/validators/docker-compose.yaml index 41fe121c..6f2f2690 100644 --- a/content/validators/docker-compose.yaml +++ b/content/validators/docker-compose.yaml @@ -1,3 +1,8 @@ +# A fixed project name: the container names below are fixed too, so a stack +# started from a new release directory must be the SAME Compose project, or +# Docker refuses with "container name ... is already in use". With it, `up -d` +# from the new version's directory recreates the old version's containers. +name: genlayer-node services: webdriver-container: container_name: genlayer-node-webdriver @@ -7,24 +12,54 @@ services: - no-new-privileges:true environment: PORT: 4444 + HEALTHCHECK_AUTO_RESTART: "${HEALTHCHECK_AUTO_RESTART:-true}" ports: - "${WEBDRIVER_PORT:-4444}:4444" restart: unless-stopped - # docker-socket-proxy — hardened, read-only Docker API gateway used ONLY so the - # node can read the WebDriver container's resource stats (CPU/memory/traffic) for - # metrics. Runs with the `node` OR `monitoring` profile, so it comes up alongside + labels: + com.genlayer.health-recovery.logs: "true" + com.genlayer.network: "${NETWORK_NAME:-unknown}" + healthcheck: + test: + - CMD-SHELL + - | + set -eu + boot=$$(awk '{print $$22}' /proc/1/stat) + if curl -fsS --max-time 10 "http://localhost:$${PORT:-4444}/healthcheck?mode=text&url=https%3A%2F%2Ftest-server.genlayer.com%2Fstatic%2Fgenvm%2Fhello.html"; then + printf '%s 0\n' "$$boot" > /tmp/webdriver-health-failures + exit 0 + fi + age=$$(awk -v ticks="$$boot" -v hz="$$(getconf CLK_TCK)" '{print int($$1-ticks/hz)}' /proc/uptime) + [ "$$age" -ge 10 ] || exit 1 + read -r previous failures 2>/dev/null < /tmp/webdriver-health-failures || failures=0 + [ "$${previous:-}" = "$$boot" ] || failures=0 + failures=$$((failures + 1)) + printf '%s %s\n' "$$boot" "$$failures" > /tmp/webdriver-health-failures + if [ "$$failures" -ge 4 ] && [ "$${HEALTHCHECK_AUTO_RESTART:-true}" != false ]; then + printf 'Container webdriver (%s) found to be unhealthy - Restarting container now\n' "$$(cat /etc/hostname)" > /proc/1/fd/1 + children=$$(cat /proc/1/task/1/children) + for child in $$children; do kill -KILL "$$child"; done + fi + exit 1 + interval: 30s + timeout: 12s + retries: 4 + start_period: 10s + # docker-socket-proxy — read-only Docker API gateway for the node's WebDriver + # resource stats and Alloy's recovery-log reader. Runs with the `node` OR + # `monitoring` profile, so it comes up alongside # a node container AND alongside a monitoring/telemetry bring-up — the latter is # how a binary/systemd node (node not containerized) gets its webdriver metrics. # - # WHY THIS EXISTS — and why it is NOT a new exposure or a "leak": + # The node and Alloy use this read-only gateway instead of the host socket. # Reading container stats requires the Docker API. Previously the node reached # the host /var/run/docker.sock directly (bind-mounted into the container, or via # docker-group membership for a binary node) — but that grants the FULL Docker # API as root (a `:ro` flag does NOT restrict API calls), so a node compromise - # could escalate to host root. This proxy REPLACES that: it is the only thing - # touching the socket, restricted to read-only container endpoints (CONTAINERS=1; - # POST=0 => it returns 403 for create / exec / images / everything else). Net - # exposure goes DOWN, not up. + # could escalate to host root. This proxy REPLACES the node's socket access, + # restricted to read-only container endpoints (CONTAINERS=1; + # POST=0 => it returns 403 for create / exec / image operations). + # Health checks stop only their own service process; recovery has no Docker API access. # - Published ONLY on 127.0.0.1:2375 (host loopback), never 0.0.0.0: 2375 is # the unauthenticated Docker API, so it must not leave the host. A container # node reaches it over the compose network (tcp://docker-socket-proxy:2375); @@ -35,14 +70,15 @@ services: # socket: if DOCKER_HOST is unset the node skips webdriver metrics entirely # (non-fatal). Note those are the webdriver container's CPU/memory/traffic series # (genlayer_node_*{component="webdriver"}) that feed validator resource dashboards — - # so keep DOCKER_HOST set to keep those panels populated. To disable metrics - # instead: set metrics.collectors.webdriver.enabled: false and delete this service. + # so keep DOCKER_HOST set to keep those panels populated. Disabling metrics + # permits removing the node's DOCKER_HOST; Alloy still needs this gateway. docker-socket-proxy: - image: tecnativa/docker-socket-proxy:0.3.0 + image: tecnativa/docker-socket-proxy:v0.5.0 container_name: genlayer-node-docker-proxy environment: CONTAINERS: 1 # GET /containers/{json,inspect,stats} — read-only via POST=0 VERSION: 1 # required by the Docker client's API-version negotiation + NETWORKS: 1 # read-only network metadata required by Alloy Docker discovery volumes: - /var/run/docker.sock:/var/run/docker.sock:ro ports: @@ -64,23 +100,27 @@ services: - monitoring genlayer-node: image: yeagerai/genlayer-node:${NODE_VERSION:-latest} - # The keystore password is NOT on this command line. It reaches the node as + # No entrypoint here: the image's own entrypoint precompiles GenVM for the + # host CPU on the first start, then runs `genlayernode run`. The keystore + # password is NOT on a command line. It reaches the node as # GENLAYERNODE_PASSWORD (environment below), which the run command reads in # place of --password. A command-line argument is visible in process # listings on the host; an environment variable is not. - entrypoint: ["/app/bin/genlayernode", "run"] container_name: genlayer-node restart: unless-stopped env_file: - path: ./.env required: false environment: - # ":?" makes Compose refuse to start when NODE_PASSWORD is unset or empty, - # before any container runs. There is no fallback: this is the password - # that encrypts the validator's signing key, and a default shared by every - # operator who forgot to set one is a key anyone can open. The release - # binary carries no default either — it is compiled out of production. - GENLAYERNODE_PASSWORD: "${NODE_PASSWORD:?NODE_PASSWORD is not set. It is the keystore password that unlocks the validator key. Set it in .env; see README section 4.}" + # No default: this is the password that encrypts the validator's signing + # key, and a default shared by every operator who forgot to set one is a + # key anyone can open (the release binary carries none either). The node + # itself refuses validator mode with an empty password ("password is + # required for validator mode"); a full node needs none. Not a ":?" guard: + # Compose interpolates every service, also under another profile, so a + # guard here would stop `docker compose up -d webdriver-container` (the + # binary install path) and a full node, which never read it. + GENLAYERNODE_PASSWORD: "${NODE_PASSWORD:-}" # WebDriver metrics collector reaches Docker via the read-only proxy # (see docker-socket-proxy above), never the host socket directly. DOCKER_HOST: tcp://docker-socket-proxy:2375 @@ -89,8 +129,11 @@ services: - "${NODE_OPS_PORT:-9153}:9153" volumes: - ${NODE_CONFIG_PATH:-./configs/node/config.yaml}:/app/configs/node/config.yaml:ro + # The trusted protocol manifest of the network; consensus.manifestpath names this path. + - ${NODE_MANIFEST_PATH:-./configs/node/protocol-manifest.json}:/app/configs/node/protocol-manifest.json:ro - ${NODE_DATA_PATH:-./data}:/app/data - - ./genvm-module-web-docker.yaml:/app/opt/genvm/${GENVM_VERSION}/config/genvm-module-web.yaml + # opt/genvm/current links to the image's GenVM tree, so no version variable is needed. + - ./genvm-module-web-docker.yaml:/app/opt/genvm/current/config/genvm-module-web.yaml security_opt: - no-new-privileges:true cap_drop: diff --git a/content/validators/env.example b/content/validators/env.example index 60d972b3..c65d5345 100644 --- a/content/validators/env.example +++ b/content/validators/env.example @@ -12,8 +12,9 @@ WEBDRIVER_PORT=4444 # Docker image version NODE_VERSION=latest -# Keystore password (unlocks the pre-imported validator wallet). Required — the -# stack refuses to start without it. Choose your own; never reuse an example +# Keystore password (unlocks the pre-imported validator wallet). Required for a +# validator: the node refuses validator mode without it. A full node and the +# WebDriver-only stack need none. Choose your own; never reuse an example # value. It reaches the node as an environment variable, not a command-line # argument. See README section 4 for reading it in without echo. NODE_PASSWORD= @@ -21,6 +22,9 @@ NODE_PASSWORD= # Path to node configuration file NODE_CONFIG_PATH=./configs/node/config.yaml +# Path to the trusted protocol manifest of your network +NODE_MANIFEST_PATH=./configs/node/protocol-manifest.json + # Path to node data directory (for persistence) NODE_DATA_PATH=./data @@ -35,6 +39,11 @@ ANTHROPICKEY= XAIKEY= GEMINIKEY= ATOMAKEY= +COMPUT3KEY= +IOINTELLIGENCEKEY= +LIBERTAI_API_KEY= +MORPHEUS_API_KEY= +OPENAIKEY= # Optional logging level override GENLAYERNODE_LOGGING_LEVEL=INFO @@ -90,6 +99,9 @@ CENTRAL_LOKI_PASSWORD= # Node identification NODE_ID=validator-001 VALIDATOR_NAME=MyValidator +# Network label for recovery logs and alert routing; use the deployed network. +# Leave empty for an unknown label, rather than guessing a production network. +NETWORK_NAME= # Node metrics endpoint (optional) # Endpoint where the node exposes its Prometheus metrics @@ -110,7 +122,6 @@ NODE_METRICS_ENDPOINT=host.docker.internal:9153 # Example for multiple nodes: # SCRAPE_TARGETS_JSON=[{"__address__":"host.docker.internal:9250","instance":"0x...","validator_name":"node-1"},{"__address__":"host.docker.internal:9251","instance":"0x...","validator_name":"node-2"}] # -# To generate for local nodes, use: ./generate-dev-targets.sh # SCRAPE_TARGETS_JSON= # ============================================================================= @@ -130,8 +141,8 @@ LOG_FILE_PATTERN=/var/log/genlayer/node*.log # Metrics scraping intervals (optional) # How often to scrape metrics from the node -# Default: 15s for node metrics, 10s timeout -METRICS_SCRAPE_INTERVAL=15s +# Default: 60s for node metrics, 10s timeout +METRICS_SCRAPE_INTERVAL=60s METRICS_SCRAPE_TIMEOUT=10s # Alloy internal health monitoring interval (optional) @@ -142,9 +153,9 @@ ALLOY_SELF_MONITORING_INTERVAL=60s # Loki log batching configuration (optional) # Controls how logs are batched before sending to central Loki -# Default: 1MiB batch size, 1s wait time +# Default: 1MiB batch size, 60s wait time LOKI_BATCH_SIZE=1MiB -LOKI_BATCH_WAIT=1s +LOKI_BATCH_WAIT=60s # ============================================================================= # Deployment Instructions @@ -177,5 +188,8 @@ LOKI_BATCH_WAIT=1s # - Grafana Alloy handles both metrics and logs in a single service # - To customize monitoring, edit alloy-config.river # - To filter metrics (reduce bandwidth), uncomment the filtering rule in alloy-config.river -# - Ensure firewall allows outbound connections to monitoring server (port 8180) +# - Ensure the firewall allows outbound HTTPS to CENTRAL_MONITORING_URL and CENTRAL_LOKI_URL # + +# Keep health checks but disable automatic container-local recycling when false. +HEALTHCHECK_AUTO_RESTART=true diff --git a/content/validators/greybox-setup-guide.md b/content/validators/greybox-setup-guide.md index 08709109..58fbb563 100644 --- a/content/validators/greybox-setup-guide.md +++ b/content/validators/greybox-setup-guide.md @@ -45,33 +45,37 @@ echo "OPENROUTERKEY=sk-or-v1-your-key-here" >> ${ENV_FILE} The tarball ships with a unified config that includes all backends (openrouter, morpheus, heurist, ionet, etc.). +`NODE_DIR` is the directory of your node release (where you extracted the +tarball). On a host with the `/opt/genlayer-node/bin` symlink layout, it is +`/opt/genlayer-node/`. + ```bash -VERSION=$(readlink /opt/genlayer-node/bin | sed 's|/bin||; s|.*/||') +NODE_DIR=/path/to/your/node/release -cp /opt/genlayer-node/${VERSION}/opt/genvm/${GENVM_VERSION}/config/genvm-modules-llm-release.yaml \ - /opt/genlayer-node/${VERSION}/opt/genvm/${GENVM_VERSION}/config/genvm-module-llm.yaml +cp ${NODE_DIR}/opt/genvm/current/config/genvm-modules-llm-release.yaml \ + ${NODE_DIR}/opt/genvm/current/config/genvm-module-llm.yaml ``` ### 4. Switch to greybox strategy ```bash sed -i 's/genvm-llm-default\.lua/genvm-llm-greybox.lua/' \ - /opt/genlayer-node/${VERSION}/opt/genvm/${GENVM_VERSION}/config/genvm-module-llm.yaml + ${NODE_DIR}/opt/genvm/current/config/genvm-module-llm.yaml ``` ### 5. Verify the config ```bash # Check lua script path -grep lua_script_path /opt/genlayer-node/${VERSION}/opt/genvm/${GENVM_VERSION}/config/genvm-module-llm.yaml +grep lua_script_path ${NODE_DIR}/opt/genvm/current/config/genvm-module-llm.yaml # Expected: lua_script_path: ${exeDir}/../config/genvm-llm-greybox.lua # Check openrouter is present -grep -A2 'openrouter:' /opt/genlayer-node/${VERSION}/opt/genvm/${GENVM_VERSION}/config/genvm-module-llm.yaml +grep -A2 'openrouter:' ${NODE_DIR}/opt/genvm/current/config/genvm-module-llm.yaml # Expected: enabled: true # Check the greybox Lua file exists -ls -la /opt/genlayer-node/${VERSION}/opt/genvm/${GENVM_VERSION}/config/genvm-llm-greybox.lua +ls -la ${NODE_DIR}/opt/genvm/current/config/genvm-llm-greybox.lua ``` ### 6. Start the node @@ -102,7 +106,7 @@ To revert to the default generated chain: sudo systemctl stop genlayer-node sed -i 's/genvm-llm-greybox\.lua/genvm-llm-default.lua/' \ - /opt/genlayer-node/${VERSION}/opt/genvm/${GENVM_VERSION}/config/genvm-module-llm.yaml + ${NODE_DIR}/opt/genvm/current/config/genvm-module-llm.yaml sudo systemctl start genlayer-node ``` @@ -178,7 +182,7 @@ After editing the YAML, restart the LLM module (see "Updating Greybox on a Runni **All models exhausted error:** - OpenRouter may be down or your key is invalid - Check your key at https://openrouter.ai/settings/keys -- Run `task node:doctor` — the "LLM Providers" section probes each enabled backend and reports which ones actually answer, rather than only whether a key variable is set +- Run `./bin/genlayernode doctor` from `${NODE_DIR}` — the "LLM Providers" section probes each enabled backend and reports which ones actually answer, rather than only whether a key variable is set - Fallback providers (heurist, ionet) also need valid keys if you want fallback to work **Configure more than one provider.** A backend only enters the fallback chain diff --git a/pages/api-references/genlayer-node.mdx b/pages/api-references/genlayer-node.mdx index 90a751c3..a37ea68e 100644 --- a/pages/api-references/genlayer-node.mdx +++ b/pages/api-references/genlayer-node.mdx @@ -1837,24 +1837,25 @@ curl -X GET http://localhost:9153/health ```json { "status": "healthy", - "network": "", + "network": "bradbury-phase2", "node_version": "v0.6.0", "protocol_version": "0.3.0", "genvm_version": "v0.0.18", "validator_address": "0x1234567890abcdef1234567890abcdef12345678", "validator_moniker": "my-validator", "conditions": { - "chain.rpc": { "state": "healthy", "severity": "critical", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, - "chain.websocket": { "state": "healthy", "severity": "critical", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, - "chain.sync": { "state": "healthy", "severity": "critical", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:02:14Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, - "consensus.ban": { "state": "healthy", "severity": "critical", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, - "consensus.stake": { "state": "healthy", "severity": "critical", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, - "consensus.operator_funds": { "state": "healthy", "severity": "critical", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, - "consensus.prime": { "state": "healthy", "severity": "critical", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:04:41Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, - "genvm.llm": { "state": "healthy", "severity": "critical", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:31Z", "updated_at": "2026-09-01T10:29:56Z", "last_ok": "2026-09-01T10:29:56Z" }, - "genvm.web": { "state": "healthy", "severity": "critical", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:31Z", "updated_at": "2026-09-01T10:29:56Z", "last_ok": "2026-09-01T10:29:56Z" }, - "node.disk": { "state": "healthy", "severity": "critical", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, - "node.snapshot_age": { "state": "healthy", "severity": "info", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" } + "chain.rpc": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, + "chain.websocket": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, + "chain.sync": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:02:14Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, + "consensus.ban": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, + "consensus.stake": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, + "consensus.operator_funds": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, + "consensus.prime": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:04:41Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, + "genvm.llm": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:31Z", "updated_at": "2026-09-01T10:29:56Z", "last_ok": "2026-09-01T10:29:56Z" }, + "genvm.web": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:31Z", "updated_at": "2026-09-01T10:29:56Z", "last_ok": "2026-09-01T10:29:56Z" }, + "node.disk": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, + "node.snapshot_age": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, + "webdriver.render": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" } } } ``` @@ -1867,8 +1868,7 @@ curl -X GET http://localhost:9153/health - `validator_address` (string, validator mode only): The validator's address - `validator_moniker` (string, validator mode only): The validator's on-chain moniker; reported alongside the address even when empty - `conditions` (object): the full catalog. Every entry is always present — this is not a list of what is broken. - - `state` (string): current state - - `severity` (string): the catalog's fixed class — `info`, `warning`, or `critical`. Not the state: a `critical` condition can be `healthy`. + - `state` (string): current observed health; use this field to determine whether the capability is working. - `occurrences` (number): accepted failed observations since process start - `consecutive_failures` (number): current failure streak; zero after recovery - `since`, `updated_at`, `last_ok` (string, optional): ISO 8601 lifecycle timestamps @@ -1877,6 +1877,8 @@ curl -X GET http://localhost:9153/health - `action` (string, present only when not healthy): what the operator should do - `details` (object, present only when not healthy): typed fields explicitly allowlisted by the catalog entry. Raw provider payloads, prompts, credentials, URLs, and stack traces are rejected. +`severity` is not returned by this endpoint. Alert priority belongs to the alerting policy; clients that previously read `conditions.*.severity` should use `state` for current health. + **States** | State | Meaning | Alerted | @@ -1900,6 +1902,28 @@ A condition can never exceed the liveness cap its catalog entry declares. The Ge `consensus.ban`, `consensus.stake`, `chain.rpc`, `chain.websocket` and `chain.sync` are capped at `unavailable`, because each of them stops the node doing its job. +**WebDriver render check** + +`webdriver.render` runs the same fresh render probe as `doctor`, using +`webdriver_host` from the configured GenVM's `genvm-module-web.yaml`. It requires +HTTP 200, `Resulting-Status: 200`, and exactly `Hello world!` from the diagnostic +page, within 10 seconds. It bypasses WebDriver's cached `/healthcheck` response. +A failure reports `degraded` with reason `webdriver_check_failed` and safe +`failure`, `http_status` and `page_status` details. The endpoint still returns +HTTP 200: a browser outage does not stop the node syncing and voting. + +Allow time for the ten-second render deadline when setting a client timeout. +`task devtools:nodes:health` defaults to 15 seconds per request so a browser +timeout can return its diagnosis instead of appearing as an unreachable node. + +`genvm.web` continues reporting the broader web module's boot and transaction +results. A successful direct browser check does not clear a web-module failure. + +For `genvm.start_manager: false`, the manager owns the browser configuration. +The node leaves `webdriver.render` unknown instead of probing an unused local +root. Doctor likewise reports the browser as not checked; run it on the node +that owns the manager configuration. Observed `genvm.web` results still apply. + **Example: the LLM provider is dead — HTTP 200** ```json @@ -1908,7 +1932,6 @@ A condition can never exceed the liveness cap its catalog entry declares. The Ge "conditions": { "genvm.llm": { "state": "degraded", - "severity": "critical", "reason": "llm_request_failed", "impact": "This validator cannot execute a transaction that needs an LLM. It votes Timeout, and the consensus contracts may penalize a Timeout vote.", "action": "Check the affected LLM provider service and payment or billing status, restore access, or configure a working fallback. If credentials are changed through .env, restart the node.", @@ -1930,7 +1953,6 @@ A condition can never exceed the liveness cap its catalog entry declares. The Ge "conditions": { "consensus.ban": { "state": "unavailable", - "severity": "critical", "reason": "validator_banned", "impact": "The consensus contracts do not select this validator for a round. The node earns nothing.", "action": "Wait for the stated epoch. If the ban is permanent, restake with a new validator.", diff --git a/pages/api-references/genlayer-node/ops/health.mdx b/pages/api-references/genlayer-node/ops/health.mdx index 8ecf1de0..fdd2fe13 100644 --- a/pages/api-references/genlayer-node/ops/health.mdx +++ b/pages/api-references/genlayer-node/ops/health.mdx @@ -21,24 +21,25 @@ curl -X GET http://localhost:9153/health ```json { "status": "healthy", - "network": "", + "network": "bradbury-phase2", "node_version": "v0.6.0", "protocol_version": "0.3.0", "genvm_version": "v0.0.18", "validator_address": "0x1234567890abcdef1234567890abcdef12345678", "validator_moniker": "my-validator", "conditions": { - "chain.rpc": { "state": "healthy", "severity": "critical", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, - "chain.websocket": { "state": "healthy", "severity": "critical", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, - "chain.sync": { "state": "healthy", "severity": "critical", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:02:14Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, - "consensus.ban": { "state": "healthy", "severity": "critical", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, - "consensus.stake": { "state": "healthy", "severity": "critical", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, - "consensus.operator_funds": { "state": "healthy", "severity": "critical", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, - "consensus.prime": { "state": "healthy", "severity": "critical", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:04:41Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, - "genvm.llm": { "state": "healthy", "severity": "critical", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:31Z", "updated_at": "2026-09-01T10:29:56Z", "last_ok": "2026-09-01T10:29:56Z" }, - "genvm.web": { "state": "healthy", "severity": "critical", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:31Z", "updated_at": "2026-09-01T10:29:56Z", "last_ok": "2026-09-01T10:29:56Z" }, - "node.disk": { "state": "healthy", "severity": "critical", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, - "node.snapshot_age": { "state": "healthy", "severity": "info", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" } + "chain.rpc": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, + "chain.websocket": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, + "chain.sync": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:02:14Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, + "consensus.ban": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, + "consensus.stake": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, + "consensus.operator_funds": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, + "consensus.prime": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:04:41Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, + "genvm.llm": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:31Z", "updated_at": "2026-09-01T10:29:56Z", "last_ok": "2026-09-01T10:29:56Z" }, + "genvm.web": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:31Z", "updated_at": "2026-09-01T10:29:56Z", "last_ok": "2026-09-01T10:29:56Z" }, + "node.disk": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, + "node.snapshot_age": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" }, + "webdriver.render": { "state": "healthy", "occurrences": 0, "consecutive_failures": 0, "since": "2026-09-01T08:00:03Z", "updated_at": "2026-09-01T10:31:12Z", "last_ok": "2026-09-01T10:31:12Z" } } } ``` @@ -51,8 +52,7 @@ curl -X GET http://localhost:9153/health - `validator_address` (string, validator mode only): The validator's address - `validator_moniker` (string, validator mode only): The validator's on-chain moniker; reported alongside the address even when empty - `conditions` (object): the full catalog. Every entry is always present — this is not a list of what is broken. - - `state` (string): current state - - `severity` (string): the catalog's fixed class — `info`, `warning`, or `critical`. Not the state: a `critical` condition can be `healthy`. + - `state` (string): current observed health; use this field to determine whether the capability is working. - `occurrences` (number): accepted failed observations since process start - `consecutive_failures` (number): current failure streak; zero after recovery - `since`, `updated_at`, `last_ok` (string, optional): ISO 8601 lifecycle timestamps @@ -61,6 +61,8 @@ curl -X GET http://localhost:9153/health - `action` (string, present only when not healthy): what the operator should do - `details` (object, present only when not healthy): typed fields explicitly allowlisted by the catalog entry. Raw provider payloads, prompts, credentials, URLs, and stack traces are rejected. +`severity` is not returned by this endpoint. Alert priority belongs to the alerting policy; clients that previously read `conditions.*.severity` should use `state` for current health. + **States** | State | Meaning | Alerted | @@ -84,6 +86,28 @@ A condition can never exceed the liveness cap its catalog entry declares. The Ge `consensus.ban`, `consensus.stake`, `chain.rpc`, `chain.websocket` and `chain.sync` are capped at `unavailable`, because each of them stops the node doing its job. +**WebDriver render check** + +`webdriver.render` runs the same fresh render probe as `doctor`, using +`webdriver_host` from the configured GenVM's `genvm-module-web.yaml`. It requires +HTTP 200, `Resulting-Status: 200`, and exactly `Hello world!` from the diagnostic +page, within 10 seconds. It bypasses WebDriver's cached `/healthcheck` response. +A failure reports `degraded` with reason `webdriver_check_failed` and safe +`failure`, `http_status` and `page_status` details. The endpoint still returns +HTTP 200: a browser outage does not stop the node syncing and voting. + +Allow time for the ten-second render deadline when setting a client timeout. +`task devtools:nodes:health` defaults to 15 seconds per request so a browser +timeout can return its diagnosis instead of appearing as an unreachable node. + +`genvm.web` continues reporting the broader web module's boot and transaction +results. A successful direct browser check does not clear a web-module failure. + +For `genvm.start_manager: false`, the manager owns the browser configuration. +The node leaves `webdriver.render` unknown instead of probing an unused local +root. Doctor likewise reports the browser as not checked; run it on the node +that owns the manager configuration. Observed `genvm.web` results still apply. + **Example: the LLM provider is dead — HTTP 200** ```json @@ -92,7 +116,6 @@ A condition can never exceed the liveness cap its catalog entry declares. The Ge "conditions": { "genvm.llm": { "state": "degraded", - "severity": "critical", "reason": "llm_request_failed", "impact": "This validator cannot execute a transaction that needs an LLM. It votes Timeout, and the consensus contracts may penalize a Timeout vote.", "action": "Check the affected LLM provider service and payment or billing status, restore access, or configure a working fallback. If credentials are changed through .env, restart the node.", @@ -114,7 +137,6 @@ A condition can never exceed the liveness cap its catalog entry declares. The Ge "conditions": { "consensus.ban": { "state": "unavailable", - "severity": "critical", "reason": "validator_banned", "impact": "The consensus contracts do not select this validator for a round. The node earns nothing.", "action": "Wait for the stated epoch. If the ban is permanent, restake with a new validator.", diff --git a/pages/validators/changelog.mdx b/pages/validators/changelog.mdx index bac7ca4b..2b203bd7 100644 --- a/pages/validators/changelog.mdx +++ b/pages/validators/changelog.mdx @@ -1,5 +1,37 @@ # Changelog +## v0.6.0-rc.2 + +### New features + +- Check WebDriver rendering in `/health` and `doctor` with the same fresh render of a test page (10 s limit); a failure sets the `webdriver.render` condition to `degraded` with reason `webdriver_check_failed` +- The release compose file restarts a WebDriver that stops rendering (after 4 failed health checks) and an Alloy that stays unhealthy; set `HEALTHCHECK_AUTO_RESTART=false` to keep the health checks and turn off the restarts +- The release monitoring profile sends the restart events of WebDriver and Alloy to central monitoring, with the `NETWORK_NAME` label from `.env` + +### Misc + +- Docker: mount custom GenVM configs (for example `genvm-module-llm.yaml` for OpenRouter/greybox) at `/app/opt/genvm/current/config/`; v0.6 no longer reads `/app/third_party/genvm/config/`, so a v0.5 `docker-compose.override.yml` mount there is ignored and the image default loads +- `/health` no longer returns `severity` in `conditions`; use `state` for the current health +- The Bradbury example config (`configs/node/bradbury.yaml.example`) uses the GenLayer RPC host `rpc.testnet-chain.genlayer.com` (HTTPS and WSS); RPC keys still come from Conduit +- The release compose file mounts the protocol manifest from `NODE_MANIFEST_PATH` (default `./configs/node/protocol-manifest.json`) at the path that `consensus.manifestpath` names +- The release package links `opt/genvm/current` to its GenVM tree, and its `config.yaml` sets `genvm.root_dir: ./opt/genvm/current` +- The example `.env` lists all 11 LLM provider keys of the release LLM config, and sets the monitoring defaults `METRICS_SCRAPE_INTERVAL=60s` and `LOKI_BATCH_WAIT=60s` +- The release README shows how to run `doctor` under Docker +- Update `tecnativa/docker-socket-proxy` to v0.5.0; the proxy also gives Alloy read-only network data +- Update `github.com/ethereum/go-ethereum` to v1.17.8, `labstack/echo` to v4.16.0, `golang.org/x/crypto` to v0.58.0 and `golang.org/x/net` to v0.61.0 + +### Bug fixes + +- The release compose file no longer requires `NODE_PASSWORD` for every service: `docker compose up -d webdriver-container` (the binary install) and a full node work with it empty, and a validator still refuses to start without it +- The release compose file has a fixed project name (`genlayer-node`), so starting the stack from a new release directory replaces the old one's containers instead of failing with `container name ... is already in use` +- `install.sh` keeps `genvm.root_dir: ./opt/genvm/current` in a package that has that link, as the docs say +- A synced node no longer logs a false `ERR operator condition requires attention` for `chain.sync` at start: the condition stays unknown until the sync loop has measured the lag +- Fix web access from contracts under the release compose file: the GenVM web config is mounted at the path GenVM reads, so GenVM uses the WebDriver container instead of `localhost` +- The release compose file no longer replaces the image entrypoint, so the first start precompiles GenVM for the host CPU (the first start takes longer) +- The example config sets `genvm.root_dir` to the shipped GenVM (`./opt/genvm/v0.6.0-rc10`), not `v0.6.0-rc1` +- Update the GenVM web config of the release to the schema and keys of the shipped GenVM +- Correct the API reference: `LeaderRevealing` is status `13`, and the vote values include `5` (`DeterministicViolation`) + ## v0.6.0-rc.1 v0.6 is a breaking release. It runs only against the v0.6 consensus contracts (release tag `v0.6-rc-1`), and all validators upgrade together: receipt hashes, storage-fee votes, spendable contract balance and child budgets change. diff --git a/pages/validators/genvm-configuration.mdx b/pages/validators/genvm-configuration.mdx index 1e82ac55..32d1980a 100644 --- a/pages/validators/genvm-configuration.mdx +++ b/pages/validators/genvm-configuration.mdx @@ -46,7 +46,7 @@ Repeat for each GenVM instance if running multiple. A module reload picks up YAM ## Configuration Files -All configuration lives in the GenVM tree of the release, at `opt/genvm//config/`. v0.6.0-rc.1 ships GenVM v0.6.0-rc10: `opt/genvm/v0.6.0-rc10/config/` (in the Docker image: `/app/opt/genvm/v0.6.0-rc10/config/`). Three files: +All configuration lives in the GenVM tree of the release, at `opt/genvm/current/config/` (in the Docker image: `/app/opt/genvm/current/config/`). `opt/genvm/current` links to the GenVM version the release ships. Three files: | File | What it controls | Should you modify? | |---|---|---| diff --git a/pages/validators/greybox-setup.mdx b/pages/validators/greybox-setup.mdx index 909eb774..35afcff1 100644 --- a/pages/validators/greybox-setup.mdx +++ b/pages/validators/greybox-setup.mdx @@ -47,33 +47,37 @@ echo "OPENROUTERKEY=sk-or-v1-your-key-here" >> ${ENV_FILE} The tarball ships with a unified config that includes all backends (openrouter, morpheus, heurist, ionet, etc.). +`NODE_DIR` is the directory of your node release (where you extracted the +tarball). On a host with the `/opt/genlayer-node/bin` symlink layout, it is +`/opt/genlayer-node/`. + ```bash -VERSION=$(readlink /opt/genlayer-node/bin | sed 's|/bin||; s|.*/||') +NODE_DIR=/path/to/your/node/release -cp /opt/genlayer-node/${VERSION}/opt/genvm/${GENVM_VERSION}/config/genvm-modules-llm-release.yaml \ - /opt/genlayer-node/${VERSION}/opt/genvm/${GENVM_VERSION}/config/genvm-module-llm.yaml +cp ${NODE_DIR}/opt/genvm/current/config/genvm-modules-llm-release.yaml \ + ${NODE_DIR}/opt/genvm/current/config/genvm-module-llm.yaml ``` ### 4. Switch to greybox strategy ```bash sed -i 's/genvm-llm-default\.lua/genvm-llm-greybox.lua/' \ - /opt/genlayer-node/${VERSION}/opt/genvm/${GENVM_VERSION}/config/genvm-module-llm.yaml + ${NODE_DIR}/opt/genvm/current/config/genvm-module-llm.yaml ``` ### 5. Verify the config ```bash # Check lua script path -grep lua_script_path /opt/genlayer-node/${VERSION}/opt/genvm/${GENVM_VERSION}/config/genvm-module-llm.yaml +grep lua_script_path ${NODE_DIR}/opt/genvm/current/config/genvm-module-llm.yaml # Expected: lua_script_path: ${exeDir}/../config/genvm-llm-greybox.lua # Check openrouter is present -grep -A2 'openrouter:' /opt/genlayer-node/${VERSION}/opt/genvm/${GENVM_VERSION}/config/genvm-module-llm.yaml +grep -A2 'openrouter:' ${NODE_DIR}/opt/genvm/current/config/genvm-module-llm.yaml # Expected: enabled: true # Check the greybox Lua file exists -ls -la /opt/genlayer-node/${VERSION}/opt/genvm/${GENVM_VERSION}/config/genvm-llm-greybox.lua +ls -la ${NODE_DIR}/opt/genvm/current/config/genvm-llm-greybox.lua ``` ### 6. Start the node @@ -104,7 +108,7 @@ To revert to the default generated chain: sudo systemctl stop genlayer-node sed -i 's/genvm-llm-greybox\.lua/genvm-llm-default.lua/' \ - /opt/genlayer-node/${VERSION}/opt/genvm/${GENVM_VERSION}/config/genvm-module-llm.yaml + ${NODE_DIR}/opt/genvm/current/config/genvm-module-llm.yaml sudo systemctl start genlayer-node ``` @@ -180,7 +184,7 @@ After editing the YAML, restart the LLM module (see "Updating Greybox on a Runni **All models exhausted error:** - OpenRouter may be down or your key is invalid - Check your key at https://openrouter.ai/settings/keys -- Run `task node:doctor` — the "LLM Providers" section probes each enabled backend and reports which ones actually answer, rather than only whether a key variable is set +- Run `./bin/genlayernode doctor` from `${NODE_DIR}` — the "LLM Providers" section probes each enabled backend and reports which ones actually answer, rather than only whether a key variable is set - Fallback providers (heurist, ionet) also need valid keys if you want fallback to work **Configure more than one provider.** A backend only enters the fallback chain diff --git a/pages/validators/monitoring.mdx b/pages/validators/monitoring.mdx index f2a869c2..3c1da34f 100644 --- a/pages/validators/monitoring.mdx +++ b/pages/validators/monitoring.mdx @@ -120,6 +120,8 @@ CENTRAL_LOKI_PASSWORD= # Node identification NODE_ID=validator-001 VALIDATOR_NAME=MyValidator +# Network label for recovery logs and alert routing +NETWORK_NAME=bradbury-phase2 # Usually defaults are fine NODE_METRICS_ENDPOINT=host.docker.internal:9153 @@ -127,6 +129,8 @@ LOG_FILE_PATTERN=/var/log/genlayer/node*.log METRICS_SCRAPE_INTERVAL=15s # Host directory where the node writes its log files (mounted into Alloy) NODE_LOGS_PATH=./data/node/logs +# false keeps the container health checks but turns off automatic restarts +HEALTHCHECK_AUTO_RESTART=true ``` 2. **Add or verify the Alloy service in docker-compose.yaml.** Use the docker-compose.yaml from the release package as it is. It already defines the alloy service under the monitoring profile, together with the docker-socket-proxy service that alloy depends on. Alloy also mounts alloy-healthcheck.sh from the same package. Copying only the alloy block into another compose file does not work: it will not start without docker-socket-proxy and alloy-healthcheck.sh. The block is shown here for reference: @@ -144,14 +148,20 @@ NODE_LOGS_PATH=./data/node/logs # docker compose --profile monitoring up -d alloy: image: grafana/alloy:v1.20.1 + init: true container_name: genlayer-node-alloy + labels: + com.genlayer.health-recovery.logs: "true" + com.genlayer.network: "${NETWORK_NAME:-unknown}" + depends_on: + - docker-socket-proxy # The guard runs before Alloy: with the monitoring profile up but the # endpoints or credentials unset, the container stops with a message # naming what to set, instead of half-running. Half-running is worse than # stopping here in both directions: pushing to a shared endpoint with a # shared default credential pollutes it for everyone, and pushing to an - # unreachable one grows the WAL until the healthcheck below reads the - # growth as a stall and deletes it. The $$ keeps compose from + # unreachable one leaves telemetry unavailable and can grow the WAL. The + # healthcheck preserves that buffer. The $$ keeps compose from # interpolating; the shell only tests emptiness — no value ever lands on # a command line. entrypoint: ["/bin/sh", "-c"] @@ -178,12 +188,29 @@ NODE_LOGS_PATH=./data/node/logs - ${NODE_LOGS_PATH:-./data/node/logs}:/var/log/genlayer:ro - alloy_data:/var/lib/alloy healthcheck: - test: ["CMD", "sh", "/etc/alloy/healthcheck.sh"] + test: + - CMD-SHELL + - | + set -eu + sh /etc/alloy/healthcheck.sh && exit 0 + [ "$${HEALTHCHECK_AUTO_RESTART:-true}" != false ] || exit 1 + [ "$$(cat /proc/1/comm)" = docker-init ] || exit 1 + boot=$$(awk '{print $$22}' /proc/1/stat) + age=$$(awk -v ticks="$$boot" -v hz="$$(getconf CLK_TCK)" '{print int($$1-ticks/hz)}' /proc/uptime) + [ "$$age" -ge 120 ] || exit 1 + children=$$(cat /proc/1/task/1/children) + printf 'Container alloy (%s) found to be unhealthy - Restarting container now\n' "$$(cat /etc/hostname)" > /proc/1/fd/1 + kill -TERM 1 + sleep 5 + # A stopped service cannot handle TERM; init's child can still receive KILL. + for child in $$children; do kill -KILL "$$child"; done + exit 1 interval: 120s timeout: 15s retries: 1 start_period: 120s environment: + - HEALTHCHECK_AUTO_RESTART=${HEALTHCHECK_AUTO_RESTART:-true} # Central monitoring endpoints and credentials: NO defaults, on purpose. # An endpoint may be public knowledge; a credential never is, and a # default one ships the same key to everyone holding this file. The @@ -292,6 +319,65 @@ loki.source.file "genlayer" { } // Write logs to central Loki instance +// Read opted-in WebDriver/Alloy recovery events through the existing read-only +// Docker proxy. Containers log their own restart attempts before exiting. +discovery.docker "health_recovery" { + host = "http://docker-socket-proxy:2375" + refresh_interval = "5s" + + filter { + name = "label" + values = [ + "com.genlayer.health-recovery.logs=true", + ] + } +} + +discovery.relabel "health_recovery" { + targets = discovery.docker.health_recovery.targets + + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_project"] + target_label = "compose_project" + } + + rule { + source_labels = ["__meta_docker_container_label_com_genlayer_network"] + target_label = "network" + } + + rule { + source_labels = ["network"] + regex = "" + target_label = "network" + replacement = "unknown" + } +} + +loki.source.docker "health_recovery" { + // Docker's TCP transport keeps the log stream open. With an HTTP(S) host, + // Alloy bounds the entire stream by refresh_interval and repeatedly reopens it. + host = "tcp://docker-socket-proxy:2375" + targets = discovery.relabel.health_recovery.output + refresh_interval = "5s" + labels = { + job = "genlayer-node", + component = "health-recovery", + instance = sys.env("NODE_ID"), + validator_name = sys.env("VALIDATOR_NAME"), + } + forward_to = [loki.process.health_recovery.receiver] +} + +// Collect only restart events, not Alloy's own operational log stream. +loki.process "health_recovery" { + stage.match { + selector = "{component=\"health-recovery\"} != \"found to be unhealthy - Restarting container now\"" + action = "drop" + } + forward_to = [loki.write.central.receiver] +} + loki.write "central" { endpoint { url = sys.env("CENTRAL_LOKI_URL") @@ -423,13 +509,16 @@ Look for messages indicating successful batch sending (no error codes like 401, ## Container Health Checks -In v0.6.0-rc.1 the release docker-compose.yaml gives the Alloy container a health check (`alloy-healthcheck.sh`, every 120 seconds). It detects a stalled push to the central server: when Alloy's write-ahead log keeps growing, the check clears it and marks the container unhealthy, and Docker restarts Alloy (`restart: unless-stopped`). The WebDriver container has no health check in this release. +The release docker-compose.yaml gives the webdriver and Alloy containers health checks. When a check keeps failing, the container stops its own process and Docker restarts it (`restart: unless-stopped`). A container you stop by hand stays stopped. Each restart attempt is written to the container's logs; with monitoring on, Alloy forwards these lines with `component="health-recovery"`. + +To keep the health checks but turn off the automatic restarts, set `HEALTHCHECK_AUTO_RESTART=false` in `.env` and recreate the services. -Check the Alloy container's health: +Check container health and recovery events: ```bash copy +docker inspect --format '{{.State.Health.Status}}' genlayer-node-webdriver docker inspect --format '{{.State.Health.Status}}' genlayer-node-alloy -docker compose logs --since 30m alloy +docker compose logs --since 30m webdriver-container alloy ``` ## Troubleshooting diff --git a/pages/validators/setup-guide.mdx b/pages/validators/setup-guide.mdx index 3a393d56..0f0ee823 100644 --- a/pages/validators/setup-guide.mdx +++ b/pages/validators/setup-guide.mdx @@ -240,6 +240,7 @@ One full node can be shared between multiple validators. The optimal validator-t You should see a list like this ```sh + v0.6.0-rc.2 v0.6.0-rc.1 ``` @@ -247,7 +248,7 @@ One full node can be shared between multiple validators. The optimal validator-t 2. Download the packaged application ```sh copy - export version=v0.6.0-rc.1 # example target; confirm it is published and approved before use + export version=v0.6.0-rc.2 # example target; confirm it is published and approved before use base=https://storage.googleapis.com/gh-af/genlayer-node/bin/amd64/${version} curl -fI "${base}/genlayer-node-linux-amd64-${version}.tar.gz" curl -fI "${base}/genlayer-node_${version}_SHA256SUMS" @@ -295,7 +296,7 @@ One full node can be shared between multiple validators. The optimal validator-t Run it in the final install location: it writes this directory's absolute path into the GenVM binaries. If you move the installation later, run it again there. ```sh copy - python3 ./opt/genvm/v0.6.0-rc10/bin/setup.py # v0.6.0-rc.1 ships GenVM v0.6.0-rc10: check with `ls opt/genvm` + python3 ./opt/genvm/current/bin/setup.py ``` It does following: @@ -316,7 +317,7 @@ The file needs to be located at `configs/node/config.yaml` You can use the following example configuration. **Note:** For most users, you will only need to replace the `rollup` and `consensus` sections with the values for your target network (see [Network-Specific Consensus Configuration](#network-specific-consensus-configuration) below). -Set `genvm.root_dir` to the GenVM directory the release ships. v0.6.0-rc.1 ships `opt/genvm/v0.6.0-rc10`, but its config template says `./opt/genvm/v0.6.0-rc1`: set `genvm.root_dir: ./opt/genvm/v0.6.0-rc10` (check with `ls opt/genvm`). `install.sh` writes the right value for you. +The `config.yaml` in the release package sets `genvm.root_dir: ./opt/genvm/current`, a link to the GenVM version the release ships. Keep that value: the example below names the versioned directory, which the package also contains. ```yaml copy # rollup configuration @@ -332,9 +333,9 @@ rollup: consensus: # The deployment-specific fields are empty: take them from your network's documentation. consensusaddress: "" # AddressManager contract address for this deployment. - manifestpath: "" # Trusted producer protocol-manifest.json for this deployment. The deploy places it at configs/node/protocol-manifest.json; relative paths resolve against the node root. A mismatch with consensusaddress is refused at startup. + manifestpath: "configs/node/protocol-manifest.json" # Trusted producer protocol manifest for this deployment: save your network's manifest (from its documentation) at this path; a dev-env deploy places its own here too. Relative paths resolve against the node root. A mismatch with consensusaddress is refused at startup. chainid: "" # DECIMAL chain id this deployment was made on (not hex, no leading zeros). Held independently of the manifest so a manifest describing another deployment is caught. - # genesis: 17326 # (Optional) Genesis block number for this consensus deployment. If not provided, it will be auto-detected by searching for the first log from the ConsensusMain contract. + # genesis: 47930 # (Optional) Genesis block number for this consensus deployment. If not provided, it will be auto-detected by searching for the first log from the ConsensusMain contract. # chain_submit_buffer: "30s" # (Optional) Slack reserved at the end of each consensus step to submit the resulting on-chain tx (commit vote, reveal vote, proposed receipt) before the slot closes. Tune to underlying settlement chain latency. Default 30s. # admissiontimeout: "15s" # (Optional) Bound on the startup deployment admission check. Never unbounded. Default 15s. # data directory @@ -516,7 +517,7 @@ node: # max_debug_field_bytes: 262144 # 256 KB per field # genvm configuration genvm: - root_dir: ./opt/genvm/v0.6.0-rc1 + root_dir: ./opt/genvm/v0.6.0-rc10 start_manager: true # if true node will start genvm manager itself manager_url: http://127.0.0.1:3999 # Permits are charged PER EXECUTION by the genvm manager, and one @@ -624,7 +625,7 @@ mkdir -p configs/node curl -fsSL https://docs.genlayer.com/validators/bradbury.protocol-manifest.json -o configs/node/protocol-manifest.json ``` -The binary reads it from that path. With Docker, mount it into the container yourself: the v0.6.0-rc.1 compose file does not (see [Running the Node using docker-compose](#running-the-node-using-docker-compose)). +The binary reads it from that path. Docker Compose mounts the same file into the container (`NODE_MANIFEST_PATH`, default `./configs/node/protocol-manifest.json`). **WebSocket API key.** The Bradbury HTTPS RPC needs no key, but its WebSocket endpoint rejects connections without an API key. @@ -789,8 +790,6 @@ Once you have configured everything, you are ready to start the node. docker compose up -d # Starts the WebDriver needed by the GenVM web module ``` - In v0.6.0-rc.1 every `docker compose` command fails while `NODE_PASSWORD` is empty, also for the WebDriver alone. For a binary node, set any non-empty value in the `.env` next to `docker-compose.yaml`. - The node starts the GenVM manager and its modules itself (`genvm.start_manager: true` in the config template). **WebDriver resource metrics (optional).** To report the WebDriver container's CPU, memory and traffic as `genlayer_node_*{component="webdriver"}` series, also start the read-only Docker API proxy, and give only the node process `DOCKER_HOST=tcp://127.0.0.1:2375` (see step 4): @@ -843,16 +842,7 @@ You can also run the GenLayer node using Docker and Docker Compose. Release cand Before the first start, in the directory with your `docker-compose.yaml`: - `configs/node/config.yaml` holds your network's `rollup` and `consensus` values (see [Configuration](#configuration)). -- `configs/node/protocol-manifest.json` is your network's trusted manifest (see [Network-Specific Consensus Configuration](#network-specific-consensus-configuration)). The v0.6.0-rc.1 compose file does not mount it: add a `docker-compose.override.yml` next to `docker-compose.yaml`, which Docker Compose reads automatically: - - ```yaml copy - services: - genlayer-node: - volumes: - - ./configs/node/protocol-manifest.json:/app/configs/node/protocol-manifest.json:ro - # Only if you customize the LLM config (for example for greyboxing): - # - ./genvm-module-llm.yaml:/app/opt/genvm/v0.6.0-rc10/config/genvm-module-llm.yaml:ro - ``` +- `configs/node/protocol-manifest.json` is your network's trusted manifest (see [Network-Specific Consensus Configuration](#network-specific-consensus-configuration)). - The node container runs as uid `10001`, and the bind-mounted `./data` keeps its host owner. Let that user write to it once: ```sh copy @@ -862,6 +852,11 @@ Before the first start, in the directory with your `docker-compose.yaml`: 1. Create a `docker-compose.yaml` file with the following content: ```yaml copy +# A fixed project name: the container names below are fixed too, so a stack +# started from a new release directory must be the SAME Compose project, or +# Docker refuses with "container name ... is already in use". With it, `up -d` +# from the new version's directory recreates the old version's containers. +name: genlayer-node services: webdriver-container: container_name: genlayer-node-webdriver @@ -871,24 +866,54 @@ services: - no-new-privileges:true environment: PORT: 4444 + HEALTHCHECK_AUTO_RESTART: "${HEALTHCHECK_AUTO_RESTART:-true}" ports: - "${WEBDRIVER_PORT:-4444}:4444" restart: unless-stopped - # docker-socket-proxy — hardened, read-only Docker API gateway used ONLY so the - # node can read the WebDriver container's resource stats (CPU/memory/traffic) for - # metrics. Runs with the `node` OR `monitoring` profile, so it comes up alongside + labels: + com.genlayer.health-recovery.logs: "true" + com.genlayer.network: "${NETWORK_NAME:-unknown}" + healthcheck: + test: + - CMD-SHELL + - | + set -eu + boot=$$(awk '{print $$22}' /proc/1/stat) + if curl -fsS --max-time 10 "http://localhost:$${PORT:-4444}/healthcheck?mode=text&url=https%3A%2F%2Ftest-server.genlayer.com%2Fstatic%2Fgenvm%2Fhello.html"; then + printf '%s 0\n' "$$boot" > /tmp/webdriver-health-failures + exit 0 + fi + age=$$(awk -v ticks="$$boot" -v hz="$$(getconf CLK_TCK)" '{print int($$1-ticks/hz)}' /proc/uptime) + [ "$$age" -ge 10 ] || exit 1 + read -r previous failures 2>/dev/null < /tmp/webdriver-health-failures || failures=0 + [ "$${previous:-}" = "$$boot" ] || failures=0 + failures=$$((failures + 1)) + printf '%s %s\n' "$$boot" "$$failures" > /tmp/webdriver-health-failures + if [ "$$failures" -ge 4 ] && [ "$${HEALTHCHECK_AUTO_RESTART:-true}" != false ]; then + printf 'Container webdriver (%s) found to be unhealthy - Restarting container now\n' "$$(cat /etc/hostname)" > /proc/1/fd/1 + children=$$(cat /proc/1/task/1/children) + for child in $$children; do kill -KILL "$$child"; done + fi + exit 1 + interval: 30s + timeout: 12s + retries: 4 + start_period: 10s + # docker-socket-proxy — read-only Docker API gateway for the node's WebDriver + # resource stats and Alloy's recovery-log reader. Runs with the `node` OR + # `monitoring` profile, so it comes up alongside # a node container AND alongside a monitoring/telemetry bring-up — the latter is # how a binary/systemd node (node not containerized) gets its webdriver metrics. # - # WHY THIS EXISTS — and why it is NOT a new exposure or a "leak": + # The node and Alloy use this read-only gateway instead of the host socket. # Reading container stats requires the Docker API. Previously the node reached # the host /var/run/docker.sock directly (bind-mounted into the container, or via # docker-group membership for a binary node) — but that grants the FULL Docker # API as root (a `:ro` flag does NOT restrict API calls), so a node compromise - # could escalate to host root. This proxy REPLACES that: it is the only thing - # touching the socket, restricted to read-only container endpoints (CONTAINERS=1; - # POST=0 => it returns 403 for create / exec / images / everything else). Net - # exposure goes DOWN, not up. + # could escalate to host root. This proxy REPLACES the node's socket access, + # restricted to read-only container endpoints (CONTAINERS=1; + # POST=0 => it returns 403 for create / exec / image operations). + # Health checks stop only their own service process; recovery has no Docker API access. # - Published ONLY on 127.0.0.1:2375 (host loopback), never 0.0.0.0: 2375 is # the unauthenticated Docker API, so it must not leave the host. A container # node reaches it over the compose network (tcp://docker-socket-proxy:2375); @@ -899,14 +924,15 @@ services: # socket: if DOCKER_HOST is unset the node skips webdriver metrics entirely # (non-fatal). Note those are the webdriver container's CPU/memory/traffic series # (genlayer_node_*{component="webdriver"}) that feed validator resource dashboards — - # so keep DOCKER_HOST set to keep those panels populated. To disable metrics - # instead: set metrics.collectors.webdriver.enabled: false and delete this service. + # so keep DOCKER_HOST set to keep those panels populated. Disabling metrics + # permits removing the node's DOCKER_HOST; Alloy still needs this gateway. docker-socket-proxy: - image: tecnativa/docker-socket-proxy:0.3.0 + image: tecnativa/docker-socket-proxy:v0.5.0 container_name: genlayer-node-docker-proxy environment: CONTAINERS: 1 # GET /containers/{json,inspect,stats} — read-only via POST=0 VERSION: 1 # required by the Docker client's API-version negotiation + NETWORKS: 1 # read-only network metadata required by Alloy Docker discovery volumes: - /var/run/docker.sock:/var/run/docker.sock:ro ports: @@ -928,23 +954,27 @@ services: - monitoring genlayer-node: image: yeagerai/genlayer-node:${NODE_VERSION:-latest} - # The keystore password is NOT on this command line. It reaches the node as + # No entrypoint here: the image's own entrypoint precompiles GenVM for the + # host CPU on the first start, then runs `genlayernode run`. The keystore + # password is NOT on a command line. It reaches the node as # GENLAYERNODE_PASSWORD (environment below), which the run command reads in # place of --password. A command-line argument is visible in process # listings on the host; an environment variable is not. - entrypoint: ["/app/bin/genlayernode", "run"] container_name: genlayer-node restart: unless-stopped env_file: - path: ./.env required: false environment: - # ":?" makes Compose refuse to start when NODE_PASSWORD is unset or empty, - # before any container runs. There is no fallback: this is the password - # that encrypts the validator's signing key, and a default shared by every - # operator who forgot to set one is a key anyone can open. The release - # binary carries no default either — it is compiled out of production. - GENLAYERNODE_PASSWORD: "${NODE_PASSWORD:?NODE_PASSWORD is not set. It is the keystore password that unlocks the validator key. Set it in .env; see README section 4.}" + # No default: this is the password that encrypts the validator's signing + # key, and a default shared by every operator who forgot to set one is a + # key anyone can open (the release binary carries none either). The node + # itself refuses validator mode with an empty password ("password is + # required for validator mode"); a full node needs none. Not a ":?" guard: + # Compose interpolates every service, also under another profile, so a + # guard here would stop `docker compose up -d webdriver-container` (the + # binary install path) and a full node, which never read it. + GENLAYERNODE_PASSWORD: "${NODE_PASSWORD:-}" # WebDriver metrics collector reaches Docker via the read-only proxy # (see docker-socket-proxy above), never the host socket directly. DOCKER_HOST: tcp://docker-socket-proxy:2375 @@ -953,8 +983,11 @@ services: - "${NODE_OPS_PORT:-9153}:9153" volumes: - ${NODE_CONFIG_PATH:-./configs/node/config.yaml}:/app/configs/node/config.yaml:ro + # The trusted protocol manifest of the network; consensus.manifestpath names this path. + - ${NODE_MANIFEST_PATH:-./configs/node/protocol-manifest.json}:/app/configs/node/protocol-manifest.json:ro - ${NODE_DATA_PATH:-./data}:/app/data - - ./genvm-module-web-docker.yaml:/app/opt/genvm/${GENVM_VERSION}/config/genvm-module-web.yaml + # opt/genvm/current links to the image's GenVM tree, so no version variable is needed. + - ./genvm-module-web-docker.yaml:/app/opt/genvm/current/config/genvm-module-web.yaml security_opt: - no-new-privileges:true cap_drop: @@ -981,7 +1014,7 @@ services: webdriver_host: http://webdriver-container:4444 ``` - The v0.6.0-rc.1 compose file mounts it over the image's own copy at `/app/opt/genvm/${GENVM_VERSION}/config/genvm-module-web.yaml`. Set `GENVM_VERSION=v0.6.0-rc10` in `.env`: without it the file lands in the wrong place and GenVM cannot reach the WebDriver. The binary keeps using `opt/genvm/v0.6.0-rc10/config/genvm-module-web.yaml`, which points at `localhost:4444`. + The compose file mounts it over the image's own copy at `/app/opt/genvm/current/config/genvm-module-web.yaml`, so no version variable is needed. The binary keeps using `opt/genvm/current/config/genvm-module-web.yaml`, which points at `localhost:4444`. 3. Set the required environment variables in a `.env` file: @@ -1004,8 +1037,9 @@ WEBDRIVER_PORT=4444 # Docker image version NODE_VERSION=latest -# Keystore password (unlocks the pre-imported validator wallet). Required — the -# stack refuses to start without it. Choose your own; never reuse an example +# Keystore password (unlocks the pre-imported validator wallet). Required for a +# validator: the node refuses validator mode without it. A full node and the +# WebDriver-only stack need none. Choose your own; never reuse an example # value. It reaches the node as an environment variable, not a command-line # argument. See README section 4 for reading it in without echo. NODE_PASSWORD= @@ -1013,6 +1047,9 @@ NODE_PASSWORD= # Path to node configuration file NODE_CONFIG_PATH=./configs/node/config.yaml +# Path to the trusted protocol manifest of your network +NODE_MANIFEST_PATH=./configs/node/protocol-manifest.json + # Path to node data directory (for persistence) NODE_DATA_PATH=./data @@ -1027,6 +1064,11 @@ ANTHROPICKEY= XAIKEY= GEMINIKEY= ATOMAKEY= +COMPUT3KEY= +IOINTELLIGENCEKEY= +LIBERTAI_API_KEY= +MORPHEUS_API_KEY= +OPENAIKEY= # Optional logging level override GENLAYERNODE_LOGGING_LEVEL=INFO @@ -1082,6 +1124,9 @@ CENTRAL_LOKI_PASSWORD= # Node identification NODE_ID=validator-001 VALIDATOR_NAME=MyValidator +# Network label for recovery logs and alert routing; use the deployed network. +# Leave empty for an unknown label, rather than guessing a production network. +NETWORK_NAME= # Node metrics endpoint (optional) # Endpoint where the node exposes its Prometheus metrics @@ -1102,7 +1147,6 @@ NODE_METRICS_ENDPOINT=host.docker.internal:9153 # Example for multiple nodes: # SCRAPE_TARGETS_JSON=[{"__address__":"host.docker.internal:9250","instance":"0x...","validator_name":"node-1"},{"__address__":"host.docker.internal:9251","instance":"0x...","validator_name":"node-2"}] # -# To generate for local nodes, use: ./generate-dev-targets.sh # SCRAPE_TARGETS_JSON= # ============================================================================= @@ -1122,8 +1166,8 @@ LOG_FILE_PATTERN=/var/log/genlayer/node*.log # Metrics scraping intervals (optional) # How often to scrape metrics from the node -# Default: 15s for node metrics, 10s timeout -METRICS_SCRAPE_INTERVAL=15s +# Default: 60s for node metrics, 10s timeout +METRICS_SCRAPE_INTERVAL=60s METRICS_SCRAPE_TIMEOUT=10s # Alloy internal health monitoring interval (optional) @@ -1134,9 +1178,9 @@ ALLOY_SELF_MONITORING_INTERVAL=60s # Loki log batching configuration (optional) # Controls how logs are batched before sending to central Loki -# Default: 1MiB batch size, 1s wait time +# Default: 1MiB batch size, 60s wait time LOKI_BATCH_SIZE=1MiB -LOKI_BATCH_WAIT=1s +LOKI_BATCH_WAIT=60s # ============================================================================= # Deployment Instructions @@ -1169,16 +1213,17 @@ LOKI_BATCH_WAIT=1s # - Grafana Alloy handles both metrics and logs in a single service # - To customize monitoring, edit alloy-config.river # - To filter metrics (reduce bandwidth), uncomment the filtering rule in alloy-config.river -# - Ensure firewall allows outbound connections to monitoring server (port 8180) +# - Ensure the firewall allows outbound HTTPS to CENTRAL_MONITORING_URL and CENTRAL_LOKI_URL # + +# Keep health checks but disable automatic container-local recycling when false. +HEALTHCHECK_AUTO_RESTART=true ``` This is the release package's `.env.example`: copy it to `.env` next to `docker-compose.yaml`. The release package also ships `docker-compose.yaml` and `genvm-module-web-docker.yaml`. Set at least: -- `NODE_PASSWORD`: the keystore password, with no default. In v0.6.0-rc.1 every `docker compose` command fails while it is empty, also for a full node or the WebDriver alone. A validator sets the operator key's password (the export passphrase); a full node sets any non-empty value. -- `GENVM_VERSION=v0.6.0-rc10`: the GenVM version v0.6.0-rc.1 ships. The compose file uses it to mount `genvm-module-web-docker.yaml`; `.env.example` does not list it. -- `COMPOSE_PROJECT_NAME=genlayer-node`: the v0.6.0-rc.1 compose file has no fixed project name, so Compose names the project after the directory. With a fixed name, a new version run from a new directory replaces the old containers instead of failing with "container name ... already in use". Passing `-p genlayer-node` to every `docker compose` command does the same. -- `NODE_VERSION`: pin the exact version approved for Bradbury Phase 2, today `NODE_VERSION=v0.6.0-rc.1`. The `latest` image can belong to an older release family. +- `NODE_PASSWORD`: the keystore password. A validator needs it: the node refuses validator mode without a password, and there is no default. A full node and the WebDriver-only stack need none. +- `NODE_VERSION`: pin the exact version approved for Bradbury Phase 2. The `latest` image can belong to an older release family. - the API key of at least one LLM provider. - `GENLAYERNODE_ROLLUP_GENLAYERCHAINRPCKEY`, when your chain RPC provider needs a key (the Bradbury WebSocket does). @@ -1223,7 +1268,7 @@ docker compose --profile node up -d For a quick setup using Docker Compose: -1. **Set the monitoring variables in your `.env` file.** The `.env` example above lists them: the six `CENTRAL_*` endpoint and credential values, `NODE_ID` and `VALIDATOR_NAME`. All six `CENTRAL_*` values are required: Alloy stops with a message when one is missing. Get the endpoints and credentials from the Foundation team, and see the [Monitoring Guide](/validators/monitoring) for the values to use. +1. **Set the monitoring variables in your `.env` file.** The `.env` example above lists them: the six `CENTRAL_*` endpoint and credential values, `NODE_ID`, `VALIDATOR_NAME` and `NETWORK_NAME`. All six `CENTRAL_*` values are required: Alloy stops with a message when one is missing. Get the endpoints and credentials from the Foundation team, and see the [Monitoring Guide](/validators/monitoring) for the values to use. 2. **Run docker compose**: ```bash copy diff --git a/pages/validators/upgrade.mdx b/pages/validators/upgrade.mdx index 3bd3826b..a173377f 100644 --- a/pages/validators/upgrade.mdx +++ b/pages/validators/upgrade.mdx @@ -55,16 +55,15 @@ Breaking changes involve a new consensus deployment. The old database is incompa - Bradbury Phase 2 is a new deployment on a new chain, with a new `consensusaddress` and a new staking contract. Phase 1 validators are not carried over: join again as a validator, with stake, on Phase 2. Follow the [breaking upgrade](#breaking-change-upgrade) steps. - The node needs the trusted protocol manifest of the network. Set `consensus.manifestpath` (default `configs/node/protocol-manifest.json`) and `consensus.chainid`. The release package has no network files: take the manifest and values from [Network-Specific Consensus Configuration](/validators/setup-guide#network-specific-consensus-configuration). -- GenVM moved from `third_party/genvm` to `opt/genvm/` (v0.6.0-rc.1: `opt/genvm/v0.6.0-rc10`). Set `genvm.root_dir: ./opt/genvm/v0.6.0-rc10` and run `python3 ./opt/genvm/v0.6.0-rc10/bin/setup.py` (see [download instructions](/validators/setup-guide#download-the-node-software)). +- GenVM moved from `third_party/genvm` to `opt/genvm/`, with the link `opt/genvm/current` to it. Set `genvm.root_dir: ./opt/genvm/current` and run `python3 ./opt/genvm/current/bin/setup.py` (see [download instructions](/validators/setup-guide#download-the-node-software)). - The Bradbury RPC WebSocket needs an API key. Set it in the environment as `GENLAYERNODE_ROLLUP_GENLAYERCHAINRPCKEY`; it has no `config.yaml` key. See [Testnet Bradbury](/validators/setup-guide#testnet-bradbury). - The `genlayer_debug` (`gen_dbg_*`) and `zksync` (`zks_*`) RPC groups are off in the default `config.yaml` (`node.rpc.endpoints`). - The data store is split into `genvm.db` and `ops.db`. On startup the node renames an old `genlayer.db` to `genvm.db`; it refuses to start if both exist. -- Docker: `NODE_PASSWORD` has no default, and in v0.6.0-rc.1 every `docker compose` command fails while it is empty. It reaches the node as `GENLAYERNODE_PASSWORD`, not as `--password`. See [Running the Node using docker-compose](/validators/setup-guide#running-the-node-using-docker-compose). -- Docker: the v0.6.0-rc.1 compose file does not mount the protocol manifest. Mount it in a `docker-compose.override.yml` (see [Running the Node using docker-compose](/validators/setup-guide#running-the-node-using-docker-compose)). +- Docker: `NODE_PASSWORD` has no default, and a validator needs it: the node refuses validator mode without a password. It reaches the node as `GENLAYERNODE_PASSWORD`, not as `--password`. See [Running the Node using docker-compose](/validators/setup-guide#running-the-node-using-docker-compose). - Docker: the node container runs as uid `10001`. Make `./data` writable by it once: `sudo chown -R 10001:10001 ./data`. - Docker: the node no longer mounts `/var/run/docker.sock`. It reaches Docker through the read-only `docker-socket-proxy` service. -- Docker: if you mount your own GenVM config files (for example a customized LLM config for greyboxing, in a `docker-compose.override.yml`), move every mount from `/app/third_party/genvm/config/` to `/app/opt/genvm/v0.6.0-rc10/config/` (v0.6.0-rc.1). v0.6 no longer reads the old path, so a mount left there is ignored without an error and the image's default config loads. -- Docker: the v0.6.0-rc.1 compose file has no fixed project name. Set `COMPOSE_PROJECT_NAME=genlayer-node` in `.env` (or pass `-p genlayer-node`), so `docker compose up` from a new version's directory replaces the containers of the previous version instead of failing with "container name ... already in use". +- Docker: if you mount your own GenVM config files (for example a customized LLM config for greyboxing, in a `docker-compose.override.yml`), move every mount from `/app/third_party/genvm/config/` to `/app/opt/genvm/current/config/`. v0.6 no longer reads the old path, so a mount left there is ignored without an error and the image's default config loads. +- Docker: the compose project is named `genlayer-node`, so `docker compose up` from a new version's directory replaces the containers of the previous version instead of failing with "container name ... already in use". - Monitoring: the `CENTRAL_*` URLs and credentials have no defaults. Alloy stops at startup if any is unset. See [Monitoring](/validators/monitoring). - Do not use `genlayernode run --upgrade` in v0.6: the call that resumes the node is not available yet, so the node waits forever.