diff --git a/assets/sourceos/runbooks/cloudshell-devsecops.yaml b/assets/sourceos/runbooks/cloudshell-devsecops.yaml new file mode 100644 index 00000000000..10008e9e512 --- /dev/null +++ b/assets/sourceos/runbooks/cloudshell-devsecops.yaml @@ -0,0 +1,24 @@ +name: cloudshell-devsecops +description: "Cloudshell-fog DevSecOps session — bastion SSH, SOCKS5 proxy, remote exec patterns" +tags: [cloudshell, devsecops, ssh, bastion, proxy, k3s] +vars: + REMOTE_CMD: "kubectl get nodes" + SRC: "~/.kube/config-k3s-twin" + DST: "cloudshell:~/.kube/config-k3s-twin" +steps: + - cmd: csh-status + desc: "Print full cloudshell-fog connection config and live tunnel/proxy status." + - cmd: csh-exec "whoami && hostname && uptime" + desc: "Non-interactive smoke-test — verify bastion reachability and identity." + - cmd: csh-exec "${REMOTE_CMD}" + desc: "Run an arbitrary remote command on the cloudshell bastion. Edit REMOTE_CMD var to taste." + - cmd: ktunnel start + desc: "Forward k3s API port (localhost:16443 → k3s-twin:6443) through the bastion." + - cmd: kproxy start + desc: "Open SOCKS5 proxy on localhost:1080 via cloudshell — routes sovereign browsing through the fog network." + - cmd: csh-copy "${SRC}" "${DST}" + desc: "Copy kubeconfig or any file to/from cloudshell via scp. Set SRC/DST vars: prefix 'cloudshell:' for remote side." + - cmd: csh + desc: "Drop into a full interactive SSH session on the cloudshell-fog bastion (CMD+SHIFT+K)." + - cmd: ktunnel stop + desc: "Tear down the k3s API tunnel cleanly when done." diff --git a/assets/sourceos/runbooks/k3s-cluster-admin.yaml b/assets/sourceos/runbooks/k3s-cluster-admin.yaml new file mode 100644 index 00000000000..fdb9665b05b --- /dev/null +++ b/assets/sourceos/runbooks/k3s-cluster-admin.yaml @@ -0,0 +1,25 @@ +name: k3s-cluster-admin +description: "k3s twin cluster admin — open tunnel, verify connectivity, inspect workloads" +tags: [k3s, devsecops, cloudshell, tunnel, kubectl] +vars: + NAMESPACE: "kube-system" + KUBECONFIG: "${CLOUDSHELL_K3S_KUBECONFIG:-$HOME/.kube/config-k3s-twin}" +steps: + - cmd: csh-status + desc: "Show cloudshell-fog config and current tunnel/proxy state." + - cmd: ktunnel start + desc: "Open k3s API tunnel: localhost:16443 → k3s-twin:6443 via cloudshell-fog bastion. Safe to re-run if already up." + - cmd: k3s get nodes -o wide + desc: "Verify k3s twin reachability and node readiness via the tunnel." + - cmd: k3s get pods -n ${NAMESPACE} --field-selector='status.phase!=Running' + desc: "Surface any pods that are NOT running in the target namespace — the first triage signal." + - cmd: k3s get events -n ${NAMESPACE} --sort-by='.metadata.creationTimestamp' | tail -30 + desc: "Last 30 cluster events sorted by timestamp — quick anomaly scan." + - cmd: k3s top nodes + desc: "CPU/memory pressure across nodes (requires metrics-server)." + - cmd: k3s top pods -n ${NAMESPACE} --sort-by=memory + desc: "Top pods by memory in the target namespace." + - cmd: k3s get all -A --show-labels + desc: "Full cluster inventory with labels — useful for verifying GitOps-applied state." + - cmd: ktunnel status + desc: "Confirm tunnel is still alive after the admin session." diff --git a/assets/sourceos/runbooks/k3s-incident-triage.yaml b/assets/sourceos/runbooks/k3s-incident-triage.yaml new file mode 100644 index 00000000000..5e48240599b --- /dev/null +++ b/assets/sourceos/runbooks/k3s-incident-triage.yaml @@ -0,0 +1,27 @@ +name: k3s-incident-triage +description: "k3s twin incident triage — rapid diagnosis for pod failures, OOM, crashloops, storage" +tags: [k3s, devsecops, incident, triage, cloudshell] +vars: + NAMESPACE: "default" + POD: "" +steps: + - cmd: ktunnel start + desc: "Ensure k3s API tunnel is open. Safe to re-run." + - cmd: k3s get pods -A --field-selector='status.phase!=Running' -o wide + desc: "All non-running pods across every namespace — the incident blast radius at a glance." + - cmd: k3s get events -A --field-selector='type=Warning' --sort-by='.metadata.creationTimestamp' | tail -40 + desc: "Warning events cluster-wide, newest last." + - cmd: k3s describe pod ${POD} -n ${NAMESPACE} + desc: "Full pod description for the affected pod. Set POD var to the pod name." + - cmd: k3s logs ${POD} -n ${NAMESPACE} --previous --tail=100 + desc: "Logs from the previous (crashed) container instance — the real error trace." + - cmd: k3s logs ${POD} -n ${NAMESPACE} --tail=100 + desc: "Logs from the current container instance." + - cmd: k3s get pvc -n ${NAMESPACE} + desc: "PVC status in the affected namespace — catch Pending volumes blocking pod scheduling." + - cmd: k3s describe node $(k3s get pods ${POD} -n ${NAMESPACE} -o jsonpath='{.spec.nodeName}') + desc: "Node conditions and allocatable resources for the pod's assigned node." + - cmd: csh-exec "journalctl -u k3s --since '30 minutes ago' | tail -60" + desc: "k3s systemd service journal on the cloudshell-fog host — catches API server and etcd panics." + - cmd: k3s rollout restart deployment/${POD%-*} -n ${NAMESPACE} + desc: "Controlled rollout restart of the affected deployment. Only run after diagnosing root cause."