Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 37 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@ STOVEPIPE_STACK_COMPOSE_FILE = example/stovepipe/docker-compose.yml
# Fixed project name for local manual testing (tests use unique random names)
STOVEPIPE_LOCAL_PROJECT = stovepipe

# Runway compose files
RUNWAY_ORCHESTRATOR_COMPOSE_FILE = example/runway/orchestrator/server/docker-compose.yml

# Fixed project name for local manual testing (tests use unique random names)
RUNWAY_LOCAL_PROJECT = runway

# yamlfmt version for YAML formatting (override with: make fmt YAMLFMT_VERSION=v0.16.0)
YAMLFMT_VERSION ?= v0.16.0

Expand All @@ -45,7 +51,7 @@ define assert_clean
fi
endef

.PHONY: build build-all-linux build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-gateway-linux build-stovepipe-orchestrator-linux check-gazelle check-mocks check-tidy clean clean-proto deps e2e-test fmt gazelle integration-test integration-test-submitqueue-consumer integration-test-extensions integration-test-submitqueue-gateway integration-test-submitqueue-orchestrator license-fix lint lint-fmt lint-license local-submitqueue-clean local-submitqueue-gateway-start local-submitqueue-gateway-stop local-init-submitqueue-schemas local-init-stovepipe-queue-schema local-submitqueue-logs local-submitqueue-orchestrator-start local-submitqueue-orchestrator-stop local-submitqueue-ps local-submitqueue-restart local-submitqueue-start local-stop local-stovepipe-gateway-start local-stovepipe-orchestrator-start local-stovepipe-start mocks proto query-deps query-targets run-client-submitqueue-gateway run-client-submitqueue-orchestrator run-client-stovepipe-gateway run-client-stovepipe-orchestrator run-queue-admin test test-no-cache tidy tidy-bazel tidy-go help
.PHONY: build build-all-linux build-runway-orchestrator-linux build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-gateway-linux build-stovepipe-orchestrator-linux check-gazelle check-mocks check-tidy clean clean-proto deps e2e-test fmt gazelle integration-test integration-test-submitqueue-consumer integration-test-extensions integration-test-submitqueue-gateway integration-test-submitqueue-orchestrator license-fix lint lint-fmt lint-license local-init-runway-queue-schema local-submitqueue-clean local-submitqueue-gateway-start local-submitqueue-gateway-stop local-init-submitqueue-schemas local-init-stovepipe-queue-schema local-submitqueue-logs local-submitqueue-orchestrator-start local-submitqueue-orchestrator-stop local-submitqueue-ps local-submitqueue-restart local-submitqueue-start local-stop local-runway-orchestrator-start local-stovepipe-gateway-start local-stovepipe-orchestrator-start local-stovepipe-start mocks proto query-deps query-targets run-client-submitqueue-gateway run-client-submitqueue-orchestrator run-client-stovepipe-gateway run-client-stovepipe-orchestrator run-queue-admin test test-no-cache tidy tidy-bazel tidy-go help


build: ## Build all services and examples
Expand All @@ -54,9 +60,17 @@ build: ## Build all services and examples
@echo "Build complete!"

# Build Linux binaries required for Docker containers
build-all-linux: build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-gateway-linux build-stovepipe-orchestrator-linux ## Build all Linux binaries for Docker
build-all-linux: build-runway-orchestrator-linux build-submitqueue-gateway-linux build-submitqueue-orchestrator-linux build-stovepipe-gateway-linux build-stovepipe-orchestrator-linux ## Build all Linux binaries for Docker
@echo "All Linux binaries ready for Docker"

build-runway-orchestrator-linux: ## Build Runway orchestrator Linux binary for Docker
@echo "Building Runway orchestrator Linux binary for Docker..."
@$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //example/runway/orchestrator/server:orchestrator
@mkdir -p .docker-bin
@cp -f bazel-bin/example/runway/orchestrator/server/orchestrator_/orchestrator .docker-bin/runway-orchestrator 2>/dev/null || \
cp -f bazel-bin/example/runway/orchestrator/server/orchestrator .docker-bin/runway-orchestrator
@echo "Runway orchestrator Linux binary ready at .docker-bin/runway-orchestrator"

build-submitqueue-gateway-linux: ## Build Gateway Linux binary for Docker
@echo "Building Gateway Linux binary for Docker..."
@$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //example/submitqueue/gateway/server:gateway
Expand Down Expand Up @@ -215,6 +229,14 @@ local-init-stovepipe-queue-schema: ## Apply queue schema only (mysql-queue) for
done
@echo "✅ Stovepipe queue schema applied successfully"

local-init-runway-queue-schema: ## Apply queue schema for Runway compose stacks
@echo "Applying queue schema to mysql-queue (Runway; no app database)..."
@for file in extension/messagequeue/mysql/schema/*.sql; do \
echo " - Applying $$(basename $$file)..."; \
docker exec -i $(RUNWAY_LOCAL_PROJECT)-mysql-queue-1 mysql -uroot -proot submitqueue < $$file 2>&1 | grep -v "Using a password" || true; \
done
@echo "✅ Runway queue schema applied successfully"

local-submitqueue-logs: ## View logs from all running services
@$(COMPOSE) -f $(COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) logs -f

Expand Down Expand Up @@ -280,8 +302,21 @@ local-stop: ## Stop all services (keep data)
@echo "Stopping all services..."
@$(COMPOSE) -f $(COMPOSE_FILE) -p $(SUBMITQUEUE_LOCAL_PROJECT) down
@$(COMPOSE) -f $(STOVEPIPE_STACK_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) down
@$(COMPOSE) -f $(RUNWAY_ORCHESTRATOR_COMPOSE_FILE) -p $(RUNWAY_LOCAL_PROJECT) down
@echo "Services stopped. Data volumes preserved."

local-runway-orchestrator-start: build-runway-orchestrator-linux ## Start Runway orchestrator locally (orchestrator + MySQL queue)
@echo "Starting Runway orchestrator with compose..."
@$(COMPOSE) -f $(RUNWAY_ORCHESTRATOR_COMPOSE_FILE) -p $(RUNWAY_LOCAL_PROJECT) up -d --build --wait
@echo "Applying queue schema..."
@$(MAKE) -s local-init-runway-queue-schema
@echo ""
@echo "✅ Runway orchestrator is running!"
@echo ""
@$(COMPOSE) -f $(RUNWAY_ORCHESTRATOR_COMPOSE_FILE) -p $(RUNWAY_LOCAL_PROJECT) ps
@echo ""
@echo "MySQL Queue port: $$(docker port $(RUNWAY_LOCAL_PROJECT)-mysql-queue-1 3306 2>/dev/null | cut -d: -f2 || echo 'unknown')"

local-stovepipe-logs: ## View logs from all running Stovepipe services
@$(COMPOSE) -f $(STOVEPIPE_STACK_COMPOSE_FILE) -p $(STOVEPIPE_LOCAL_PROJECT) logs -f

Expand Down
30 changes: 30 additions & 0 deletions example/runway/orchestrator/server/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
load("@rules_go//go:def.bzl", "go_binary", "go_library")

go_library(
name = "server_lib",
srcs = ["main.go"],
importpath = "github.com/uber/submitqueue/example/runway/orchestrator/server",
visibility = ["//visibility:private"],
deps = [
"//core/consumer",
"//core/errs",
"//core/errs/generic",
"//core/errs/mysql",
"//extension/messagequeue",
"//extension/messagequeue/mysql",
"//runway/core/topickey",
"//runway/extension/vcs",
"//runway/extension/vcs/noop",
"//runway/orchestrator/controller/check",
"//runway/orchestrator/controller/land",
"@com_github_go_sql_driver_mysql//:mysql",
"@com_github_uber_go_tally//:tally",
"@org_uber_go_zap//:zap",
],
)

go_binary(
name = "server",
embed = [":server_lib"],
visibility = ["//visibility:public"],
)
11 changes: 11 additions & 0 deletions example/runway/orchestrator/server/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
FROM debian:bookworm-slim

RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/*
WORKDIR /root/

# Built via: make build-runway-orchestrator-linux
COPY .docker-bin/runway-orchestrator ./orchestrator

EXPOSE 8080

CMD ["./orchestrator"]
43 changes: 43 additions & 0 deletions example/runway/orchestrator/server/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# Docker Compose for Runway orchestrator manual testing
#
# Runway is stateless — no application database, only the queue infrastructure.
#
# IMPORTANT: Before running compose, build the Linux binary:
# make build-runway-orchestrator-linux
#
# Quick start:
# make local-runway-orchestrator-start

services:
# Queue Database - Messaging infrastructure (messages, offsets, partition leases)
# Runway has no application database — it is stateless.
mysql-queue:
image: mysql:8.0
environment:
MYSQL_ROOT_PASSWORD: root
MYSQL_DATABASE: submitqueue
ports:
- "3306" # Random ephemeral port to avoid conflicts
healthcheck:
# Use 127.0.0.1 (TCP) instead of localhost (Unix socket). MySQL treats
# "localhost" as a socket connection, which can be ready before the TCP
# listener — causing dependent services that connect over TCP to fail.
test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-proot"]
interval: 5s
timeout: 5s
retries: 10

orchestrator-service:
build:
context: ${REPO_ROOT}
dockerfile: example/runway/orchestrator/server/Dockerfile
ports:
- "8080" # Random ephemeral port to avoid conflicts
environment:
- PORT=:8080
# Queue infrastructure connection (separate database)
- QUEUE_MYSQL_DSN=root:root@tcp(mysql-queue:3306)/submitqueue?parseTime=true
- HOSTNAME=runway-orchestrator-dev
depends_on:
mysql-queue:
condition: service_healthy
Loading