diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..5c04aa0 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,68 @@ +name: Publish + +on: + push: + tags: ["v*"] + +permissions: + contents: read + +jobs: + pypi: + name: Publish to PyPI + runs-on: ubuntu-latest + permissions: + id-token: write + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.14' + + - name: Install uv + uses: astral-sh/setup-uv@v3 + + - name: Build package + run: uv build + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + docker: + name: Publish Docker image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=latest + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..c473127 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,33 @@ +# Changelog + +All notable changes to this project are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.1.0] - 2026-10-09 + +Initial release. + +### Added + +- Celery event consumer that appends task and worker events to PostgreSQL as an + append-only audit log. +- Async REST API (FastAPI) for querying task history and performing task, + worker, and queue operations. +- MCP server exposing 28 tools as a thin wrapper over the REST API. +- Event-sourced task state with timelines, retry chains, and orphan detection. +- Task actions: revoke, retry, and execute tasks by name. +- Worker management: list, inspect, scale, restart, and shut down workers. +- Queue monitoring for any kombu broker (message and consumer counts). +- Declarative workflow automations (trigger → conditions → actions) with + cooldowns, rate limits, and circuit breakers. +- Prometheus metrics at `/metrics`. +- Optional API key authentication for the REST API and MCP server. +- Docker Compose setup with an optional `demo` profile and a bundled demo + Celery workload. + +[Unreleased]: https://github.com/KalvadTech/taskowl/compare/v0.1.0...HEAD +[0.1.0]: https://github.com/KalvadTech/taskowl/releases/tag/v0.1.0 diff --git a/Dockerfile b/Dockerfile index 29033ee..42ed012 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,7 @@ RUN uv sync --locked --no-dev --no-install-project COPY README.md ./ COPY src ./src +COPY examples ./examples RUN uv sync --locked --no-dev EXPOSE 8000 diff --git a/README.md b/README.md index a94c389..ca9f1e4 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,92 @@ -taskowl +TaskOwl -# taskowl +# TaskOwl -[![Documentation](https://img.shields.io/badge/docs-github_pages-blue)](https://kalvadtech.github.io/taskowl/) +[![CI](https://github.com/KalvadTech/taskowl/actions/workflows/ci.yml/badge.svg)](https://github.com/KalvadTech/taskowl/actions/workflows/ci.yml) +[![Docs](https://img.shields.io/badge/docs-github_pages-blue)](https://kalvadtech.github.io/taskowl/) +[![Python 3.14](https://img.shields.io/badge/python-3.14-blue.svg)](https://www.python.org/) +[![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) +[![Docker](https://img.shields.io/badge/docker-ghcr.io-blue?logo=docker)](https://github.com/KalvadTech/taskowl/pkgs/container/taskowl) -Modern Celery task monitoring with MCP integration. No UI, just data. +**Celery monitoring for humans and AI agents. No UI, just data.** + +Celery gives you workers and tasks. TaskOwl gives you observability, history, +control, and an AI interface for the whole cluster. + +Celery's event stream contains a huge amount of useful operational information, +but once an event has passed, it is gone. TaskOwl turns that stream into a +persistent operational history and exposes it through REST and MCP. + +## Why TaskOwl? + +``` +Celery workers + │ + │ events + ▼ + Broker + │ + ▼ + TaskOwl Consumer + │ + ▼ + PostgreSQL + │ + ├── REST API + │ + ├── Prometheus + │ + └── MCP + │ + ▼ + AI assistant +``` + +TaskOwl subscribes to Celery's events, appends every one to PostgreSQL as an +append-only log, and reconstructs current task, worker, and queue state from it. +The REST API and the MCP server are thin interfaces over that same data — so +scripts, automations, and AI agents all get the same capabilities. + +It is **not** another Flower. No dashboard to click through; instead, durable +history and a control plane you can drive programmatically. See +[Why TaskOwl?](https://kalvadtech.github.io/taskowl/why-taskowl/) for the +full story. + +## Ask your infrastructure + +Point any MCP client at TaskOwl and operate the cluster in natural language +(**28 MCP tools** under the hood): + +```text +You: Show me failed payment tasks in the last hour. + +TaskOwl: 17 failed + 12 payments.charge + 3 payments.refund + 2 payments.capture + + Most common error: + ConnectionError: upstream timeout +``` + +```text +You: Retry the failed payments.charge tasks. + +TaskOwl: Retried 12 tasks. + Retry chain preserved. +``` + +```text +You: Which workers are online? + +TaskOwl: 3 workers online: celery@worker1, celery@worker2, celery@worker3. +``` + +```text +You: Restart the pool on celery@worker1. + +TaskOwl: Pool restarted on celery@worker1. +``` ## Features @@ -20,21 +102,56 @@ Modern Celery task monitoring with MCP integration. No UI, just data. - **PostgreSQL backend**: Production-ready, async throughout - **Broker-agnostic**: RabbitMQ, LavinMQ, Redis, or any Celery/kombu broker -## Documentation +## Quick Start -The full documentation lives on [GitHub Pages](https://kalvadtech.github.io/taskowl/) — -setup, configuration, a complete usage guide, and troubleshooting. +### Try it with Docker (5 minutes) -## Quick Start +The fastest way to see TaskOwl working is the bundled demo: PostgreSQL, +RabbitMQ, TaskOwl, a demo Celery worker, and a task producer. + +```bash +git clone https://github.com/KalvadTech/taskowl.git +cd taskowl +docker compose --profile demo up --build +``` -### Prerequisites +TaskOwl is now running: -- Python 3.14+ -- PostgreSQL 14+ -- A Celery broker (RabbitMQ, LavinMQ, Redis, ...) -- [uv](https://github.com/astral-sh/uv) +| Service | URL | +|---|---| +| REST API | http://localhost:8000 | +| REST API docs | http://localhost:8000/docs | +| MCP server | http://localhost:8001/mcp | +| RabbitMQ management | http://localhost:15672 (guest / guest) | -### Installation +The demo producer continuously creates tasks (including some that fail), so you +can start asking questions immediately. For just the infrastructure without the +demo workload, run `docker compose up --build`. + +### Connect an MCP client + +The MCP server runs on `http://localhost:8001/mcp` (Streamable HTTP). For +[opencode](https://opencode.ai): + +```json +{ + "mcp": { + "taskowl": { + "type": "remote", + "url": "http://localhost:8001/mcp", + "enabled": true, + "oauth": false + } + } +} +``` + +Then ask: *"Show me the current tasks."* + +### Run from source + +Requires Python 3.14+, PostgreSQL 14+, a Celery broker, and +[uv](https://github.com/astral-sh/uv). ```bash git clone https://github.com/KalvadTech/taskowl.git @@ -57,7 +174,7 @@ make mcp # MCP server on :8001 ### Connect your Celery app -taskowl listens to Celery's **events** stream, which workers emit only if enabled: +TaskOwl listens to Celery's **events** stream, which workers emit only if enabled: ```python # celery_app.py @@ -76,25 +193,20 @@ Or start your worker with `-E`: celery -A myapp worker -E --loglevel=info ``` -> **Note**: If events are not enabled, taskowl simply sees nothing — no tasks, +> **Note**: If events are not enabled, TaskOwl simply sees nothing — no tasks, > no workers. -### Connect an MCP client +## Documentation -The MCP server runs on `http://localhost:8001/mcp` (Streamable HTTP). For opencode: +The full documentation lives on [GitHub Pages](https://kalvadtech.github.io/taskowl/) — +setup, configuration, a complete usage guide, security, and troubleshooting. -```json -{ - "mcp": { - "taskowl": { - "type": "remote", - "url": "http://localhost:8001/mcp", - "enabled": true, - "oauth": false - } - } -} -``` +## Security + +TaskOwl can **operate** your cluster, not just observe it: execute tasks, revoke +tasks, restart pools, and shut down workers. Authentication is optional and off +by default — set `API_KEY` before exposing it to any network. See the +[Security](https://kalvadtech.github.io/taskowl/security/) page. ## Contributing @@ -108,4 +220,4 @@ MIT — see [LICENSE](LICENSE) for details. ## Acknowledgments - [Flower](https://github.com/mher/flower) — the original Celery monitor -- [Kanchi](https://github.com/getkanchi/kanchi) — modern Celery monitoring inspiration \ No newline at end of file +- [Kanchi](https://github.com/getkanchi/kanchi) — modern Celery monitoring inspiration diff --git a/docker-compose.yml b/docker-compose.yml index 9f7449c..e6b10d1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,6 +43,37 @@ services: api: condition: service_started + demo-worker: + build: . + profiles: ["demo"] + command: + - uv + - run + - --no-sync + - celery + - -A + - examples.demo.app + - worker + - -E + - --loglevel=info + environment: + <<: *taskowl-env + PYTHONPATH: /app + depends_on: + rabbitmq: + condition: service_healthy + + demo-producer: + build: . + profiles: ["demo"] + command: ["uv", "run", "--no-sync", "python", "-m", "examples.demo.producer"] + environment: + <<: *taskowl-env + PYTHONPATH: /app + depends_on: + demo-worker: + condition: service_started + postgres: image: postgres:18.6 environment: diff --git a/docs/contributing.md b/docs/contributing.md index 787a4ae..43a30fb 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -1,6 +1,6 @@ # Contributing -Thank you for your interest in contributing to taskowl! +Thank you for your interest in contributing to TaskOwl! ## Development setup diff --git a/docs/index.md b/docs/index.md index 9c2f0b3..a2aa534 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,12 +1,14 @@ -# taskowl +# TaskOwl Modern Celery task monitoring with MCP integration. No UI, just data. -taskowl watches your Celery cluster's **event stream**, stores every event in +TaskOwl watches your Celery cluster's **event stream**, stores every event in PostgreSQL as an append-only audit log, and exposes that data — plus task, worker, and queue operations — through a REST API and a set of MCP tools for LLM-driven monitoring and management. +New here? Start with [Why TaskOwl?](why-taskowl.md). + ## Features - **MCP-first**: Query and manage tasks, workers, and queues via the Model Context Protocol @@ -24,12 +26,26 @@ LLM-driven monitoring and management. ## Architecture ``` -Celery workers ──events──▶ Broker ──▶ taskowl consumer ──▶ PostgreSQL - │ - REST API ◀───────────────────────────┘ - ▲ - │ HTTP - MCP server ──▶ LLM / MCP client +Celery workers + │ + │ events + ▼ + Broker + │ + ▼ + TaskOwl Consumer + │ + ▼ + PostgreSQL + │ + ├── REST API + │ + ├── Prometheus + │ + └── MCP + │ + ▼ + AI assistant ``` - **Consumer** (separate process) captures Celery events and appends them to @@ -39,9 +55,13 @@ Celery workers ──events──▶ Broker ──▶ taskowl consumer ──▶ ## Get started -Jump into the [Installation](setup/installation.md) guide to run taskowl, or head +Jump into the [Installation](setup/installation.md) guide to run TaskOwl, or head straight to the [Usage Guide](usage/index.md) for the MCP tools and REST API. !!! note - taskowl only sees what your Celery workers emit. Make sure - [events are enabled](usage/celery-app.md) — otherwise taskowl sees nothing. \ No newline at end of file + TaskOwl only sees what your Celery workers emit. Make sure + [events are enabled](usage/celery-app.md) — otherwise TaskOwl sees nothing. + +!!! warning + TaskOwl can operate your cluster, not just observe it. Read the + [Security](security.md) page before exposing it to a network. diff --git a/docs/security.md b/docs/security.md new file mode 100644 index 0000000..68f5d77 --- /dev/null +++ b/docs/security.md @@ -0,0 +1,125 @@ +# Security + +TaskOwl is not merely an observer. It can **operate** your Celery cluster: run +tasks, revoke them, restart pools, shut down workers, and manage automations that +do all of the above. Treat it as a control plane and secure it accordingly. + +!!! danger + Authentication is **optional and off by default**. If `API_KEY` is not set, + every REST endpoint and the MCP server are open to anyone who can reach + them. Always set `API_KEY` in any environment that is not a local sandbox. + +## Authentication + +TaskOwl has a single shared-secret API key, set via the `API_KEY` environment +variable. When set, all requests to the REST API and the MCP server must include: + +```http +Authorization: Bearer +``` + +- **REST API** — enforced by the `verify_api_key` dependency + (`src/taskowl/auth.py:19`) on every `/api/*` route. +- **MCP server** — enforced by `AuthMiddleware` (`src/taskowl/auth.py:46`) + before any MCP request is processed. + +When `API_KEY` is **unset**, both layers disable themselves and every request is +allowed. This is convenient for a laptop demo and unacceptable in production. + +### Always open endpoints + +Regardless of `API_KEY`, these endpoints are unauthenticated: + +| Endpoint | Why | +|---|---| +| `/health` | Liveness check for orchestrators | +| `/` | Service metadata | +| `/metrics` | Prometheus scraping (see below) | + +## Read-only vs destructive operations + +Everything under `/api/*` requires the API key. The distinction below is about +**impact**, not authorization — a leaked key is enough to run any of them. + +### Read-only + +| Operation | Endpoint | +|---|---| +| List / get / search tasks | `GET /api/tasks`, `GET /api/tasks/{id}` | +| Task types, summary, orphans | `GET /api/tasks/types`, `/summary`, `/orphaned` | +| Task timeline / retry chain | `GET /api/tasks/{id}/timeline`, `/chain` | +| Worker status / stats / list | `GET /api/workers`, `/api/workers/list`, `/{name}/stats` | +| Active / scheduled / reserved tasks | `GET /api/workers/active-tasks`, `/scheduled`, `/reserved` | +| Queues | `GET /api/queues` | +| List / get automations and runs | `GET /api/automations`, `/{id}`, `/{id}/runs`, `/{id}/status` | + +!!! note + Some "read" operations use Celery's control bus to ping workers + (`list_workers`, `get_worker_stats`, active/scheduled/reserved). They are + non-mutating but still require live workers to respond. + +### Destructive / state-changing + +| Operation | Endpoint | Effect | +|---|---|---| +| Revoke a task | `POST /api/tasks/{id}/revoke` | Cancels; `?terminate=true` kills a running task | +| Retry a task | `POST /api/tasks/{id}/retry` | Creates and enqueues a new task | +| Execute a task | `POST /api/tasks/execute` | Runs **any registered task** by name, with arbitrary args | +| Shut down a worker | `POST /api/workers/{name}/shutdown` | Stops a worker after in-flight tasks | +| Scale a worker pool | `POST /api/workers/{name}/scale` | Adds/removes worker processes | +| Restart a worker pool | `POST /api/workers/{name}/restart` | Restarts the pool (optionally reloading modules) | +| Create/update/delete/toggle automation | `POST|PUT|DELETE /api/automations...` | Alters automation behavior, including task execution | + +`execute_task` is the sharpest edge: it does not check task registration, and +the MCP tool exposes it directly. An API key holder, human or AI, can invoke any +task the worker knows about. Restrict who holds the key. + +## The broker is also a control surface + +TaskOwl's worker controls and task sends go through the Celery broker using +`app.control` / `app.send_task`. The API key only protects TaskOwl's HTTP and MCP +interfaces — **anyone with broker access has the same power.** Secure your broker +(credentials, network, vhost ACLs) independently of TaskOwl. + +## `/metrics` exposure + +`/metrics` is intentionally unauthenticated so Prometheus can scrape it without +the TaskOwl API key. It exposes task names, worker hostnames, and automation IDs. +Only expose it to trusted networks, or place it behind a reverse proxy with +network-level or token-based auth. + +## Webhooks and automation definitions + +Automation actions can POST to arbitrary URLs: + +- `slack_webhook` — sends to a Slack-compatible `webhook_url`. +- `webhook` — sends an arbitrary JSON `payload` to a `url`. + +Both support `{event.field}` interpolation resolved at fire time. Treat +automation definitions like secrets: + +- Webhook URLs frequently embed credentials (Slack incoming-webhook URLs are + bearer tokens). Anyone who can read an automation can read its URL. +- Anyone who can `POST /api/automations` can configure TaskOwl to call out to an + arbitrary endpoint on your behalf. Restrict automation write access. + +Use the built-in safety knobs (`cooldown_seconds`, `max_runs_per_window` + +`window_seconds`, `circuit_breaker`) to bound action storms, and review run +history via `GET /api/automations/{id}/runs`. + +## Network exposure checklist + +- Set `API_KEY` everywhere except disposable local sandboxes. +- Bind the API (`TASKOWL_HOST`) and MCP (`MCP_HOST`) servers to private + interfaces, or front them with a reverse proxy that terminates TLS. +- Do not expose `/metrics` publicly. +- Secure PostgreSQL and the broker; both grant control-level access. +- Rotate `API_KEY` like any other credential, and audit who can reach the + automation and task-action endpoints. + +## See also + +- [Configuration](setup/configuration.md) — `API_KEY` and related variables. +- [Task Actions](usage/task-actions.md) — the destructive task operations. +- [Automations](usage/automations.md) — actions and safety controls. +- [Metrics](usage/metrics.md) — the unauthenticated metrics endpoint. diff --git a/docs/setup/configuration.md b/docs/setup/configuration.md index aec2b8d..4ad8852 100644 --- a/docs/setup/configuration.md +++ b/docs/setup/configuration.md @@ -23,7 +23,7 @@ All configuration is via environment variables. ## Brokers -taskowl works with any Celery/kombu broker via `CELERY_BROKER_URL`: +TaskOwl works with any Celery/kombu broker via `CELERY_BROKER_URL`: ```bash export CELERY_BROKER_URL="amqp://guest:guest@localhost:5672//" # RabbitMQ / LavinMQ diff --git a/docs/setup/index.md b/docs/setup/index.md index 967a51a..dd72872 100644 --- a/docs/setup/index.md +++ b/docs/setup/index.md @@ -1,6 +1,6 @@ # Setup -taskowl runs as three separate processes that share a PostgreSQL database and a +TaskOwl runs as three separate processes that share a PostgreSQL database and a Celery broker: - **API** — FastAPI REST server (default port `8000`) diff --git a/docs/setup/installation.md b/docs/setup/installation.md index ab7e8b2..75e8723 100644 --- a/docs/setup/installation.md +++ b/docs/setup/installation.md @@ -1,13 +1,50 @@ # Installation -## Prerequisites +## Docker Compose (recommended) + +The fastest way to run TaskOwl is with Docker Compose, which starts the API, +consumer, and MCP server alongside PostgreSQL and RabbitMQ. + +```bash +git clone https://github.com/KalvadTech/taskowl.git +cd taskowl +docker compose up --build +``` + +The API is on `http://localhost:8000`, the MCP server on +`http://localhost:8001/mcp`, and the RabbitMQ management UI on +`http://localhost:15672` (guest / guest). + +### Try it with the bundled demo + +To also start a demo Celery worker and a task producer — so there is activity to +query immediately — use the `demo` profile: + +```bash +docker compose --profile demo up --build +``` + +This brings up: + +- RabbitMQ +- PostgreSQL +- TaskOwl (API, consumer, MCP) +- a demo Celery worker (`examples/demo`) +- a demo task producer + +Then connect an MCP client to `http://localhost:8001/mcp` and ask +*"Show me the current tasks."* + +## From source + +### Prerequisites - Python 3.14+ - PostgreSQL 14+ - A Celery broker (RabbitMQ, LavinMQ, Redis, ...) - [uv](https://github.com/astral-sh/uv) -## Install and migrate +### Install and migrate ```bash git clone https://github.com/KalvadTech/taskowl.git @@ -20,7 +57,7 @@ export CELERY_BROKER_URL="amqp://guest:guest@localhost:5672//" make migrate ``` -## Run the three processes +### Run the three processes Open three terminals: @@ -30,18 +67,6 @@ make consume # Celery event consumer make mcp # MCP server on :8001 ``` -## Docker Compose - -taskowl ships a `docker-compose.yml` that runs the API, consumer, and MCP -server alongside PostgreSQL and RabbitMQ: - -```bash -docker compose up --build -``` - -The API is on `http://localhost:8000`, the MCP server on -`http://localhost:8001/mcp`. - ## Verify it's working 1. Confirm the consumer connected to the broker: @@ -58,4 +83,8 @@ The API is on `http://localhost:8000`, the MCP server on ``` 3. Make sure your [Celery app emits events](../usage/celery-app.md) — without them - taskowl sees nothing. \ No newline at end of file + TaskOwl sees nothing. + +!!! warning + TaskOwl can operate your cluster, not just observe it. Set `API_KEY` before + exposing it to a network — see [Security](../security.md). diff --git a/docs/usage/celery-app.md b/docs/usage/celery-app.md index 3d201b6..914357d 100644 --- a/docs/usage/celery-app.md +++ b/docs/usage/celery-app.md @@ -1,7 +1,7 @@ # Configuring your Celery app -taskowl listens to Celery's **events** stream, which workers emit only if -enabled. Add this to your Celery application so taskowl can see your tasks and +TaskOwl listens to Celery's **events** stream, which workers emit only if +enabled. Add this to your Celery application so TaskOwl can see your tasks and workers: ```python @@ -29,5 +29,5 @@ celery -A myapp worker -E --loglevel=info ``` !!! note - If events are not enabled, taskowl simply sees nothing — no tasks, no + If events are not enabled, TaskOwl simply sees nothing — no tasks, no workers. Enabling events is the one integration required. \ No newline at end of file diff --git a/docs/usage/index.md b/docs/usage/index.md index 422f373..b93dc20 100644 --- a/docs/usage/index.md +++ b/docs/usage/index.md @@ -1,6 +1,6 @@ # Usage Guide -This guide walks through everything you can do with taskowl: querying tasks, +This guide walks through everything you can do with TaskOwl: querying tasks, managing workers, inspecting queues, and building workflow automations. ## Available MCP tools @@ -39,7 +39,7 @@ Once your MCP client is connected, you can ask: ## Structure -- [Configuring Celery](celery-app.md) — make taskowl see your tasks and workers +- [Configuring Celery](celery-app.md) — make TaskOwl see your tasks and workers - [MCP Clients](mcp-clients.md) — connect opencode or any MCP client - [Tasks](tasks.md) — query tasks, filters, timelines, chains, summaries, orphans - [Task Actions](task-actions.md) — revoke, retry, execute diff --git a/docs/usage/mcp-clients.md b/docs/usage/mcp-clients.md index 5e85745..ff33fec 100644 --- a/docs/usage/mcp-clients.md +++ b/docs/usage/mcp-clients.md @@ -22,5 +22,6 @@ Add a remote MCP server to your `opencode.json`: ## Other MCP clients Point your MCP client at the Streamable HTTP endpoint `http://localhost:8001/mcp`. -If authentication is enabled, send the taskowl API key as -`Authorization: Bearer ` with each request. \ No newline at end of file +If authentication is enabled, send the TaskOwl API key as +`Authorization: Bearer ` with each request. See [Security](../security.md) +for how authentication is enforced. \ No newline at end of file diff --git a/docs/usage/metrics.md b/docs/usage/metrics.md index 84bf845..a5c3c4f 100644 --- a/docs/usage/metrics.md +++ b/docs/usage/metrics.md @@ -1,6 +1,6 @@ # Metrics -taskowl exposes Prometheus metrics at `/metrics` on the API server. +TaskOwl exposes Prometheus metrics at `/metrics` on the API server. ```bash curl http://localhost:8000/metrics @@ -27,5 +27,5 @@ scrape_configs: !!! warning `/metrics` is intentionally unauthenticated so Prometheus can scrape it - without the taskowl API key. Only expose it to trusted networks or behind a - reverse proxy. \ No newline at end of file + without the TaskOwl API key. Only expose it to trusted networks or behind a + reverse proxy. See [Security](../security.md). \ No newline at end of file diff --git a/docs/usage/task-actions.md b/docs/usage/task-actions.md index 8bede63..6608db1 100644 --- a/docs/usage/task-actions.md +++ b/docs/usage/task-actions.md @@ -3,6 +3,11 @@ Write operations on tasks. All require authentication (`API_KEY`) and follow the pattern `actions.py → main.py → mcp/tools.py`. +!!! warning + These operations change cluster state — `execute_task` runs any registered + task by name, and `revoke` can terminate a running task. See + [Security](../security.md) before exposing them. + ## revoke_task Revoke (cancel) a task. Optionally terminate it if it is currently running. diff --git a/docs/why-taskowl.md b/docs/why-taskowl.md new file mode 100644 index 0000000..95623f7 --- /dev/null +++ b/docs/why-taskowl.md @@ -0,0 +1,106 @@ +# Why TaskOwl? + +Celery gives you workers and tasks. **TaskOwl gives you observability, history, +control, and an AI interface for the whole cluster.** + +Celery's event stream contains a huge amount of useful operational information — +task states, retries, worker heartbeats, queue activity — but once an event has +passed, it is gone. TaskOwl turns that stream into a persistent operational +history and exposes it through REST and MCP. + +## Before / after + +**Without TaskOwl** + +```text +Celery + ├── workers + ├── broker + └── events → mostly ephemeral +``` + +**With TaskOwl** + +```text +Celery workers + │ + │ events + ▼ + Broker + │ + ▼ + TaskOwl Consumer + │ + ▼ + PostgreSQL + │ + ├── REST API + │ + ├── Prometheus + │ + └── MCP + │ + ▼ + AI assistant +``` + +Celery's events are emitted only if enabled. TaskOwl's consumer subscribes to +that stream, appends every event to PostgreSQL as an append-only log, and +reconstructs current task, worker, and queue state from it. The REST API and the +MCP server are thin read/write interfaces over that same data. + +## TaskOwl is not another Flower + +Flower is a great real-time Celery monitor, and TaskOwl was inspired by it. But +the two solve different problems. TaskOwl is built around **durable history** and +**programmatic/AI-driven operations**, and deliberately ships **no UI**. + +| | Flower | TaskOwl | +| --------------------------- | --------- | ------- | +| Real-time Celery monitoring | ✓ | ✓ | +| Persistent event history | | ✓ | +| REST API | | ✓ | +| MCP | | ✓ | +| AI-assisted operations | | ✓ | +| Task retry/revoke/execute | ✓/partial | ✓ | +| Worker management | ✓ | ✓ | +| Queue monitoring | ✓ | ✓ | +| Declarative automations | | ✓ | +| Prometheus | | ✓ | +| UI required | ✓ | **No** | + +If you want a dashboard to click through, use Flower. If you want your Celery +cluster's history and controls available to scripts, automations, and AI agents, +use TaskOwl. They can happily run side by side. + +## Design principles + +**MCP-first** +The system is designed to be operated programmatically and through AI agents, +not through a GUI. + +**PostgreSQL as the source of operational history** +Celery events become durable data rather than transient messages. + +**API-first** +MCP is a thin interface over the REST API, so automation and human tooling use +the same capabilities. Everything you can do via MCP you can also do with `curl`. + +**No UI** +TaskOwl focuses on exposing operational data and control. Use Grafana, your +existing observability stack, an MCP client, or build your own interface. + +**Async throughout** +Built for modern Python deployments. + +## What TaskOwl is not + +TaskOwl is intentionally scoped. It is not a dashboard, an authentication UI, a +multi-tenant platform, a log aggregator, or a distributed tracing system. Those +are valuable, but they are not this project. **No UI, just data.** + +## Next steps + +- [Install TaskOwl](setup/installation.md) — the Docker path takes about five minutes. +- [Usage Guide](usage/index.md) — the MCP tools and REST API. +- [Security](security.md) — TaskOwl can operate your cluster, not just observe it. diff --git a/examples/__init__.py b/examples/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/demo/__init__.py b/examples/demo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/examples/demo/app.py b/examples/demo/app.py new file mode 100644 index 0000000..a2c3abf --- /dev/null +++ b/examples/demo/app.py @@ -0,0 +1,61 @@ +"""A tiny demo Celery app for TaskOwl. + +Start a worker with events enabled: + + celery -A examples.demo.app worker -E --loglevel=info + +The tasks here are intentionally varied so the event stream (and therefore +TaskOwl's history) has something interesting to show: quick successes, slow +tasks, and a payment task that fails and retries. +""" + +import os +import random +import time + +from celery import Celery + +broker = os.environ.get("CELERY_BROKER_URL", "amqp://guest:guest@localhost:5672//") + +app = Celery("taskowl_demo", broker=broker) + +app.conf.update( + task_send_sent_event=True, + worker_send_task_events=True, + worker_heartbeat_interval=2, + task_acks_late=True, +) + + +@app.task +def add(x: int, y: int) -> int: + """Add two numbers.""" + return x + y + + +@app.task +def send_email(to: str) -> str: + """Pretend to send an email.""" + time.sleep(random.uniform(0.1, 1.0)) + return f"email sent to {to}" + + +@app.task +def generate_report(days: int) -> dict: + """Pretend to generate a slow report.""" + time.sleep(random.uniform(1.0, 3.0)) + return {"days": days, "rows": random.randint(100, 5000)} + + +@app.task( + bind=True, + autoretry_for=(ConnectionError,), + retry_backoff=True, + max_retries=3, +) +def charge_payment(self, order_id: int, amount: float) -> dict: + """Pretend to charge a payment; fails intermittently to exercise retries.""" + time.sleep(random.uniform(0.2, 1.5)) + if random.random() < 0.4: + raise ConnectionError("upstream timeout") + return {"order_id": order_id, "amount": amount, "status": "captured"} diff --git a/examples/demo/producer.py b/examples/demo/producer.py new file mode 100644 index 0000000..ecb3bc5 --- /dev/null +++ b/examples/demo/producer.py @@ -0,0 +1,28 @@ +"""Continuously produce demo tasks for the TaskOwl demo. + +Run it alongside ``examples.demo.app``: + + python -m examples.demo.producer +""" + +import random +import time + +from examples.demo.app import add, charge_payment, generate_report, send_email + + +def main() -> None: + """Send a steady stream of demo tasks.""" + order_id = 1000 + while True: + add.delay(random.randint(1, 100), random.randint(1, 100)) + send_email.delay(f"user{random.randint(1, 100)}@example.com") + charge_payment.delay(order_id, round(random.uniform(5, 500), 2)) + if random.random() < 0.2: + generate_report.delay(random.choice([7, 30, 90])) + order_id += 1 + time.sleep(random.uniform(1.0, 3.0)) + + +if __name__ == "__main__": + main() diff --git a/mkdocs.yml b/mkdocs.yml index 3584418..2798dd1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,4 +1,4 @@ -site_name: taskowl +site_name: TaskOwl site_description: Modern Celery task monitoring with MCP integration site_url: https://kalvadtech.github.io/taskowl/ repo_url: https://github.com/KalvadTech/taskowl @@ -31,6 +31,7 @@ theme: - content.tabs.link nav: - Overview: index.md + - Why TaskOwl?: why-taskowl.md - Getting Started: - Setup: setup/index.md - Installation: setup/installation.md @@ -46,6 +47,7 @@ nav: - Automations: usage/automations.md - Metrics: usage/metrics.md - Troubleshooting: troubleshooting.md + - Security: security.md - Contributing: contributing.md markdown_extensions: - admonition diff --git a/pyproject.toml b/pyproject.toml index 4a63edf..1ea3373 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,9 @@ dependencies = [ [project.urls] Homepage = "https://github.com/KalvadTech/taskowl" Repository = "https://github.com/KalvadTech/taskowl" -Documentation = "https://github.com/KalvadTech/taskowl" +Documentation = "https://kalvadtech.github.io/taskowl/" +Changelog = "https://github.com/KalvadTech/taskowl/blob/main/CHANGELOG.md" +Issues = "https://github.com/KalvadTech/taskowl/issues" [dependency-groups] dev = [