diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ac61988..1afeecd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,6 +14,27 @@ on: workflow_dispatch: jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + python-version: "3.11" + enable-cache: true + + - name: Sync dependencies + run: uv sync + + # Check only, never fix. Contributors auto-fix locally via the pre-commit hook. + - name: Ruff lint + run: uv run ruff check . + + - name: Ruff format + run: uv run ruff format --check . + test: runs-on: ubuntu-latest steps: @@ -32,7 +53,7 @@ jobs: run: uv run pytest release: - needs: test + needs: [lint, test] if: github.event_name == 'push' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest concurrency: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..564cdbe --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,19 @@ +# Install with: uv run pre-commit install +# Installs both the pre-commit (ruff) and commit-msg (commitizen) hooks. +default_install_hook_types: [pre-commit, commit-msg] +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. Keep in sync with the dev-group ruff pin in pyproject.toml. + rev: v0.15.21 + hooks: + # Run the linter. + - id: ruff-check + args: [ --fix ] + # Run the formatter. + - id: ruff-format + - repo: https://github.com/commitizen-tools/commitizen + rev: v4.10.1 + hooks: + # Validate Conventional Commit messages (semantic-release parses these). + - id: commitizen + stages: [commit-msg] diff --git a/docs/contributing.md b/docs/contributing.md index af014ff..829adb0 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -23,9 +23,16 @@ You can help improve the project in several ways: This project uses [uv](https://docs.astral.sh/uv/) for dependency management, building, and publishing. - Install dependencies (including the `dev` group): `uv sync` +- Set up the git hooks: `uv run pre-commit install` (wires up both the ruff and + commit-message hooks in one step) - Run the tests: `uv run pytest` - Build the package locally: `uv build` +[ruff](https://docs.astral.sh/ruff/) runs on every commit and auto-fixes lint and +formatting locally. CI runs the same checks in check-only mode, so a PR won't merge +if anything is left unformatted. The commit-message hook rejects commits that don't +follow the Conventional Commits format below. + ## Releases and Commit Messages Releases are fully automated with [python-semantic-release](https://python-semantic-release.readthedocs.io/). The next version, changelog, git tag, GitHub Release, and PyPI upload are all derived from commit messages on `main`, so **do not bump the version by hand**. diff --git a/example/scripts/01_load_training_data.py b/example/scripts/01_load_training_data.py index eec4891..f6935b1 100644 --- a/example/scripts/01_load_training_data.py +++ b/example/scripts/01_load_training_data.py @@ -1,10 +1,10 @@ """ Step 1: Generate synthetic housing data and load into BigQuery offline_features table. """ + import os import uuid import numpy as np -import pandas as pd from datetime import datetime, timezone from pathlib import Path from dotenv import load_dotenv @@ -27,16 +27,16 @@ np.random.seed(RANDOM_SEED) -cities = list(range(5)) # 0-4 representing 5 cities -states = list(range(3)) # 0-2 representing 3 states +cities = list(range(5)) # 0-4 representing 5 cities +states = list(range(3)) # 0-2 representing 3 states -bedrooms = np.random.randint(1, 6, N_ROWS).astype(float) -bathrooms = np.random.randint(1, 4, N_ROWS).astype(float) -area_sqft = np.random.randint(800, 4000, N_ROWS).astype(float) -lot_size = np.random.randint(2000, 10000, N_ROWS).astype(float) +bedrooms = np.random.randint(1, 6, N_ROWS).astype(float) +bathrooms = np.random.randint(1, 4, N_ROWS).astype(float) +area_sqft = np.random.randint(800, 4000, N_ROWS).astype(float) +lot_size = np.random.randint(2000, 10000, N_ROWS).astype(float) year_built = np.random.randint(1960, 2023, N_ROWS).astype(float) -city = np.random.choice(cities, N_ROWS).astype(float) -state = np.random.choice(states, N_ROWS).astype(float) +city = np.random.choice(cities, N_ROWS).astype(float) +state = np.random.choice(states, N_ROWS).astype(float) now = datetime.now(timezone.utc) rows = [ diff --git a/example/scripts/02_train_model.py b/example/scripts/02_train_model.py index a66f9b8..0ad2a3e 100644 --- a/example/scripts/02_train_model.py +++ b/example/scripts/02_train_model.py @@ -1,6 +1,7 @@ """ Step 2: Pull features from BigQuery and train a RandomForest model, logging to MLflow. """ + import os import numpy as np import mlflow @@ -16,18 +17,28 @@ load_dotenv(Path.cwd() / ".env") MLFLOW_URL = os.environ.get("MLFLOW_URL") -PROJECT = os.environ.get("BIGQUERY_PROJECT") +PROJECT = os.environ.get("BIGQUERY_PROJECT") if not MLFLOW_URL or not PROJECT: - raise SystemExit("MLFLOW_URL or BIGQUERY_PROJECT missing. Run `deployml get-urls` to write .env.") -DATASET = os.getenv("BIGQUERY_DATASET", "mlops") -EXPERIMENT = "housing-price-prediction" -FEATURE_COLS = ["bedrooms", "bathrooms", "area_sqft", "lot_size", "year_built", "city", "state"] + raise SystemExit( + "MLFLOW_URL or BIGQUERY_PROJECT missing. Run `deployml get-urls` to write .env." + ) +DATASET = os.getenv("BIGQUERY_DATASET", "mlops") +EXPERIMENT = "housing-price-prediction" +FEATURE_COLS = [ + "bedrooms", + "bathrooms", + "area_sqft", + "lot_size", + "year_built", + "city", + "state", +] # Pull features from BigQuery print(f"Querying BigQuery {PROJECT}.{DATASET}.offline_features ...") client = bigquery.Client(project=PROJECT) -query = f"SELECT {', '.join(FEATURE_COLS)} FROM `{PROJECT}.{DATASET}.offline_features`" -df = client.query(query).to_dataframe() +query = f"SELECT {', '.join(FEATURE_COLS)} FROM `{PROJECT}.{DATASET}.offline_features`" +df = client.query(query).to_dataframe() print(f"āœ“ Loaded {len(df)} rows from BigQuery") # Generate target (same formula as seed_model.py) @@ -42,7 +53,9 @@ X = df[FEATURE_COLS] y = df["price"] -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) +X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.2, random_state=42 +) print(f"Connecting to MLflow at {MLFLOW_URL} ...") mlflow.set_tracking_uri(MLFLOW_URL) @@ -56,8 +69,8 @@ model.fit(X_train, y_train) preds = model.predict(X_test) - rmse = root_mean_squared_error(y_test, preds) - r2 = r2_score(y_test, preds) + rmse = root_mean_squared_error(y_test, preds) + r2 = r2_score(y_test, preds) print("Logging to MLflow ...") mlflow.log_params(params) diff --git a/example/scripts/03_register_model.py b/example/scripts/03_register_model.py index 4d46be8..6ba0c43 100644 --- a/example/scripts/03_register_model.py +++ b/example/scripts/03_register_model.py @@ -1,6 +1,7 @@ """ Step 3: Register the best MLflow run as HousingPriceModel and promote to Production. """ + import os import mlflow from pathlib import Path @@ -12,15 +13,17 @@ MLFLOW_URL = os.environ.get("MLFLOW_URL") if not MLFLOW_URL: raise SystemExit("MLFLOW_URL missing. Run `deployml get-urls` to write .env.") -MODEL_NAME = "HousingPriceModel" -EXPERIMENT = "housing-price-prediction" +MODEL_NAME = "HousingPriceModel" +EXPERIMENT = "housing-price-prediction" mlflow.set_tracking_uri(MLFLOW_URL) client = MlflowClient() experiment = client.get_experiment_by_name(EXPERIMENT) if experiment is None: - raise ValueError(f"Experiment '{EXPERIMENT}' not found. Run 02_train_model.py first.") + raise ValueError( + f"Experiment '{EXPERIMENT}' not found. Run 02_train_model.py first." + ) runs = client.search_runs( experiment_ids=[experiment.experiment_id], @@ -31,13 +34,13 @@ raise ValueError("No runs found. Run 02_train_model.py first.") best_run = runs[0] -run_id = best_run.info.run_id -rmse = best_run.data.metrics.get("rmse") +run_id = best_run.info.run_id +rmse = best_run.data.metrics.get("rmse") print(f"āœ“ Best run: {run_id} RMSE: {rmse:.2f}") model_uri = f"runs:/{run_id}/model" -result = mlflow.register_model(model_uri, MODEL_NAME) -version = result.version +result = mlflow.register_model(model_uri, MODEL_NAME) +version = result.version print(f"āœ“ Registered as '{MODEL_NAME}' version {version}") client.transition_model_version_stage( diff --git a/example/scripts/04_make_predictions.py b/example/scripts/04_make_predictions.py index b349b6a..83c1028 100644 --- a/example/scripts/04_make_predictions.py +++ b/example/scripts/04_make_predictions.py @@ -2,6 +2,7 @@ Step 4: Pull features from BigQuery and send prediction requests to FastAPI. FastAPI automatically logs each prediction to the BigQuery predictions table. """ + import os import requests from pathlib import Path @@ -11,14 +12,16 @@ load_dotenv(Path.cwd() / ".env") FASTAPI_URL = os.environ.get("FASTAPI_URL") -PROJECT = os.environ.get("BIGQUERY_PROJECT") +PROJECT = os.environ.get("BIGQUERY_PROJECT") if not FASTAPI_URL or not PROJECT: - raise SystemExit("FASTAPI_URL or BIGQUERY_PROJECT missing. Run `deployml get-urls` to write .env.") -DATASET = os.getenv("BIGQUERY_DATASET", "mlops") -N_PREDICT = 50 + raise SystemExit( + "FASTAPI_URL or BIGQUERY_PROJECT missing. Run `deployml get-urls` to write .env." + ) +DATASET = os.getenv("BIGQUERY_DATASET", "mlops") +N_PREDICT = 50 -client = bigquery.Client(project=PROJECT) -query = f""" +client = bigquery.Client(project=PROJECT) +query = f""" SELECT entity_id, bedrooms, bathrooms, area_sqft, lot_size, year_built, city, state FROM `{PROJECT}.{DATASET}.offline_features` LIMIT {N_PREDICT} @@ -31,14 +34,14 @@ payload = { "entity_id": row["entity_id"], "features": { - "bedrooms": row["bedrooms"], - "bathrooms": row["bathrooms"], - "area_sqft": row["area_sqft"], - "lot_size": row["lot_size"], + "bedrooms": row["bedrooms"], + "bathrooms": row["bathrooms"], + "area_sqft": row["area_sqft"], + "lot_size": row["lot_size"], "year_built": row["year_built"], - "city": row["city"], - "state": row["state"], - } + "city": row["city"], + "state": row["state"], + }, } resp = requests.post(f"{FASTAPI_URL}/predict", json=payload, timeout=10) if resp.status_code == 200 and resp.json().get("prediction", -1) != -1: diff --git a/example/scripts/05_generate_ground_truth.py b/example/scripts/05_generate_ground_truth.py index 6d51300..ecc86ae 100644 --- a/example/scripts/05_generate_ground_truth.py +++ b/example/scripts/05_generate_ground_truth.py @@ -2,6 +2,7 @@ Step 5: Generate fake ground truth values for each prediction and store in BigQuery. In production this would be real outcomes (e.g. actual sale price) matched back to predictions. """ + import os import numpy as np from pathlib import Path @@ -15,7 +16,7 @@ if not PROJECT: raise SystemExit("BIGQUERY_PROJECT missing. Run `deployml get-urls` to write .env.") DATASET = os.getenv("BIGQUERY_DATASET", "mlops") -NOISE = 15000 # std dev of fake noise around predicted value +NOISE = 15000 # std dev of fake noise around predicted value client = bigquery.Client(project=PROJECT) @@ -28,12 +29,12 @@ print(f"āœ“ Found {len(predictions)} predictions to generate ground truth for") np.random.seed(0) -now = datetime.now(timezone.utc) +now = datetime.now(timezone.utc) rows = [ { - "entity_id": row["entity_id"], + "entity_id": row["entity_id"], "event_timestamp": now.isoformat(), - "actual_value": float(row["predicted_value"]) + np.random.normal(0, NOISE), + "actual_value": float(row["predicted_value"]) + np.random.normal(0, NOISE), } for row in predictions ] diff --git a/example/scripts/06_compute_drift_metrics.py b/example/scripts/06_compute_drift_metrics.py index 3dfb85a..972c224 100644 --- a/example/scripts/06_compute_drift_metrics.py +++ b/example/scripts/06_compute_drift_metrics.py @@ -5,6 +5,7 @@ - feature_mean_shift: difference between training mean and recent prediction mean per feature - mae: mean absolute error between predictions and ground truth """ + import os import numpy as np from pathlib import Path @@ -17,19 +18,29 @@ PROJECT = os.environ.get("BIGQUERY_PROJECT") if not PROJECT: raise SystemExit("BIGQUERY_PROJECT missing. Run `deployml get-urls` to write .env.") -DATASET = os.getenv("BIGQUERY_DATASET", "mlops") -FEATURE_COLS = ["bedrooms", "bathrooms", "area_sqft", "lot_size", "year_built", "city", "state"] +DATASET = os.getenv("BIGQUERY_DATASET", "mlops") +FEATURE_COLS = [ + "bedrooms", + "bathrooms", + "area_sqft", + "lot_size", + "year_built", + "city", + "state", +] client = bigquery.Client(project=PROJECT) # Training feature distributions (baseline) -train_query = f"SELECT {', '.join(FEATURE_COLS)} FROM `{PROJECT}.{DATASET}.offline_features`" -train_df = client.query(train_query).to_dataframe() +train_query = ( + f"SELECT {', '.join(FEATURE_COLS)} FROM `{PROJECT}.{DATASET}.offline_features`" +) +train_df = client.query(train_query).to_dataframe() # Recent predictions (simulate "serving" features by using training data subset as proxy) # In production you'd log features alongside predictions and query those recent_query = f""" - SELECT {', '.join(FEATURE_COLS)} + SELECT {", ".join(FEATURE_COLS)} FROM `{PROJECT}.{DATASET}.offline_features` ORDER BY event_timestamp DESC LIMIT 50 @@ -44,7 +55,7 @@ """ mae_df = client.query(mae_query).to_dataframe() -now = datetime.now(timezone.utc) +now = datetime.now(timezone.utc) window_end = now window_start = now.replace(hour=0, minute=0, second=0, microsecond=0) rows = [] @@ -52,28 +63,32 @@ # Feature mean shift per feature for col in FEATURE_COLS: shift = float(recent_df[col].mean() - train_df[col].mean()) - rows.append({ - "metric_timestamp": now.isoformat(), - "metric_type": "feature_mean_shift", - "feature_name": col, - "value": shift, - "window_start": window_start.isoformat(), - "window_end": window_end.isoformat(), - "model_version": "HousingPriceModel/Production", - }) + rows.append( + { + "metric_timestamp": now.isoformat(), + "metric_type": "feature_mean_shift", + "feature_name": col, + "value": shift, + "window_start": window_start.isoformat(), + "window_end": window_end.isoformat(), + "model_version": "HousingPriceModel/Production", + } + ) # MAE if len(mae_df) > 0: mae = float(np.mean(np.abs(mae_df["predicted_value"] - mae_df["actual_value"]))) - rows.append({ - "metric_timestamp": now.isoformat(), - "metric_type": "mae", - "feature_name": None, - "value": mae, - "window_start": window_start.isoformat(), - "window_end": window_end.isoformat(), - "model_version": "HousingPriceModel/Production", - }) + rows.append( + { + "metric_timestamp": now.isoformat(), + "metric_type": "mae", + "feature_name": None, + "value": mae, + "window_start": window_start.isoformat(), + "window_end": window_end.isoformat(), + "model_version": "HousingPriceModel/Production", + } + ) print(f" MAE: {mae:.2f}") errors = client.insert_rows_json(f"{PROJECT}.{DATASET}.drift_metrics", rows) diff --git a/example/scripts/07_setup_grafana.py b/example/scripts/07_setup_grafana.py index af88f8e..85a53e8 100644 --- a/example/scripts/07_setup_grafana.py +++ b/example/scripts/07_setup_grafana.py @@ -12,8 +12,8 @@ gcloud using GRAFANA_ADMIN_PASSWORD_SECRET_ID from .env (run `deployml get-urls`). Override by exporting GRAFANA_PASSWORD if you prefer. """ + import os -import json import subprocess import requests from pathlib import Path @@ -22,9 +22,11 @@ load_dotenv(Path.cwd() / ".env") GRAFANA_URL = os.environ.get("GRAFANA_URL") -PROJECT = os.environ.get("BIGQUERY_PROJECT") +PROJECT = os.environ.get("BIGQUERY_PROJECT") if not GRAFANA_URL or not PROJECT: - raise SystemExit("GRAFANA_URL or BIGQUERY_PROJECT missing. Run `deployml get-urls` to write .env.") + raise SystemExit( + "GRAFANA_URL or BIGQUERY_PROJECT missing. Run `deployml get-urls` to write .env." + ) GRAFANA_URL = GRAFANA_URL.rstrip("/") DATASET = os.getenv("BIGQUERY_DATASET", "mlops") GRAFANA_USER = os.getenv("GRAFANA_USER", "admin") @@ -38,9 +40,19 @@ "or run `deployml get-urls` so .env has GRAFANA_ADMIN_PASSWORD_SECRET_ID." ) proc = subprocess.run( - ["gcloud", "secrets", "versions", "access", "latest", - "--secret", secret_id, "--project", PROJECT], - capture_output=True, text=True, + [ + "gcloud", + "secrets", + "versions", + "access", + "latest", + "--secret", + secret_id, + "--project", + PROJECT, + ], + capture_output=True, + text=True, ) if proc.returncode != 0: raise SystemExit(f"Failed to fetch Grafana password: {proc.stderr.strip()}") @@ -59,12 +71,12 @@ def get_or_create_bigquery_datasource(): return uid payload = { - "name": "BigQuery", - "type": "grafana-bigquery-datasource", + "name": "BigQuery", + "type": "grafana-bigquery-datasource", "access": "proxy", "jsonData": { "authenticationType": "gce", - "defaultProject": PROJECT, + "defaultProject": PROJECT, }, } resp = session.post(f"{GRAFANA_URL}/api/datasources", json=payload) @@ -78,10 +90,10 @@ def build_dashboard(ds_uid: str) -> dict: def bq_target(sql: str, ref: str) -> dict: return { "datasource": {"type": "grafana-bigquery-datasource", "uid": ds_uid}, - "rawQuery": True, - "rawSql": sql, - "refId": ref, - "format": "time_series", + "rawQuery": True, + "rawSql": sql, + "refId": ref, + "format": "time_series", } predictions_sql = f""" @@ -112,42 +124,50 @@ def bq_target(sql: str, ref: str) -> dict: panels = [ { - "id": 1, "title": "Prediction Volume", - "type": "timeseries", "gridPos": {"x": 0, "y": 0, "w": 12, "h": 8}, + "id": 1, + "title": "Prediction Volume", + "type": "timeseries", + "gridPos": {"x": 0, "y": 0, "w": 12, "h": 8}, "targets": [bq_target(predictions_sql, "A")], }, { - "id": 2, "title": "Mean Predicted Price", - "type": "timeseries", "gridPos": {"x": 12, "y": 0, "w": 12, "h": 8}, + "id": 2, + "title": "Mean Predicted Price", + "type": "timeseries", + "gridPos": {"x": 12, "y": 0, "w": 12, "h": 8}, "targets": [bq_target(mean_pred_sql, "A")], }, { - "id": 3, "title": "Feature Mean Shift", - "type": "timeseries", "gridPos": {"x": 0, "y": 8, "w": 12, "h": 8}, + "id": 3, + "title": "Feature Mean Shift", + "type": "timeseries", + "gridPos": {"x": 0, "y": 8, "w": 12, "h": 8}, "targets": [bq_target(drift_sql, "A")], }, { - "id": 4, "title": "MAE (Predictions vs Ground Truth)", - "type": "timeseries", "gridPos": {"x": 12, "y": 8, "w": 12, "h": 8}, + "id": 4, + "title": "MAE (Predictions vs Ground Truth)", + "type": "timeseries", + "gridPos": {"x": 12, "y": 8, "w": 12, "h": 8}, "targets": [bq_target(mae_sql, "A")], }, ] return { "dashboard": { - "title": "Housing Price Model Monitoring", - "tags": ["mlops", "deployml"], - "timezone": "browser", - "panels": panels, + "title": "Housing Price Model Monitoring", + "tags": ["mlops", "deployml"], + "timezone": "browser", + "panels": panels, "schemaVersion": 36, - "version": 1, + "version": 1, }, "overwrite": True, - "folderId": 0, + "folderId": 0, } -ds_uid = get_or_create_bigquery_datasource() +ds_uid = get_or_create_bigquery_datasource() dashboard = build_dashboard(ds_uid) resp = session.post(f"{GRAFANA_URL}/api/dashboards/db", json=dashboard) diff --git a/pyproject.toml b/pyproject.toml index 32f6f95..6f4c685 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,11 +49,19 @@ source-exclude = [ dev = [ "pytest>=8", "python-semantic-release>=10,<11", + "pre-commit>=4", + "ruff==0.15.21", # pinned to match .pre-commit-config.yaml so CI and the hook agree ] [tool.pytest.ini_options] testpaths = ["tests"] +[tool.ruff] +# Defaults are fine for a student-contributor repo: lint set E + F, and the +# black-compatible formatter. The pre-commit hook auto-fixes; CI only checks. +# recycling_bin/ is archived dead code, left untouched. +extend-exclude = ["recycling_bin"] + [tool.semantic_release] version_toml = ["pyproject.toml:project.version"] allow_zero_version = true diff --git a/src/deployml/__init__.py b/src/deployml/__init__.py index 2794b35..2ef2a77 100644 --- a/src/deployml/__init__.py +++ b/src/deployml/__init__.py @@ -5,11 +5,11 @@ from .diagnostics import run_doctor, check_system, DeployMLDoctor __all__ = [ - 'deploy', - 'load', - 'DeploymentStack', - 'ServiceURLs', - 'run_doctor', - 'check_system', - 'DeployMLDoctor' + "deploy", + "load", + "DeploymentStack", + "ServiceURLs", + "run_doctor", + "check_system", + "DeployMLDoctor", ] diff --git a/src/deployml/api.py b/src/deployml/api.py index 7cd6d59..8b91098 100644 --- a/src/deployml/api.py +++ b/src/deployml/api.py @@ -5,14 +5,14 @@ Example usage: import deployml.api as api - + # Get teardown status status = api.get_teardown_status( project_id="my-project", region="us-west1", workspace_name="my-stack" ) - + # Update teardown schedule result = api.update_teardown_schedule( project_id="my-project", @@ -20,7 +20,7 @@ workspace_name="my-stack", duration_hours=6 ) - + # Cancel teardown api.cancel_teardown( project_id="my-project", @@ -28,6 +28,7 @@ workspace_name="my-stack" ) """ + import json from pathlib import Path from datetime import datetime, timedelta, timezone @@ -37,7 +38,7 @@ from .utils.teardown import ( calculate_cron_from_timestamp, load_deployment_metadata, - save_deployment_metadata + save_deployment_metadata, ) @@ -45,17 +46,17 @@ def get_teardown_status( project_id: str, region: str, workspace_name: str, - deployml_dir: Optional[Path] = None + deployml_dir: Optional[Path] = None, ) -> Dict[str, Any]: """ Get teardown status from Cloud Scheduler. - + Args: project_id: GCP project ID region: GCP region workspace_name: Workspace name deployml_dir: Optional path to deployment directory for metadata - + Returns: Dictionary with status information including: - exists: bool - Whether the scheduler job exists @@ -67,46 +68,57 @@ def get_teardown_status( - metadata: Optional[Dict] - Local metadata if available """ scheduler_job_name = f"deployml-teardown-{workspace_name}" - + result = run_tool( - "gcloud", ["scheduler", "jobs", "describe", scheduler_job_name, - "--project", project_id, "--location", region, "--format", "json"], + "gcloud", + [ + "scheduler", + "jobs", + "describe", + scheduler_job_name, + "--project", + project_id, + "--location", + region, + "--format", + "json", + ], capture_output=True, text=True, ) - + status = { "exists": False, "scheduler_job_name": scheduler_job_name, } - + # Load local metadata if available if deployml_dir: metadata = load_deployment_metadata(deployml_dir) status["metadata"] = metadata - + if result.returncode != 0: return status - + try: job_info = json.loads(result.stdout) status["exists"] = True status["state"] = job_info.get("state", "UNKNOWN") status["schedule"] = job_info.get("schedule", "") status["time_zone"] = job_info.get("timeZone", "UTC") - + # Parse schedule time schedule_time = job_info.get("scheduleTime", "") if schedule_time: - time_str = schedule_time.replace('Z', '+00:00') + time_str = schedule_time.replace("Z", "+00:00") status["schedule_time"] = datetime.fromisoformat(time_str) - + # Parse last attempt time last_attempt_time = job_info.get("lastAttemptTime", "") if last_attempt_time and last_attempt_time != "1970-01-01T00:00:00Z": - time_str = last_attempt_time.replace('Z', '+00:00') + time_str = last_attempt_time.replace("Z", "+00:00") status["last_attempt_time"] = datetime.fromisoformat(time_str) - + return status except json.JSONDecodeError: return status @@ -118,11 +130,11 @@ def update_teardown_schedule( workspace_name: str, duration_hours: int, deployml_dir: Optional[Path] = None, - time_zone: str = "UTC" + time_zone: str = "UTC", ) -> Dict[str, Any]: """ Update teardown schedule programmatically. - + Args: project_id: GCP project ID region: GCP region @@ -130,7 +142,7 @@ def update_teardown_schedule( duration_hours: Hours until teardown deployml_dir: Optional path to deployment directory for metadata time_zone: Timezone (default: UTC) - + Returns: Dictionary with update result: - success: bool - Whether update succeeded @@ -139,74 +151,95 @@ def update_teardown_schedule( - error: Optional[str] - Error message if failed """ scheduler_job_name = f"deployml-teardown-{workspace_name}" - + # Check if job exists and get current timezone result = run_tool( - "gcloud", ["scheduler", "jobs", "describe", scheduler_job_name, - "--project", project_id, "--location", region, "--format", "json"], + "gcloud", + [ + "scheduler", + "jobs", + "describe", + scheduler_job_name, + "--project", + project_id, + "--location", + region, + "--format", + "json", + ], capture_output=True, text=True, ) - + if result.returncode != 0: return { "success": False, - "error": f"Cloud Scheduler job not found: {scheduler_job_name}" + "error": f"Cloud Scheduler job not found: {scheduler_job_name}", } - + # Get timezone from existing job if not provided try: job_info = json.loads(result.stdout) time_zone = job_info.get("timeZone", time_zone) except json.JSONDecodeError: pass - + # Calculate new teardown time now = datetime.now(timezone.utc) teardown_at = now + timedelta(hours=duration_hours) teardown_scheduled_timestamp = int(teardown_at.timestamp()) new_cron_schedule = calculate_cron_from_timestamp(teardown_scheduled_timestamp) - + # Update Cloud Scheduler job update_result = run_tool( "gcloud", [ - "scheduler", "jobs", "update", "http", scheduler_job_name, - "--location", region, - "--schedule", new_cron_schedule, - "--time-zone", time_zone, - "--project", project_id, - "--quiet" + "scheduler", + "jobs", + "update", + "http", + scheduler_job_name, + "--location", + region, + "--schedule", + new_cron_schedule, + "--time-zone", + time_zone, + "--project", + project_id, + "--quiet", ], capture_output=True, text=True, ) - + if update_result.returncode != 0: return { "success": False, "error": update_result.stderr, "scheduled_time": teardown_at, - "cron_schedule": new_cron_schedule + "cron_schedule": new_cron_schedule, } - + # Update local metadata if directory provided if deployml_dir: metadata = load_deployment_metadata(deployml_dir) or {} - metadata.update({ - "deployed_at": metadata.get("deployed_at", now.isoformat()), - "teardown_scheduled_at": teardown_at.isoformat(), - "teardown_enabled": True, - "duration_hours": duration_hours, - "scheduler_job_name": scheduler_job_name - }) + metadata.update( + { + "deployed_at": metadata.get("deployed_at", now.isoformat()), + "teardown_scheduled_at": teardown_at.isoformat(), + "teardown_enabled": True, + "duration_hours": duration_hours, + "scheduler_job_name": scheduler_job_name, + } + ) save_deployment_metadata(deployml_dir, metadata) - + return { "success": True, "scheduled_time": teardown_at, "cron_schedule": new_cron_schedule, - "time_zone": time_zone + "time_zone": time_zone, } @@ -214,50 +247,52 @@ def cancel_teardown( project_id: str, region: str, workspace_name: str, - deployml_dir: Optional[Path] = None + deployml_dir: Optional[Path] = None, ) -> Dict[str, Any]: """ Cancel scheduled teardown programmatically. - + Args: project_id: GCP project ID region: GCP region workspace_name: Workspace name deployml_dir: Optional path to deployment directory for metadata - + Returns: Dictionary with cancellation result: - success: bool - Whether cancellation succeeded - error: Optional[str] - Error message if failed """ scheduler_job_name = f"deployml-teardown-{workspace_name}" - + result = run_tool( - "gcloud", ["scheduler", "jobs", "delete", scheduler_job_name, - "--project", project_id, "--location", region, "--quiet"], + "gcloud", + [ + "scheduler", + "jobs", + "delete", + scheduler_job_name, + "--project", + project_id, + "--location", + region, + "--quiet", + ], capture_output=True, text=True, ) - + if result.returncode != 0: - return { - "success": False, - "error": result.stderr - } - + return {"success": False, "error": result.stderr} + # Update metadata if directory provided if deployml_dir: metadata = load_deployment_metadata(deployml_dir) if metadata: metadata["teardown_enabled"] = False save_deployment_metadata(deployml_dir, metadata) - - return {"success": True} + return {"success": True} -__all__ = [ - 'get_teardown_status', - 'update_teardown_schedule', - 'cancel_teardown' -] +__all__ = ["get_teardown_status", "update_teardown_schedule", "cancel_teardown"] diff --git a/src/deployml/cli/cli.py b/src/deployml/cli/cli.py index ffe4d6d..2dc2d07 100644 --- a/src/deployml/cli/cli.py +++ b/src/deployml/cli/cli.py @@ -1,26 +1,22 @@ -import sys import yaml import typer import re import importlib.resources as pkg_resources from deployml.utils.constants import ( TEMPLATE_DIR, - TERRAFORM_DIR, - TOOL_VARIABLES, - ANIMAL_NAMES, - FALLBACK_WORDS, REQUIRED_GCP_APIS, REQUIRED_GCP_IAM_ROLES, ) from jinja2 import Environment, FileSystemLoader from pathlib import Path from typing import Optional -import random -import string from google.cloud import storage import hashlib +import time +import json +from datetime import datetime, timedelta -from deployml.notebook.docker import build_images +from deployml.notebook.docker import build_images # Import refactored utility functions from deployml.utils.helpers import ( @@ -32,17 +28,19 @@ validate_gcp_project, validate_gcp_region, get_missing_iam_roles, - check_docker_daemon, copy_modules_to_workspace, bucket_exists, generate_bucket_name, estimate_terraform_time, - cleanup_cloud_sql_resources, cleanup_terraform_files, run_terraform_with_loading_bar, _create_docker_folder, ) -from deployml.utils.platform_compat import run_tool, resolve_tool, configure_console_encoding, robust_rmtree +from deployml.utils.platform_compat import ( + run_tool, + configure_console_encoding, + robust_rmtree, +) from deployml.utils.infracost import ( check_infracost_available, run_infracost_analysis, @@ -59,7 +57,7 @@ deploy_fastapi_to_minikube, generate_mlflow_manifests, deploy_mlflow_to_minikube, - check_minikube_running + check_minikube_running, ) from deployml.utils.kubernetes_gke import ( generate_mlflow_manifests_gke, @@ -69,7 +67,9 @@ ) -def upload_terraform_files_to_gcs(terraform_dir: Path, project_id: str, workspace_name: str): +def upload_terraform_files_to_gcs( + terraform_dir: Path, project_id: str, workspace_name: str +): """ Upload Terraform files to GCS bucket for Cloud Build teardown. Gets bucket name from Terraform state. @@ -78,86 +78,102 @@ def upload_terraform_files_to_gcs(terraform_dir: Path, project_id: str, workspac # Get terraform files bucket from Terraform state # The bucket is created by the teardown module state_proc = run_tool( - "terraform", ["state", "list"], + "terraform", + ["state", "list"], cwd=terraform_dir, capture_output=True, text=True, ) - + if state_proc.returncode != 0: typer.echo(f"WARNING: Could not read Terraform state: {state_proc.stderr}") return - + # Find the terraform_files bucket resource bucket_resource = None - for line in state_proc.stdout.split('\n'): - if 'module.teardown.google_storage_bucket.terraform_files' in line: + for line in state_proc.stdout.split("\n"): + if "module.teardown.google_storage_bucket.terraform_files" in line: bucket_resource = line.strip() break - + if not bucket_resource: - typer.echo("WARNING: Teardown module bucket not found in state. Skipping upload.") + typer.echo( + "WARNING: Teardown module bucket not found in state. Skipping upload." + ) return - + # Get bucket name from state show_proc = run_tool( - "terraform", ["state", "show", bucket_resource], + "terraform", + ["state", "show", bucket_resource], cwd=terraform_dir, capture_output=True, text=True, ) - + if show_proc.returncode != 0: typer.echo(f"WARNING: Could not get bucket name: {show_proc.stderr}") return - + # Extract bucket name from terraform state show output bucket_name = None - for line in show_proc.stdout.split('\n'): - if 'name' in line and '=' in line: - bucket_name = line.split('=')[1].strip().strip('"') + for line in show_proc.stdout.split("\n"): + if "name" in line and "=" in line: + bucket_name = line.split("=")[1].strip().strip('"') break - + if not bucket_name: typer.echo("WARNING: Could not extract bucket name from state.") return - + # Upload Terraform files storage_client = storage.Client(project=project_id) bucket = storage_client.bucket(bucket_name) - + # Upload all .tf files, .tfvars, and state files - terraform_files = list(terraform_dir.glob("*.tf")) + list(terraform_dir.glob("*.tfvars")) - terraform_files += list(terraform_dir.glob("terraform.tfstate*")) # Include state files - terraform_files += list((terraform_dir / "modules").rglob("*.tf")) if (terraform_dir / "modules").exists() else [] - + terraform_files = list(terraform_dir.glob("*.tf")) + list( + terraform_dir.glob("*.tfvars") + ) + terraform_files += list( + terraform_dir.glob("terraform.tfstate*") + ) # Include state files + terraform_files += ( + list((terraform_dir / "modules").rglob("*.tf")) + if (terraform_dir / "modules").exists() + else [] + ) + uploaded_count = 0 for tf_file in terraform_files: if tf_file.is_file(): # Create relative path from terraform_dir relative_path = tf_file.relative_to(terraform_dir) blob_path = f"{workspace_name}/terraform/{relative_path}" - + blob = bucket.blob(blob_path) blob.upload_from_filename(str(tf_file)) uploaded_count += 1 - - typer.echo(f" Uploaded {uploaded_count} Terraform files to gs://{bucket_name}/{workspace_name}/terraform/") - + + typer.echo( + f" Uploaded {uploaded_count} Terraform files to gs://{bucket_name}/{workspace_name}/terraform/" + ) + except Exception as e: typer.echo(f"WARNING: Error uploading Terraform files: {e}") import traceback + typer.echo(traceback.format_exc()) -def extract_resource_manifest(terraform_dir: Path, project_id: str, workspace_name: str, region: str) -> dict: +def extract_resource_manifest( + terraform_dir: Path, project_id: str, workspace_name: str, region: str +) -> dict: """ Extract resource details from Terraform outputs and state. Returns a manifest dictionary with all resources that need to be deleted. """ import json - from urllib.parse import urlparse - + manifest = { "workspace_name": workspace_name, "project_id": project_id, @@ -172,12 +188,13 @@ def extract_resource_manifest(terraform_dir: Path, project_id: str, workspace_na "secret_manager_secrets": [], "service_accounts": [], "cloud_build_triggers": [], - } + }, } - + # Get Terraform outputs output_proc = run_tool( - "terraform", ["output", "-json"], + "terraform", + ["output", "-json"], cwd=terraform_dir, capture_output=True, text=True, @@ -185,49 +202,55 @@ def extract_resource_manifest(terraform_dir: Path, project_id: str, workspace_na if output_proc.returncode == 0: outputs = json.loads(output_proc.stdout) - + # Extract Cloud Run service names from URLs for key, value in outputs.items(): - output_val = value.get('value', '') - + output_val = value.get("value", "") + # Cloud Run services - we'll extract from Terraform state instead of URLs # (URLs contain hash suffixes that don't match actual service names) pass # Skip URL parsing, will get from state below - + # Storage buckets - if '_bucket' in key and output_val: + if "_bucket" in key and output_val: if isinstance(output_val, str) and output_val: - manifest["resources"]["storage_buckets"].append({ - "name": output_val - }) - + manifest["resources"]["storage_buckets"].append( + {"name": output_val} + ) + # Cloud SQL instance connection name - if 'instance_connection_name' in key and output_val: + if "instance_connection_name" in key and output_val: # Format: project:region:instance - parts = str(output_val).split(':') + parts = str(output_val).split(":") if len(parts) == 3: - manifest["resources"]["cloud_sql_instances"].append({ - "name": parts[2], - "region": parts[1] - }) - + manifest["resources"]["cloud_sql_instances"].append( + {"name": parts[2], "region": parts[1]} + ) + # Query Terraform state for additional resources state_proc = run_tool( - "terraform", ["state", "list"], + "terraform", + ["state", "list"], cwd=terraform_dir, capture_output=True, text=True, ) - + if state_proc.returncode == 0: - state_resources = [r.strip() for r in state_proc.stdout.strip().split('\n') if r.strip()] - + state_resources = [ + r.strip() for r in state_proc.stdout.strip().split("\n") if r.strip() + ] + for resource in state_resources: try: # Cloud Run services (v1 and v2) - if 'google_cloud_run_service' in resource and 'google_cloud_run_v2_job' not in resource: + if ( + "google_cloud_run_service" in resource + and "google_cloud_run_v2_job" not in resource + ): show_proc = run_tool( - "terraform", ["state", "show", resource], + "terraform", + ["state", "show", resource], cwd=terraform_dir, capture_output=True, text=True, @@ -235,55 +258,87 @@ def extract_resource_manifest(terraform_dir: Path, project_id: str, workspace_na if show_proc.returncode == 0: service_name = None service_region = region - for line in show_proc.stdout.split('\n'): + for line in show_proc.stdout.split("\n"): # Strip ANSI escape codes - clean_line = re.sub(r'\x1b\[[0-9;]*m', '', line) + clean_line = re.sub(r"\x1b\[[0-9;]*m", "", line) # Check for name field (but not location, id, or other fields) - if (clean_line.strip().startswith('name') or ' = name' in clean_line.lower()) and '=' in clean_line and 'location' not in clean_line.lower() and 'id' not in clean_line.lower() and 'latest' not in clean_line.lower(): - parts = clean_line.split('=') + if ( + ( + clean_line.strip().startswith("name") + or " = name" in clean_line.lower() + ) + and "=" in clean_line + and "location" not in clean_line.lower() + and "id" not in clean_line.lower() + and "latest" not in clean_line.lower() + ): + parts = clean_line.split("=") if len(parts) >= 2: - potential_name = parts[1].strip().strip('"').strip("'") + potential_name = ( + parts[1].strip().strip('"').strip("'") + ) # Remove ANSI codes from the name itself - potential_name = re.sub(r'\x1b\[[0-9;]*m', '', potential_name) + potential_name = re.sub( + r"\x1b\[[0-9;]*m", "", potential_name + ) # Skip null, empty, or invalid values - if potential_name and potential_name.lower() != 'null' and '/' not in potential_name and '@' not in potential_name and len(potential_name) < 200: + if ( + potential_name + and potential_name.lower() != "null" + and "/" not in potential_name + and "@" not in potential_name + and len(potential_name) < 200 + ): service_name = potential_name break # Check for location or region field - elif (clean_line.strip().startswith('location') or clean_line.strip().startswith('region')) and '=' in clean_line: - parts = clean_line.split('=') + elif ( + clean_line.strip().startswith("location") + or clean_line.strip().startswith("region") + ) and "=" in clean_line: + parts = clean_line.split("=") if len(parts) >= 2: - service_region = parts[1].strip().strip('"').strip("'") - service_region = re.sub(r'\x1b\[[0-9;]*m', '', service_region) + service_region = ( + parts[1].strip().strip('"').strip("'") + ) + service_region = re.sub( + r"\x1b\[[0-9;]*m", "", service_region + ) if service_name: - manifest["resources"]["cloud_run_services"].append({ - "name": service_name, - "region": service_region - }) - + manifest["resources"]["cloud_run_services"].append( + {"name": service_name, "region": service_region} + ) + # Cloud Run Jobs - elif 'google_cloud_run_v2_job' in resource: + elif "google_cloud_run_v2_job" in resource: show_proc = run_tool( - "terraform", ["state", "show", resource], + "terraform", + ["state", "show", resource], cwd=terraform_dir, capture_output=True, text=True, ) if show_proc.returncode == 0: - for line in show_proc.stdout.split('\n'): - if 'name' in line.lower() and '=' in line and 'location' not in line.lower(): - job_name = line.split('=')[1].strip().strip('"').strip("'") + for line in show_proc.stdout.split("\n"): + if ( + "name" in line.lower() + and "=" in line + and "location" not in line.lower() + ): + job_name = ( + line.split("=")[1].strip().strip('"').strip("'") + ) if job_name: - manifest["resources"]["cloud_run_jobs"].append({ - "name": job_name, - "region": region - }) + manifest["resources"]["cloud_run_jobs"].append( + {"name": job_name, "region": region} + ) break - + # Cloud Scheduler jobs - elif 'google_cloud_scheduler_job' in resource: + elif "google_cloud_scheduler_job" in resource: show_proc = run_tool( - "terraform", ["state", "show", resource], + "terraform", + ["state", "show", resource], cwd=terraform_dir, capture_output=True, text=True, @@ -291,175 +346,205 @@ def extract_resource_manifest(terraform_dir: Path, project_id: str, workspace_na if show_proc.returncode == 0: job_name = None job_region = region - for line in show_proc.stdout.split('\n'): - if 'name' in line.lower() and '=' in line and 'location' not in line.lower(): - job_name = line.split('=')[1].strip().strip('"').strip("'") + for line in show_proc.stdout.split("\n"): + if ( + "name" in line.lower() + and "=" in line + and "location" not in line.lower() + ): + job_name = ( + line.split("=")[1].strip().strip('"').strip("'") + ) # Extract job name from URL if it's a full URL - if job_name and '/jobs/' in job_name: - job_name = job_name.split('/jobs/')[-1].split(':')[0].split('/')[-1] - elif 'region' in line.lower() and '=' in line: - job_region = line.split('=')[1].strip().strip('"').strip("'") + if job_name and "/jobs/" in job_name: + job_name = ( + job_name.split("/jobs/")[-1] + .split(":")[0] + .split("/")[-1] + ) + elif "region" in line.lower() and "=" in line: + job_region = ( + line.split("=")[1].strip().strip('"').strip("'") + ) if job_name: - manifest["resources"]["cloud_scheduler_jobs"].append({ - "name": job_name, - "region": job_region - }) - + manifest["resources"]["cloud_scheduler_jobs"].append( + {"name": job_name, "region": job_region} + ) + # Pub/Sub topics - elif 'google_pubsub_topic' in resource: + elif "google_pubsub_topic" in resource: show_proc = run_tool( - "terraform", ["state", "show", resource], + "terraform", + ["state", "show", resource], cwd=terraform_dir, capture_output=True, text=True, ) if show_proc.returncode == 0: - for line in show_proc.stdout.split('\n'): - if 'name' in line.lower() and '=' in line: - topic_name = line.split('=')[1].strip().strip('"').strip("'") + for line in show_proc.stdout.split("\n"): + if "name" in line.lower() and "=" in line: + topic_name = ( + line.split("=")[1].strip().strip('"').strip("'") + ) if topic_name: - manifest["resources"]["pubsub_topics"].append({ - "name": topic_name - }) + manifest["resources"]["pubsub_topics"].append( + {"name": topic_name} + ) break - + # Secret Manager secrets - elif 'google_secret_manager_secret' in resource: + elif "google_secret_manager_secret" in resource: show_proc = run_tool( - "terraform", ["state", "show", resource], + "terraform", + ["state", "show", resource], cwd=terraform_dir, capture_output=True, text=True, ) if show_proc.returncode == 0: - for line in show_proc.stdout.split('\n'): - if 'secret_id' in line.lower() and '=' in line: - secret_name = line.split('=')[1].strip().strip('"').strip("'") + for line in show_proc.stdout.split("\n"): + if "secret_id" in line.lower() and "=" in line: + secret_name = ( + line.split("=")[1].strip().strip('"').strip("'") + ) if secret_name: - manifest["resources"]["secret_manager_secrets"].append({ - "name": secret_name - }) + manifest["resources"][ + "secret_manager_secrets" + ].append({"name": secret_name}) break - + # Service accounts (only teardown ones to avoid deleting user SAs) - elif 'google_service_account' in resource and 'teardown' in resource: + elif "google_service_account" in resource and "teardown" in resource: show_proc = run_tool( - "terraform", ["state", "show", resource], + "terraform", + ["state", "show", resource], cwd=terraform_dir, capture_output=True, text=True, ) if show_proc.returncode == 0: - for line in show_proc.stdout.split('\n'): - if 'email' in line.lower() and '@' in line and '=' in line: - sa_email = line.split('=')[1].strip().strip('"').strip("'") + for line in show_proc.stdout.split("\n"): + if "email" in line.lower() and "@" in line and "=" in line: + sa_email = ( + line.split("=")[1].strip().strip('"').strip("'") + ) if sa_email: - manifest["resources"]["service_accounts"].append({ - "email": sa_email - }) + manifest["resources"]["service_accounts"].append( + {"email": sa_email} + ) break - + # Cloud Build triggers - elif 'google_cloudbuild_trigger' in resource: + elif "google_cloudbuild_trigger" in resource: show_proc = run_tool( - "terraform", ["state", "show", resource], + "terraform", + ["state", "show", resource], cwd=terraform_dir, capture_output=True, text=True, ) if show_proc.returncode == 0: - for line in show_proc.stdout.split('\n'): - if 'name' in line.lower() and '=' in line: - trigger_name = line.split('=')[1].strip().strip('"').strip("'") + for line in show_proc.stdout.split("\n"): + if "name" in line.lower() and "=" in line: + trigger_name = ( + line.split("=")[1].strip().strip('"').strip("'") + ) if trigger_name: - manifest["resources"]["cloud_build_triggers"].append({ - "name": trigger_name - }) + manifest["resources"][ + "cloud_build_triggers" + ].append({"name": trigger_name}) break - except Exception as e: + except Exception: # Skip resources that can't be parsed continue - + # Remove duplicates for resource_type in manifest["resources"]: seen = set() unique_resources = [] for res in manifest["resources"][resource_type]: - if resource_type in ["cloud_run_services", "cloud_run_jobs", "cloud_sql_instances", "cloud_scheduler_jobs"]: + if resource_type in [ + "cloud_run_services", + "cloud_run_jobs", + "cloud_sql_instances", + "cloud_scheduler_jobs", + ]: key = (res.get("name"), res.get("region")) elif resource_type == "service_accounts": key = res.get("email") else: key = res.get("name") - + if key and key not in seen: seen.add(key) unique_resources.append(res) manifest["resources"][resource_type] = unique_resources - + return manifest -def upload_resource_manifest(manifest: dict, terraform_dir: Path, project_id: str, workspace_name: str): +def upload_resource_manifest( + manifest: dict, terraform_dir: Path, project_id: str, workspace_name: str +): """Upload resource manifest to GCS bucket.""" import json - + try: # Get bucket name from Terraform state (same logic as upload_terraform_files_to_gcs) state_proc = run_tool( - "terraform", ["state", "list"], + "terraform", + ["state", "list"], cwd=terraform_dir, capture_output=True, text=True, ) - + if state_proc.returncode != 0: raise Exception(f"Could not read Terraform state: {state_proc.stderr}") - + bucket_resource = None - for line in state_proc.stdout.split('\n'): - if 'module.teardown.google_storage_bucket.terraform_files' in line: + for line in state_proc.stdout.split("\n"): + if "module.teardown.google_storage_bucket.terraform_files" in line: bucket_resource = line.strip() break - + if not bucket_resource: raise Exception("Teardown module bucket not found in state") - + show_proc = run_tool( - "terraform", ["state", "show", bucket_resource], + "terraform", + ["state", "show", bucket_resource], cwd=terraform_dir, capture_output=True, text=True, ) - + if show_proc.returncode != 0: raise Exception(f"Could not get bucket name: {show_proc.stderr}") - + bucket_name = None - for line in show_proc.stdout.split('\n'): - if 'name' in line and '=' in line: - bucket_name = line.split('=')[1].strip().strip('"') + for line in show_proc.stdout.split("\n"): + if "name" in line and "=" in line: + bucket_name = line.split("=")[1].strip().strip('"') break - + if not bucket_name: raise Exception("Could not extract bucket name from state") - + # Upload manifest storage_client = storage.Client(project=project_id) bucket = storage_client.bucket(bucket_name) blob = bucket.blob(f"{workspace_name}/resource-manifest.json") blob.upload_from_string(json.dumps(manifest, indent=2)) - - typer.echo(f" Resource manifest uploaded to gs://{bucket_name}/{workspace_name}/resource-manifest.json") - + + typer.echo( + f" Resource manifest uploaded to gs://{bucket_name}/{workspace_name}/resource-manifest.json" + ) + except Exception as e: typer.echo(f"WARNING: Error uploading resource manifest: {e}") raise -import re -import time -import json -from datetime import datetime, timedelta def _load_config_or_exit(config_path: Path) -> dict: """Load YAML config with clean error messages. Exits non-zero on failure.""" @@ -484,7 +569,10 @@ def _validate_deploy_config_or_exit(config: dict) -> None: """Validate the fields deploy and destroy need. Exits non-zero on missing or bad values.""" provider = config.get("provider") if not isinstance(provider, dict): - typer.secho(" Config is missing required field: provider (mapping).", fg=typer.colors.RED) + typer.secho( + " Config is missing required field: provider (mapping).", + fg=typer.colors.RED, + ) raise typer.Exit(code=1) name = provider.get("name") if name not in _SUPPORTED_PROVIDERS: @@ -498,7 +586,9 @@ def _validate_deploy_config_or_exit(config: dict) -> None: raise typer.Exit(code=1) deployment = config.get("deployment") if not isinstance(deployment, dict) or not deployment.get("type"): - typer.secho(" Config is missing required field: deployment.type.", fg=typer.colors.RED) + typer.secho( + " Config is missing required field: deployment.type.", fg=typer.colors.RED + ) raise typer.Exit(code=1) # Validate the stack shape so a malformed entry fails here with a clear message @@ -555,14 +645,16 @@ def get_version(): """Get version from package metadata""" try: from importlib.metadata import version + return version("deployml-core") except Exception: try: result = run_tool( - "git", ["describe", "--tags", "--abbrev=0"], + "git", + ["describe", "--tags", "--abbrev=0"], capture_output=True, text=True, - cwd=Path(__file__).parent.parent.parent.parent + cwd=Path(__file__).parent.parent.parent.parent, ) if result.returncode == 0: git_version = result.stdout.strip().lstrip("v") @@ -571,12 +663,16 @@ def get_version(): pass return "version unknown" + cli = typer.Typer(invoke_without_command=True) + @cli.callback() def cli_callback( ctx: typer.Context, - version: bool = typer.Option(False, "--version", "-v", help="Show version and exit"), + version: bool = typer.Option( + False, "--version", "-v", help="Show version and exit" + ), ): """DeployML CLI - Infrastructure for academia with cost analysis""" if version: @@ -592,7 +688,7 @@ def cli_callback( def doctor( project_id: str = typer.Option( "", "--project-id", "-j", help="GCP Project ID to check APIs (optional)" - ) + ), ): """ Run system checks for required tools and authentication for DeployML. @@ -617,11 +713,19 @@ def doctor( if terraform_installed: tf_ver = get_terraform_version() if tf_ver and tf_ver[0] >= 1: - typer.secho(f"\n Terraform {'.'.join(map(str, tf_ver))} (>= 1.0)", fg=typer.colors.GREEN) + typer.secho( + f"\n Terraform {'.'.join(map(str, tf_ver))} (>= 1.0)", + fg=typer.colors.GREEN, + ) elif tf_ver: - typer.secho(f"\n Terraform {'.'.join(map(str, tf_ver))} found, but requires >= 1.0", fg=typer.colors.RED) + typer.secho( + f"\n Terraform {'.'.join(map(str, tf_ver))} found, but requires >= 1.0", + fg=typer.colors.RED, + ) else: - typer.secho("\n Terraform installed, version unknown", fg=typer.colors.YELLOW) + typer.secho( + "\n Terraform installed, version unknown", fg=typer.colors.YELLOW + ) else: typer.secho("\n Terraform is not installed", fg=typer.colors.RED) @@ -638,15 +742,19 @@ def doctor( # GCP CLI if gcp_installed and gcp_authed: - typer.secho( - "\n GCP CLI installed and authenticated", fg=typer.colors.GREEN - ) + typer.secho("\n GCP CLI installed and authenticated", fg=typer.colors.GREEN) # Check enabled GCP APIs # ADC and bq are required for client libs and BigQuery work if check_gcp_adc(): - typer.secho("\n GCP Application Default Credentials configured", fg=typer.colors.GREEN) + typer.secho( + "\n GCP Application Default Credentials configured", + fg=typer.colors.GREEN, + ) else: - typer.secho("\n GCP Application Default Credentials NOT configured", fg=typer.colors.RED) + typer.secho( + "\n GCP Application Default Credentials NOT configured", + fg=typer.colors.RED, + ) typer.echo(" Fix: gcloud auth application-default login") if check_bq(): typer.secho("\n bq CLI is installed", fg=typer.colors.GREEN) @@ -661,9 +769,7 @@ def doctor( if project_id: project_id = project_id.strip() if project_id: # Check if not empty after stripping - typer.echo( - f"\n Checking enabled APIs for project: {project_id} ..." - ) + typer.echo(f"\n Checking enabled APIs for project: {project_id} ...") result = run_tool( "gcloud", [ @@ -715,7 +821,9 @@ def doctor( ) for r in missing_roles: typer.echo(f" - {r}") - typer.echo(" Fix: grant roles/owner OR each role via gcloud projects add-iam-policy-binding") + typer.echo( + " Fix: grant roles/owner OR each role via gcloud projects add-iam-policy-binding" + ) elif gcp_installed: typer.secho( "\nWARNING: GCP CLI installed but not authenticated", @@ -726,7 +834,7 @@ def doctor( # AWS CLI if aws_installed: - typer.secho(f"\n AWS CLI installed", fg=typer.colors.GREEN) + typer.secho("\n AWS CLI installed", fg=typer.colors.GREEN) else: typer.secho( "\n AWS CLI not installed", @@ -790,10 +898,16 @@ def deploy( False, "--yes", "-y", help="Skip confirmation prompts and deploy" ), generate_only: bool = typer.Option( - False, "--generate-only", "-g", help="Only generate manifests, do not apply (for GKE deployments)" + False, + "--generate-only", + "-g", + help="Only generate manifests, do not apply (for GKE deployments)", ), verbose: bool = typer.Option( - False, "--verbose", "-v", help="Stream Terraform logs to stdout instead of showing progress bar" + False, + "--verbose", + "-v", + help="Stream Terraform logs to stdout instead of showing progress bar", ), ): """ @@ -834,9 +948,7 @@ def deploy( # Set use_postgres param based on backend_store_uri (mlflow only) if tool.get("name") == "mlflow": - backend_uri = tool["params"].get( - "backend_store_uri", "" - ) + backend_uri = tool["params"].get("backend_store_uri", "") tool["params"]["use_postgres"] = backend_uri.startswith( "postgresql" ) @@ -867,14 +979,21 @@ def deploy( f"'{previous_project}'. Config now points at '{project_id}'.", fg=typer.colors.RED, ) - typer.echo(" Run `deployml destroy` first to clean up the old project,") - typer.echo(" or change `name:` in config.yaml to use a fresh workspace.") + typer.echo( + " Run `deployml destroy` first to clean up the old project," + ) + typer.echo( + " or change `name:` in config.yaml to use a fresh workspace." + ) raise typer.Exit(code=1) project_marker.write_text(project_id) region = config["provider"]["region"] if cloud == "gcp" and not validate_gcp_region(region, project_id): - typer.secho(f" Region '{region}' is not valid for GCP. Run: gcloud compute regions list", fg=typer.colors.RED) + typer.secho( + f" Region '{region}' is not valid for GCP. Run: gcloud compute regions list", + fg=typer.colors.RED, + ) raise typer.Exit(code=1) # ADC backs the Terraform google provider, so deploy must verify auth and ADC @@ -891,28 +1010,28 @@ def deploy( typer.echo(" GKE deployment detected") typer.echo(" Using Kubernetes manifests (similar to minikube)") typer.echo(" Images will be pushed to GCR") - + # Extract GKE-specific config gke_config = config.get("gke", {}) cluster_name = gke_config.get("cluster_name") zone = gke_config.get("zone") region_gke = gke_config.get("region") - + if not cluster_name: typer.echo(" GKE cluster_name must be specified in config.gke.cluster_name") raise typer.Exit(code=1) - + if not zone and not region_gke: typer.echo(" Either config.gke.zone or config.gke.region must be specified") raise typer.Exit(code=1) - + typer.echo(f" Cluster: {cluster_name}") typer.echo(f" Location: {zone or region_gke}") - + # Create manifests directory manifests_dir = DEPLOYML_DIR / "manifests" manifests_dir.mkdir(parents=True, exist_ok=True) - + # Generate Kubernetes manifests for each service in stack from deployml.utils.kubernetes_gke import ( generate_mlflow_manifests_gke, @@ -920,30 +1039,34 @@ def deploy( deploy_to_gke, connect_to_gke_cluster, ) - + # Connect to the cluster only when we will actually apply. --generate-only # renders manifests offline, so it can run before the cluster exists. - if not generate_only and not connect_to_gke_cluster(project_id, cluster_name, zone, region_gke): + if not generate_only and not connect_to_gke_cluster( + project_id, cluster_name, zone, region_gke + ): raise typer.Exit(code=1) # Process stack and generate manifests mlflow_manifest_dir = None fastapi_manifest_dir = None - + for stage in stack: for stage_name, tool in stage.items(): if stage_name == "experiment_tracking" and tool.get("name") == "mlflow": params = tool.get("params", {}) - image = params.get("image", f"gcr.io/{project_id}/mlflow/mlflow:latest") + image = params.get( + "image", f"gcr.io/{project_id}/mlflow/mlflow:latest" + ) # Leave as None when unset so the GKE generator picks its # persistent default (sqlite on the mounted PVC). A hardcoded # sqlite:///mlflow.db here would override it with an ephemeral, # container-local store that is wiped on every pod restart. backend_uri = params.get("backend_store_uri") artifact_root = params.get("artifact_root") - + mlflow_manifest_dir = manifests_dir / "mlflow" - typer.echo(f"\n Generating MLflow manifests...") + typer.echo("\n Generating MLflow manifests...") generate_mlflow_manifests_gke( output_dir=mlflow_manifest_dir, image=image, @@ -952,14 +1075,18 @@ def deploy( artifact_root=artifact_root, push_image=not image.startswith("gcr.io/"), ) - + elif stage_name == "model_serving" and tool.get("name") == "fastapi": params = tool.get("params", {}) - image = params.get("image", f"gcr.io/{project_id}/fastapi/fastapi:latest") - mlflow_uri = params.get("mlflow_tracking_uri", "http://mlflow-service:5000") - + image = params.get( + "image", f"gcr.io/{project_id}/fastapi/fastapi:latest" + ) + mlflow_uri = params.get( + "mlflow_tracking_uri", "http://mlflow-service:5000" + ) + fastapi_manifest_dir = manifests_dir / "fastapi" - typer.echo(f"\n Generating FastAPI manifests...") + typer.echo("\n Generating FastAPI manifests...") generate_fastapi_manifests_gke( output_dir=fastapi_manifest_dir, image=image, @@ -967,7 +1094,7 @@ def deploy( mlflow_tracking_uri=mlflow_uri, push_image=not image.startswith("gcr.io/"), ) - + # --generate-only stops here: manifests are rendered but not applied. # The documented flow is to then apply them with `deployml gke-apply`. if generate_only: @@ -978,7 +1105,7 @@ def deploy( # Deploy manifests if mlflow_manifest_dir and mlflow_manifest_dir.exists(): - typer.echo(f"\n Deploying MLflow to GKE...") + typer.echo("\n Deploying MLflow to GKE...") if not deploy_to_gke( manifest_dir=mlflow_manifest_dir, cluster_name=cluster_name, @@ -987,9 +1114,9 @@ def deploy( region=region_gke, ): raise typer.Exit(code=1) - + if fastapi_manifest_dir and fastapi_manifest_dir.exists(): - typer.echo(f"\n Deploying FastAPI to GKE...") + typer.echo("\n Deploying FastAPI to GKE...") if not deploy_to_gke( manifest_dir=fastapi_manifest_dir, cluster_name=cluster_name, @@ -998,11 +1125,11 @@ def deploy( region=region_gke, ): raise typer.Exit(code=1) - + typer.echo("\n GKE deployment complete!") typer.echo(f" Manifests saved to: {manifests_dir}") return - + # Continue with Terraform-based deployments (cloud_run, cloud_vm) # --- PATCH: Ensure cloud_sql_postgres module is copied for mlflow cloud_run with postgres --- if ( @@ -1051,9 +1178,7 @@ def deploy( for stage_name, tool in stage.items(): if tool.get("params", {}).get("artifact_bucket"): bucket_name = tool["params"]["artifact_bucket"] - create_bucket = tool["params"].get( - "create_artifact_bucket", True - ) + create_bucket = tool["params"].get("create_artifact_bucket", True) # Check if bucket already exists bucket_exists_flag = bucket_exists(bucket_name, project_id) @@ -1105,7 +1230,9 @@ def deploy( # Cron job images are per-job and must be set explicitly. Skip here. if stage_name == "workflow_orchestration" and tool_name == "cron": for job in params.get("jobs", []): - if not job.get("image") or job.get("image", "").startswith("gcr.io/"): + if not job.get("image") or job.get("image", "").startswith( + "gcr.io/" + ): job_name = job.get("service_name", "") job["image"] = f"{_ar_base}/{job_name}:{_image_tag}" @@ -1113,35 +1240,23 @@ def deploy( # PATCH: Use wandb_main.tf.j2 or mlflow_main.tf.j2 for cloud_run if present if deployment_type == "cloud_run": if any( - tool.get("name") == "wandb" - for stage in stack - for tool in stage.values() + tool.get("name") == "wandb" for stage in stack for tool in stage.values() ): main_template = env.get_template( f"{cloud}/{deployment_type}/wandb_main.tf.j2" ) elif any( - tool.get("name") == "mlflow" - for stage in stack - for tool in stage.values() + tool.get("name") == "mlflow" for stage in stack for tool in stage.values() ): main_template = env.get_template( f"{cloud}/{deployment_type}/mlflow_main.tf.j2" ) else: - main_template = env.get_template( - f"{cloud}/{deployment_type}/main.tf.j2" - ) + main_template = env.get_template(f"{cloud}/{deployment_type}/main.tf.j2") else: - main_template = env.get_template( - f"{cloud}/{deployment_type}/main.tf.j2" - ) - var_template = env.get_template( - f"{cloud}/{deployment_type}/variables.tf.j2" - ) - tfvars_template = env.get_template( - f"{cloud}/{deployment_type}/terraform.tfvars.j2" - ) + main_template = env.get_template(f"{cloud}/{deployment_type}/main.tf.j2") + var_template = env.get_template(f"{cloud}/{deployment_type}/variables.tf.j2") + tfvars_template = env.get_template(f"{cloud}/{deployment_type}/terraform.tfvars.j2") # Compute a stable short hash for resource names to avoid collisions name_material = f"{workspace_name}:{project_id}".encode("utf-8") @@ -1151,17 +1266,20 @@ def deploy( # Note: This is a preliminary schedule - will be updated after Terraform completes with exact time teardown_cron_schedule = "" teardown_scheduled_timestamp = 0 - + if teardown_enabled: duration_hours = teardown_config.get("duration_hours", 24) # Use timezone-aware UTC. datetime.utcnow() returns a naive datetime, # and .timestamp() on a naive datetime treats it as local time, # corrupting the schedule by the local TZ offset. from datetime import timezone as _tz + deployed_at = datetime.now(_tz.utc) teardown_at = deployed_at + timedelta(hours=duration_hours, minutes=10) teardown_scheduled_timestamp = int(teardown_at.timestamp()) - teardown_cron_schedule = calculate_cron_from_timestamp(teardown_scheduled_timestamp) + teardown_cron_schedule = calculate_cron_from_timestamp( + teardown_scheduled_timestamp + ) # Render templates if deployment_type == "cloud_vm": @@ -1222,7 +1340,8 @@ def deploy( typer.echo(f" Deploying {config.get('name', workspace_name)} to {cloud}...") run_tool( - "gcloud", ["config", "set", "project", project_id], + "gcloud", + ["config", "set", "project", project_id], cwd=DEPLOYML_TERRAFORM_DIR, ) @@ -1230,7 +1349,8 @@ def deploy( # Capture stderr so init failures (state lock, missing ADC, bucket perms) # surface a real message instead of a silent exit. init_proc = run_tool( - "terraform", ["init"], + "terraform", + ["init"], cwd=DEPLOYML_TERRAFORM_DIR, capture_output=True, text=True, @@ -1243,7 +1363,8 @@ def deploy( typer.echo(" Planning deployment...") result = run_tool( - "terraform", ["plan"], + "terraform", + ["plan"], cwd=DEPLOYML_TERRAFORM_DIR, capture_output=True, text=True, @@ -1257,9 +1378,7 @@ def deploy( # Check for cost analysis configuration cost_config = config.get("cost_analysis", {}) cost_enabled = cost_config.get("enabled", True) # Default: enabled - warning_threshold = cost_config.get( - "warning_threshold", 100.0 - ) # Default: $100 + warning_threshold = cost_config.get("warning_threshold", 100.0) # Default: $100 cost_analysis = None if cost_enabled: @@ -1270,12 +1389,12 @@ def deploy( if usage_file is None: try: bucket_amount = cost_config.get("bucket_amount") - cloudsql_amount = cost_config.get( - "cloudSQL_amount" - ) or cost_config.get("cloudsql_amount") - bigquery_amount = cost_config.get( - "bigQuery_amount" - ) or cost_config.get("bigquery_amount") + cloudsql_amount = cost_config.get("cloudSQL_amount") or cost_config.get( + "cloudsql_amount" + ) + bigquery_amount = cost_config.get("bigQuery_amount") or cost_config.get( + "bigquery_amount" + ) resource_type_default_usage = {} # Map high-level amounts to Infracost resource defaults @@ -1284,9 +1403,9 @@ def deploy( "storage_gb": float(bucket_amount) } if cloudsql_amount is not None: - resource_type_default_usage[ - "google_sql_database_instance" - ] = {"storage_gb": float(cloudsql_amount)} + resource_type_default_usage["google_sql_database_instance"] = { + "storage_gb": float(cloudsql_amount) + } if bigquery_amount is not None: resource_type_default_usage["google_bigquery_table"] = { "storage_gb": float(bigquery_amount) @@ -1322,7 +1441,8 @@ def deploy( typer.echo(f" Applying changes... (Estimated time: {estimated_time})") # Re-init before apply; capture stderr to surface failures init_proc2 = run_tool( - "terraform", ["init"], + "terraform", + ["init"], cwd=DEPLOYML_TERRAFORM_DIR, capture_output=True, text=True, @@ -1347,84 +1467,115 @@ def deploy( ) if result_code == 0: typer.echo(" Deployment complete!") - + # Upload Terraform files and resource manifest to GCS for teardown (if enabled) if teardown_enabled: try: typer.echo(" Uploading Terraform files to GCS for teardown...") - upload_terraform_files_to_gcs(DEPLOYML_TERRAFORM_DIR, project_id, workspace_name) - + upload_terraform_files_to_gcs( + DEPLOYML_TERRAFORM_DIR, project_id, workspace_name + ) + # Extract and upload resource manifest typer.echo(" Extracting resource manifest...") manifest = extract_resource_manifest( - DEPLOYML_TERRAFORM_DIR, - project_id, - workspace_name, - region + DEPLOYML_TERRAFORM_DIR, project_id, workspace_name, region + ) + upload_resource_manifest( + manifest, DEPLOYML_TERRAFORM_DIR, project_id, workspace_name ) - upload_resource_manifest(manifest, DEPLOYML_TERRAFORM_DIR, project_id, workspace_name) typer.echo(" Resource manifest uploaded successfully") - + except Exception as e: - typer.echo(f"WARNING: Warning: Could not upload files/manifest to GCS: {e}") - typer.echo(" Teardown may not work automatically. Manual teardown required.") - + typer.echo( + f"WARNING: Warning: Could not upload files/manifest to GCS: {e}" + ) + typer.echo( + " Teardown may not work automatically. Manual teardown required." + ) + # Handle auto-teardown metadata and update scheduler schedule if teardown_enabled: duration_hours = teardown_config.get("duration_hours", 24) # Timezone-aware UTC. Avoid datetime.utcnow() to keep .timestamp() correct. from datetime import timezone as _tz + deployed_at = datetime.now(_tz.utc) teardown_at = deployed_at + timedelta(hours=duration_hours) teardown_scheduled_timestamp = int(teardown_at.timestamp()) - correct_cron_schedule = calculate_cron_from_timestamp(teardown_scheduled_timestamp) + correct_cron_schedule = calculate_cron_from_timestamp( + teardown_scheduled_timestamp + ) time_zone = teardown_config.get("time_zone", "UTC") - + # Update the Cloud Scheduler job with the correct schedule scheduler_job_name = f"deployml-teardown-{workspace_name}" try: - typer.echo(f" Updating teardown schedule to: {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}") + typer.echo( + f" Updating teardown schedule to: {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}" + ) update_result = run_tool( "gcloud", [ - "scheduler", "jobs", "update", "http", scheduler_job_name, - "--location", region, - "--schedule", correct_cron_schedule, - "--time-zone", time_zone, - "--project", project_id, - "--quiet" + "scheduler", + "jobs", + "update", + "http", + scheduler_job_name, + "--location", + region, + "--schedule", + correct_cron_schedule, + "--time-zone", + time_zone, + "--project", + project_id, + "--quiet", ], capture_output=True, text=True, ) if update_result.returncode == 0: - typer.echo(f" Teardown schedule updated successfully") + typer.echo(" Teardown schedule updated successfully") else: - typer.echo(f"WARNING: Warning: Could not update scheduler schedule: {update_result.stderr}") - typer.echo(f" Schedule may be incorrect. Check manually with:") - typer.echo(f" gcloud scheduler jobs describe {scheduler_job_name} --location={region} --project={project_id}") + typer.echo( + f"WARNING: Warning: Could not update scheduler schedule: {update_result.stderr}" + ) + typer.echo(" Schedule may be incorrect. Check manually with:") + typer.echo( + f" gcloud scheduler jobs describe {scheduler_job_name} --location={region} --project={project_id}" + ) except Exception as e: - typer.echo(f"WARNING: Warning: Could not update scheduler schedule: {e}") - typer.echo(f" Schedule may be incorrect. Check manually with:") - typer.echo(f" gcloud scheduler jobs describe {scheduler_job_name} --location={region} --project={project_id}") - + typer.echo( + f"WARNING: Warning: Could not update scheduler schedule: {e}" + ) + typer.echo(" Schedule may be incorrect. Check manually with:") + typer.echo( + f" gcloud scheduler jobs describe {scheduler_job_name} --location={region} --project={project_id}" + ) + metadata = { "deployed_at": deployed_at.isoformat(), "teardown_scheduled_at": teardown_at.isoformat(), "teardown_enabled": True, "duration_hours": duration_hours, - "scheduler_job_name": scheduler_job_name + "scheduler_job_name": scheduler_job_name, } save_deployment_metadata(DEPLOYML_DIR, metadata) - - typer.echo(f"\n Auto-teardown scheduled for: {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}") + + typer.echo( + f"\n Auto-teardown scheduled for: {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}" + ) typer.echo(f" (in {duration_hours} hours)") - typer.echo(f" To cancel: deployml teardown cancel --config-path {config_path}") - + typer.echo( + f" To cancel: deployml teardown cancel --config-path {config_path}" + ) + # Show all Terraform outputs in a user-friendly way output_proc = run_tool( - "terraform", ["output", "-json"], + "terraform", + ["output", "-json"], cwd=DEPLOYML_TERRAFORM_DIR, capture_output=True, text=True, @@ -1436,7 +1587,6 @@ def deploy( typer.echo("\n DeployML Outputs:") for key, value in outputs.items(): is_sensitive = value.get("sensitive", False) - output_type = value.get("type") output_val = value.get("value") if is_sensitive: typer.secho( @@ -1455,9 +1605,7 @@ def deploy( fg=typer.colors.BRIGHT_BLUE, bold=True, ) - elif ( - isinstance(subval, str) and subval == "" - ): + elif isinstance(subval, str) and subval == "": typer.secho( f" {subkey}: [No value] (likely using SQLite or not applicable)", fg=typer.colors.YELLOW, @@ -1492,17 +1640,23 @@ def deploy( typer.echo("WARNING:Could not retrieve Terraform outputs.") else: log_file = DEPLOYML_TERRAFORM_DIR / "terraform_apply.log" - typer.secho(f"\n Terraform apply failed with exit code {result_code}", fg=typer.colors.RED, bold=True) - typer.echo(f"\n Check the Terraform log for details:") + typer.secho( + f"\n Terraform apply failed with exit code {result_code}", + fg=typer.colors.RED, + bold=True, + ) + typer.echo("\n Check the Terraform log for details:") typer.echo(f" {log_file}") - typer.echo(f"\n Common issues:") - typer.echo(" - Required GCP APIs may not be enabled (check log for API activation URLs)") + typer.echo("\n Common issues:") + typer.echo( + " - Required GCP APIs may not be enabled (check log for API activation URLs)" + ) typer.echo(" - Insufficient IAM permissions") typer.echo(" - Resource conflicts or quota limits") typer.echo("\n Last 20 lines of the log:") if log_file.exists(): try: - with open(log_file, 'r') as f: + with open(log_file, "r") as f: lines = f.readlines() for line in lines[-20:]: typer.echo(f" {line.rstrip()}") @@ -1522,7 +1676,9 @@ def get_urls( Path(".env"), "--env-path", help="Path to write .env file" ), show_secrets: bool = typer.Option( - False, "--show-secrets", help="Fetch and print Grafana admin password and MLflow DSN connection hint. Uses gcloud secrets versions access." + False, + "--show-secrets", + help="Fetch and print Grafana admin password and MLflow DSN connection hint. Uses gcloud secrets versions access.", ), ): """ @@ -1538,11 +1694,14 @@ def get_urls( terraform_dir = Path.cwd() / ".deployml" / workspace_name / "terraform" if not terraform_dir.exists(): - typer.echo(f" No deployment found at {terraform_dir}. Run 'deployml deploy' first.") + typer.echo( + f" No deployment found at {terraform_dir}. Run 'deployml deploy' first." + ) raise typer.Exit(code=1) output_proc = run_tool( - "terraform", ["output", "-json"], + "terraform", + ["output", "-json"], cwd=terraform_dir, capture_output=True, text=True, @@ -1571,7 +1730,9 @@ def get_urls( typer.secho(f" {key}: [SENSITIVE] (value hidden)", fg=typer.colors.YELLOW) elif isinstance(output_val, str): if output_val.startswith("http://") or output_val.startswith("https://"): - typer.secho(f" {key}: {output_val}", fg=typer.colors.BRIGHT_BLUE, bold=True) + typer.secho( + f" {key}: {output_val}", fg=typer.colors.BRIGHT_BLUE, bold=True + ) env_lines.append(f"{env_key}={output_val}") elif output_val == "": typer.secho(f" {key}: [No value]", fg=typer.colors.YELLOW) @@ -1582,8 +1743,14 @@ def get_urls( typer.echo(f" {key}:") for subkey, subval in output_val.items(): sub_env_key = f"{env_key}_{subkey.upper()}" - if isinstance(subval, str) and (subval.startswith("http://") or subval.startswith("https://")): - typer.secho(f" {subkey}: {subval}", fg=typer.colors.BRIGHT_BLUE, bold=True) + if isinstance(subval, str) and ( + subval.startswith("http://") or subval.startswith("https://") + ): + typer.secho( + f" {subkey}: {subval}", + fg=typer.colors.BRIGHT_BLUE, + bold=True, + ) env_lines.append(f"{sub_env_key}={subval}") elif isinstance(subval, str) and subval: typer.echo(f" {subkey}: {subval}") @@ -1601,29 +1768,49 @@ def get_urls( if show_secrets: typer.secho("\n Secrets:", fg=typer.colors.YELLOW, bold=True) # Grafana admin password - grafana_secret = outputs.get("grafana_admin_password_secret_id", {}).get("value", "") + grafana_secret = outputs.get("grafana_admin_password_secret_id", {}).get( + "value", "" + ) if grafana_secret and project_id: fetch = run_tool( - "gcloud", ["secrets", "versions", "access", "latest", - "--secret", grafana_secret, "--project", project_id], - capture_output=True, text=True, + "gcloud", + [ + "secrets", + "versions", + "access", + "latest", + "--secret", + grafana_secret, + "--project", + project_id, + ], + capture_output=True, + text=True, ) if fetch.returncode == 0: - typer.echo(f" grafana_admin_user: admin") + typer.echo(" grafana_admin_user: admin") typer.echo(f" grafana_admin_password: {fetch.stdout.strip()}") else: - typer.echo(f" grafana_admin_password: (fetch failed: {fetch.stderr.strip()})") + typer.echo( + f" grafana_admin_password: (fetch failed: {fetch.stderr.strip()})" + ) # MLflow DSN. Public IP is blocked; print the Cloud SQL Auth Proxy steps. instance = outputs.get("instance_connection_name", {}).get("value", "") dsn_secret = outputs.get("mlflow_dsn_secret_id", {}).get("value", "") if instance and dsn_secret and project_id: typer.echo("") - typer.echo(" To connect to MLflow Postgres from your laptop, run the Cloud SQL Auth Proxy:") + typer.echo( + " To connect to MLflow Postgres from your laptop, run the Cloud SQL Auth Proxy:" + ) typer.echo(f" cloud-sql-proxy {instance} --port=5432") typer.echo(" Install the proxy if you do not have it:") - typer.echo(" https://cloud.google.com/sql/docs/postgres/sql-proxy#install") + typer.echo( + " https://cloud.google.com/sql/docs/postgres/sql-proxy#install" + ) typer.echo(" Fetch the DSN with:") - typer.echo(f" gcloud secrets versions access latest --secret={dsn_secret} --project={project_id}") + typer.echo( + f" gcloud secrets versions access latest --secret={dsn_secret} --project={project_id}" + ) @cli.command() @@ -1641,12 +1828,14 @@ def destroy( False, "--yes", "-y", help="Skip confirmation prompts and destroy" ), keep_images: bool = typer.Option( - False, "--keep-images", + False, + "--keep-images", help="Keep the Artifact Registry repo and Cloud Build staging bucket. " "Use when other workspaces in the same project share the images.", ), repository: str = typer.Option( - "mlops-images", "--repository", + "mlops-images", + "--repository", help="Artifact Registry repository to delete after destroy. " "Match the --repository you passed to build-images.", ), @@ -1666,13 +1855,10 @@ def destroy( # Find the workspace DEPLOYML_DIR = Path.cwd() / ".deployml" / workspace_name DEPLOYML_TERRAFORM_DIR = DEPLOYML_DIR / "terraform" - DEPLOYML_MODULES_DIR = DEPLOYML_DIR / "terraform" / "modules" if not DEPLOYML_TERRAFORM_DIR.exists(): typer.echo(f"WARNING:No workspace found for {workspace_name}") - typer.echo( - "Nothing to destroy - infrastructure may already be cleaned up." - ) + typer.echo("Nothing to destroy - infrastructure may already be cleaned up.") return _validate_deploy_config_or_exit(config) @@ -1689,18 +1875,17 @@ def destroy( typer.echo(f"šŸŒ Project: {project_id}") typer.echo("This will permanently delete all resources!") - if not ( - yes or typer.confirm("Are you sure you want to destroy all resources?") - ): + if not (yes or typer.confirm("Are you sure you want to destroy all resources?")): typer.echo(" Destroy cancelled") return try: - typer.echo(f" Destroying infrastructure...") + typer.echo(" Destroying infrastructure...") # Set GCP project run_tool( - "gcloud", ["config", "set", "project", project_id], + "gcloud", + ["config", "set", "project", project_id], cwd=DEPLOYML_TERRAFORM_DIR, ) @@ -1709,21 +1894,38 @@ def destroy( # prevent database/user deletion and the destroy fails. region = config.get("provider", {}).get("region", "us-central1") cr_result = run_tool( - "gcloud", ["run", "services", "list", - "--project", project_id, - "--region", region, - "--format", "value(metadata.name)"], - capture_output=True, text=True + "gcloud", + [ + "run", + "services", + "list", + "--project", + project_id, + "--region", + region, + "--format", + "value(metadata.name)", + ], + capture_output=True, + text=True, ) if cr_result.returncode == 0: services = [s.strip() for s in cr_result.stdout.splitlines() if s.strip()] for service in services: typer.echo(f" Deleting Cloud Run service: {service}") run_tool( - "gcloud", ["run", "services", "delete", service, - "--project", project_id, - "--region", region, - "--quiet"], + "gcloud", + [ + "run", + "services", + "delete", + service, + "--project", + project_id, + "--region", + region, + "--quiet", + ], capture_output=True, ) @@ -1731,23 +1933,29 @@ def destroy( # doesn't try to delete them individually ā€” the instance deletion handles # that automatically, avoiding active-connection errors on destroy. state_result = run_tool( - "terraform", ["state", "list"], + "terraform", + ["state", "list"], cwd=DEPLOYML_TERRAFORM_DIR, capture_output=True, text=True, ) if state_result.returncode == 0: resources_to_remove = [ - r.strip() for r in state_result.stdout.splitlines() - if any(x in r for x in [ - "google_sql_database.", - "google_sql_user.", - ]) + r.strip() + for r in state_result.stdout.splitlines() + if any( + x in r + for x in [ + "google_sql_database.", + "google_sql_user.", + ] + ) ] for resource in resources_to_remove: typer.echo(f" Removing from state: {resource}") run_tool( - "terraform", ["state", "rm", resource], + "terraform", + ["state", "rm", resource], cwd=DEPLOYML_TERRAFORM_DIR, capture_output=True, ) @@ -1780,8 +1988,18 @@ def destroy( ): typer.echo(f" Removing Artifact Registry repo {repository}...") run_tool( - "gcloud", ["artifacts", "repositories", "delete", repository, - "--location", region, "--project", project_id, "--quiet"], + "gcloud", + [ + "artifacts", + "repositories", + "delete", + repository, + "--location", + region, + "--project", + project_id, + "--quiet", + ], capture_output=True, ) @@ -1790,11 +2008,14 @@ def destroy( # on the next build if needed. typer.echo(f" Removing Cloud Build staging bucket {cb_bucket}...") run_tool( - "gcloud", ["storage", "rm", "--recursive", cb_bucket, "--quiet"], + "gcloud", + ["storage", "rm", "--recursive", cb_bucket, "--quiet"], capture_output=True, ) else: - typer.echo(f" Keeping Artifact Registry repo {repository} and {cb_bucket}.") + typer.echo( + f" Keeping Artifact Registry repo {repository} and {cb_bucket}." + ) if clean_workspace: typer.echo(" Cleaning workspace...") @@ -1812,10 +2033,14 @@ def destroy( ) typer.echo(f" {DEPLOYML_TERRAFORM_DIR}") typer.echo("\nRecovery:") - typer.echo(" 1. Inspect residual resources: gcloud asset search-all-resources " - f"--scope=projects/{project_id}") - typer.echo(f" 2. Re-run: deployml destroy --yes") - typer.echo(f" 3. Or delete the whole project: gcloud projects delete {project_id}") + typer.echo( + " 1. Inspect residual resources: gcloud asset search-all-resources " + f"--scope=projects/{project_id}" + ) + typer.echo(" 2. Re-run: deployml destroy --yes") + typer.echo( + f" 3. Or delete the whole project: gcloud projects delete {project_id}" + ) raise typer.Exit(code=1) except Exception as e: @@ -1842,34 +2067,56 @@ def status( typer.echo(f"Workspace: {workspace_name}") typer.echo(f"Path: {deployml_dir}") if not tf_dir.exists(): - typer.secho("Status: not deployed (no terraform workspace found)", fg=typer.colors.YELLOW) + typer.secho( + "Status: not deployed (no terraform workspace found)", + fg=typer.colors.YELLOW, + ) raise typer.Exit(code=0) marker = deployml_dir / ".project_id" if marker.exists(): typer.echo(f"Project: {marker.read_text().strip()}") out_proc = run_tool( - "terraform", ["output", "-json"], - cwd=tf_dir, capture_output=True, text=True, + "terraform", + ["output", "-json"], + cwd=tf_dir, + capture_output=True, + text=True, ) if out_proc.returncode == 0 and out_proc.stdout.strip(): try: outputs = json.loads(out_proc.stdout) - urls = {k: v.get("value") for k, v in outputs.items() if isinstance(v.get("value"), str) and v.get("value", "").startswith("http")} + urls = { + k: v.get("value") + for k, v in outputs.items() + if isinstance(v.get("value"), str) + and v.get("value", "").startswith("http") + } if urls: typer.secho("Status: deployed", fg=typer.colors.GREEN) for k, v in urls.items(): typer.echo(f" {k}: {v}") else: - typer.secho("Status: workspace exists but no URL outputs found", fg=typer.colors.YELLOW) + typer.secho( + "Status: workspace exists but no URL outputs found", + fg=typer.colors.YELLOW, + ) except Exception: - typer.secho("Status: workspace exists but terraform output is not parseable", fg=typer.colors.YELLOW) + typer.secho( + "Status: workspace exists but terraform output is not parseable", + fg=typer.colors.YELLOW, + ) else: - typer.secho("Status: workspace exists but terraform output is empty", fg=typer.colors.YELLOW) + typer.secho( + "Status: workspace exists but terraform output is empty", + fg=typer.colors.YELLOW, + ) @cli.command() def teardown( - action: str = typer.Argument(..., help="Action: cancel, status, update, or schedule"), + action: str = typer.Argument( + ..., help="Action: cancel, status, update, or schedule" + ), config_path: Path = typer.Option( ..., "--config-path", "-c", help="Path to YAML config file" ), @@ -1897,7 +2144,9 @@ def teardown( elif action == "schedule": schedule_teardown(config, DEPLOYML_DIR, workspace_name, hours) else: - typer.echo(f" Unknown action: {action}. Use: cancel, status, update, or schedule") + typer.echo( + f" Unknown action: {action}. Use: cancel, status, update, or schedule" + ) raise typer.Exit(code=1) @@ -1905,17 +2154,27 @@ def cancel_teardown(config: dict, deployml_dir: Path, workspace_name: str): """Cancel scheduled teardown.""" project_id = config["provider"]["project_id"] region = config["provider"]["region"] - + # Delete Cloud Scheduler job. Cloud Scheduler uses --location, not --region. # Earlier code passed --region which gcloud rejects, so cancel silently failed. scheduler_job_name = f"deployml-teardown-{workspace_name}" result = run_tool( - "gcloud", ["scheduler", "jobs", "delete", scheduler_job_name, - "--project", project_id, "--location", region, "--quiet"], + "gcloud", + [ + "scheduler", + "jobs", + "delete", + scheduler_job_name, + "--project", + project_id, + "--location", + region, + "--quiet", + ], capture_output=True, text=True, ) - + if result.returncode == 0: typer.echo(" Scheduled teardown cancelled") # Update metadata @@ -1925,7 +2184,9 @@ def cancel_teardown(config: dict, deployml_dir: Path, workspace_name: str): save_deployment_metadata(deployml_dir, metadata) else: typer.echo(f"WARNING:Could not cancel teardown: {result.stderr}") - typer.echo(" The scheduler job may not exist or may have already been deleted.") + typer.echo( + " The scheduler job may not exist or may have already been deleted." + ) def show_teardown_status(config: dict, deployml_dir: Path, workspace_name: str): @@ -1933,63 +2194,77 @@ def show_teardown_status(config: dict, deployml_dir: Path, workspace_name: str): project_id = config["provider"]["project_id"] region = config["provider"]["region"] scheduler_job_name = f"deployml-teardown-{workspace_name}" - + # Query Cloud Scheduler job result = run_tool( - "gcloud", ["scheduler", "jobs", "describe", scheduler_job_name, - "--project", project_id, "--location", region, "--format", "json"], + "gcloud", + [ + "scheduler", + "jobs", + "describe", + scheduler_job_name, + "--project", + project_id, + "--location", + region, + "--format", + "json", + ], capture_output=True, text=True, ) - + if result.returncode != 0: typer.echo(f"WARNING:Cloud Scheduler job not found: {scheduler_job_name}") - typer.echo(" Teardown may not be scheduled or may have already been cancelled.") - + typer.echo( + " Teardown may not be scheduled or may have already been cancelled." + ) + # Check local metadata as fallback metadata = load_deployment_metadata(deployml_dir) if metadata and metadata.get("teardown_enabled"): teardown_at = datetime.fromisoformat(metadata["teardown_scheduled_at"]) - typer.echo(f"\n Local metadata shows teardown was scheduled for:") + typer.echo("\n Local metadata shows teardown was scheduled for:") typer.echo(f" {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}") return - + # Parse Cloud Scheduler job details try: job_info = json.loads(result.stdout) except json.JSONDecodeError: typer.echo(" Failed to parse Cloud Scheduler job information") return - + # Extract information schedule = job_info.get("schedule", "N/A") time_zone = job_info.get("timeZone", "UTC") state = job_info.get("state", "UNKNOWN") schedule_time = job_info.get("scheduleTime", "") last_attempt_time = job_info.get("lastAttemptTime", "") - + # Display comprehensive status typer.echo(" Auto-Teardown Status") typer.echo("=" * 60) - + # Job state typer.echo(f"Status: {state}") - + # Cron schedule typer.echo(f" Cron Schedule: {schedule}") typer.echo(f" Timezone: {time_zone}") - + # Next execution time if schedule_time: try: # Parse ISO format with Z suffix (UTC) - time_str = schedule_time.replace('Z', '+00:00') + time_str = schedule_time.replace("Z", "+00:00") next_run = datetime.fromisoformat(time_str) # Get current UTC time as timezone-aware from datetime import timezone + now = datetime.now(timezone.utc) typer.echo(f" Next Execution: {next_run.strftime('%Y-%m-%d %H:%M:%S UTC')}") - + if now < next_run: time_remaining = next_run - now hours = int(time_remaining.total_seconds() // 3600) @@ -1999,86 +2274,106 @@ def show_teardown_status(config: dict, deployml_dir: Path, workspace_name: str): time_passed = now - next_run hours_passed = int(time_passed.total_seconds() // 3600) minutes_passed = int((time_passed.total_seconds() % 3600) // 60) - typer.echo(f" WARNING: Scheduled time passed {hours_passed}h {minutes_passed}m ago") - except Exception as e: + typer.echo( + f" WARNING: Scheduled time passed {hours_passed}h {minutes_passed}m ago" + ) + except Exception: typer.echo(f" Next Execution: {schedule_time}") - + # Last attempt if last_attempt_time and last_attempt_time != "1970-01-01T00:00:00Z": try: # Parse ISO format with Z suffix (UTC) - time_str = last_attempt_time.replace('Z', '+00:00') + time_str = last_attempt_time.replace("Z", "+00:00") last_run = datetime.fromisoformat(time_str) typer.echo(f" Last Execution: {last_run.strftime('%Y-%m-%d %H:%M:%S UTC')}") except Exception: typer.echo(f" Last Execution: {last_attempt_time}") else: typer.echo(" Last Execution: Never") - + # Job name for reference typer.echo(f"\n Job Name: {scheduler_job_name}") - + # Actions typer.echo("\n Actions:") - typer.echo(f" Update: deployml teardown update --config-path ") - typer.echo(f" Cancel: deployml teardown cancel --config-path ") - typer.echo(f" View in Console: https://console.cloud.google.com/cloudscheduler/jobs/edit/{region}/{scheduler_job_name}?project={project_id}") + typer.echo(" Update: deployml teardown update --config-path ") + typer.echo(" Cancel: deployml teardown cancel --config-path ") + typer.echo( + f" View in Console: https://console.cloud.google.com/cloudscheduler/jobs/edit/{region}/{scheduler_job_name}?project={project_id}" + ) -def update_teardown_schedule(config: dict, deployml_dir: Path, workspace_name: str, duration_hours: int = 24): +def update_teardown_schedule( + config: dict, deployml_dir: Path, workspace_name: str, duration_hours: int = 24 +): """Update the scheduled teardown time.""" project_id = config["provider"]["project_id"] region = config["provider"]["region"] scheduler_job_name = f"deployml-teardown-{workspace_name}" - + # Check if Cloud Scheduler job exists result = run_tool( - "gcloud", ["scheduler", "jobs", "describe", scheduler_job_name, - "--project", project_id, "--location", region, "--format", "json"], + "gcloud", + [ + "scheduler", + "jobs", + "describe", + scheduler_job_name, + "--project", + project_id, + "--location", + region, + "--format", + "json", + ], capture_output=True, text=True, ) - + if result.returncode != 0: typer.echo(f" Cloud Scheduler job not found: {scheduler_job_name}") typer.echo(" Cannot update schedule. The teardown job may not exist.") typer.echo(" Use 'deployml deploy' with teardown.enabled: true to create it.") raise typer.Exit(code=1) - + # Get current job info to preserve timezone try: job_info = json.loads(result.stdout) time_zone = job_info.get("timeZone", "UTC") except json.JSONDecodeError: time_zone = "UTC" - + # Prompt for new schedule typer.echo(" Update Teardown Schedule") typer.echo("=" * 60) - + # Show current schedule try: schedule_time = job_info.get("scheduleTime", "") if schedule_time: - time_str = schedule_time.replace('Z', '+00:00') + time_str = schedule_time.replace("Z", "+00:00") current_time = datetime.fromisoformat(time_str) - typer.echo(f" Current Schedule: {current_time.strftime('%Y-%m-%d %H:%M:%S UTC')}") + typer.echo( + f" Current Schedule: {current_time.strftime('%Y-%m-%d %H:%M:%S UTC')}" + ) except Exception: pass - + # Duration is now passed in via CLI flag instead of interactive prompt. if duration_hours < 0: typer.echo(" Duration must be positive") raise typer.Exit(code=1) - + # Calculate new teardown time from datetime import timezone + now = datetime.now(timezone.utc) teardown_at = now + timedelta(hours=duration_hours) # Use UTC timestamp directly to avoid timezone issues teardown_scheduled_timestamp = int(teardown_at.timestamp()) new_cron_schedule = calculate_cron_from_timestamp(teardown_scheduled_timestamp) - + typer.echo(f"\n New Schedule: {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}") # Update Cloud Scheduler job @@ -2087,86 +2382,126 @@ def update_teardown_schedule(config: dict, deployml_dir: Path, workspace_name: s update_result = run_tool( "gcloud", [ - "scheduler", "jobs", "update", "http", scheduler_job_name, - "--location", region, - "--schedule", new_cron_schedule, - "--time-zone", time_zone, - "--project", project_id, - "--quiet" + "scheduler", + "jobs", + "update", + "http", + scheduler_job_name, + "--location", + region, + "--schedule", + new_cron_schedule, + "--time-zone", + time_zone, + "--project", + project_id, + "--quiet", ], capture_output=True, text=True, ) - + if update_result.returncode != 0: typer.echo(f" Failed to update schedule: {update_result.stderr}") if update_result.stdout: typer.echo(f" stdout: {update_result.stdout}") raise typer.Exit(code=1) - + # Verify the update by querying the job again verify_result = run_tool( - "gcloud", ["scheduler", "jobs", "describe", scheduler_job_name, - "--project", project_id, "--location", region, "--format", "json"], + "gcloud", + [ + "scheduler", + "jobs", + "describe", + scheduler_job_name, + "--project", + project_id, + "--location", + region, + "--format", + "json", + ], capture_output=True, text=True, ) - + if verify_result.returncode == 0: try: updated_job_info = json.loads(verify_result.stdout) updated_schedule_time = updated_job_info.get("scheduleTime", "") updated_schedule = updated_job_info.get("schedule", "") - + if updated_schedule_time: - time_str = updated_schedule_time.replace('Z', '+00:00') + time_str = updated_schedule_time.replace("Z", "+00:00") actual_time = datetime.fromisoformat(time_str) - typer.echo(f"\n Teardown schedule updated successfully!") - typer.echo(f" Scheduled time: {actual_time.strftime('%Y-%m-%d %H:%M:%S UTC')}") + typer.echo("\n Teardown schedule updated successfully!") + typer.echo( + f" Scheduled time: {actual_time.strftime('%Y-%m-%d %H:%M:%S UTC')}" + ) typer.echo(f" Cron schedule: {updated_schedule}") - + # Check if it matches what we intended - if actual_time.strftime('%Y-%m-%d %H:%M') != teardown_at.strftime('%Y-%m-%d %H:%M'): - typer.echo(f"\nWARNING: Warning: Scheduled time differs from intended time") - typer.echo(f" Intended: {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}") - typer.echo(f" Actual: {actual_time.strftime('%Y-%m-%d %H:%M:%S UTC')}") + if actual_time.strftime("%Y-%m-%d %H:%M") != teardown_at.strftime( + "%Y-%m-%d %H:%M" + ): + typer.echo( + "\nWARNING: Warning: Scheduled time differs from intended time" + ) + typer.echo( + f" Intended: {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}" + ) + typer.echo( + f" Actual: {actual_time.strftime('%Y-%m-%d %H:%M:%S UTC')}" + ) else: - typer.echo(f" Teardown schedule updated successfully!") + typer.echo(" Teardown schedule updated successfully!") typer.echo(f" Cron schedule: {updated_schedule}") except Exception as e: typer.echo(f" Teardown schedule updated (verification failed: {e})") else: - typer.echo(f" Teardown schedule updated successfully!") - typer.echo(f" (Could not verify - check with: gcloud scheduler jobs describe {scheduler_job_name} --location={region} --project={project_id})") - + typer.echo(" Teardown schedule updated successfully!") + typer.echo( + f" (Could not verify - check with: gcloud scheduler jobs describe {scheduler_job_name} --location={region} --project={project_id})" + ) + # Update local metadata metadata = load_deployment_metadata(deployml_dir) or {} - metadata.update({ - "deployed_at": metadata.get("deployed_at", now.isoformat()), - "teardown_scheduled_at": teardown_at.isoformat(), - "teardown_enabled": True, - "duration_hours": duration_hours, - "scheduler_job_name": scheduler_job_name - }) + metadata.update( + { + "deployed_at": metadata.get("deployed_at", now.isoformat()), + "teardown_scheduled_at": teardown_at.isoformat(), + "teardown_enabled": True, + "duration_hours": duration_hours, + "scheduler_job_name": scheduler_job_name, + } + ) save_deployment_metadata(deployml_dir, metadata) -def schedule_teardown(config: dict, deployml_dir: Path, workspace_name: str, duration_hours: int = 24): +def schedule_teardown( + config: dict, deployml_dir: Path, workspace_name: str, duration_hours: int = 24 +): """Schedule a new teardown.""" from datetime import timezone as _tz + deployed_at = datetime.now(_tz.utc) teardown_at = deployed_at + timedelta(hours=duration_hours) - + metadata = { "deployed_at": deployed_at.isoformat(), "teardown_scheduled_at": teardown_at.isoformat(), "teardown_enabled": True, - "duration_hours": duration_hours + "duration_hours": duration_hours, } save_deployment_metadata(deployml_dir, metadata) - - typer.echo(f" Teardown scheduled for: {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}") - typer.echo("WARNING:Note: This only updates local metadata. To actually schedule teardown,") + + typer.echo( + f" Teardown scheduled for: {teardown_at.strftime('%Y-%m-%d %H:%M:%S UTC')}" + ) + typer.echo( + "WARNING:Note: This only updates local metadata. To actually schedule teardown," + ) typer.echo(" you need to redeploy with teardown.enabled: true in your config.") @@ -2201,18 +2536,25 @@ def init( typer.echo(" --project-id is required for GCP.") raise typer.Exit(code=1) if not check_gcp_auth(): - typer.secho(" gcloud is not authenticated. Run: gcloud auth login", fg=typer.colors.RED) + typer.secho( + " gcloud is not authenticated. Run: gcloud auth login", + fg=typer.colors.RED, + ) raise typer.Exit(code=1) if not check_gcp_adc(): - typer.secho(" Application Default Credentials missing. Run: gcloud auth application-default login", fg=typer.colors.RED) + typer.secho( + " Application Default Credentials missing. Run: gcloud auth application-default login", + fg=typer.colors.RED, + ) raise typer.Exit(code=1) if not validate_gcp_project(project_id): - typer.secho(f" Project '{project_id}' not found or not accessible by your gcloud account.", fg=typer.colors.RED) + typer.secho( + f" Project '{project_id}' not found or not accessible by your gcloud account.", + fg=typer.colors.RED, + ) typer.echo(" Verify with: gcloud projects describe " + project_id) raise typer.Exit(code=1) - typer.echo( - f" Enabling required GCP APIs for project: {project_id} ..." - ) + typer.echo(f" Enabling required GCP APIs for project: {project_id} ...") result = run_tool( "gcloud", [ @@ -2221,7 +2563,7 @@ def init( *REQUIRED_GCP_APIS, "--project", project_id, - ] + ], ) if result.returncode == 0: typer.echo(" All required GCP APIs are enabled.") @@ -2239,7 +2581,7 @@ def init( else: typer.echo(f" Unknown provider: {provider}") raise typer.Exit(code=1) - + project_root = Path(path) try: project_root.mkdir(parents=True, exist_ok=True) @@ -2273,11 +2615,38 @@ def init( }, "deployment": {"type": "cloud_run"}, "stack": [ - {"experiment_tracking": {"name": "mlflow", "params": {"service_name": "mlflow-server"}}}, - {"artifact_tracking": {"name": "mlflow", "params": {"artifact_bucket": f"mlflow-artifacts-{project_id}"}}}, - {"model_registry": {"name": "mlflow", "params": {"backend_store_uri": "postgresql"}}}, - {"model_serving": {"name": "fastapi", "params": {"service_name": "fastapi-mlflow-server"}}}, - {"model_monitoring": {"name": "grafana", "params": {"service_name": "grafana-server"}}}, + { + "experiment_tracking": { + "name": "mlflow", + "params": {"service_name": "mlflow-server"}, + } + }, + { + "artifact_tracking": { + "name": "mlflow", + "params": { + "artifact_bucket": f"mlflow-artifacts-{project_id}" + }, + } + }, + { + "model_registry": { + "name": "mlflow", + "params": {"backend_store_uri": "postgresql"}, + } + }, + { + "model_serving": { + "name": "fastapi", + "params": {"service_name": "fastapi-mlflow-server"}, + } + }, + { + "model_monitoring": { + "name": "grafana", + "params": {"service_name": "grafana-server"}, + } + }, ], } else: @@ -2314,15 +2683,14 @@ def minikube_init( output_dir: Path = typer.Option( ..., "--output-dir", "-o", help="Directory to create Kubernetes manifests" ), - image: str = typer.Option( - ..., "--image", "-i", help="FastAPI Docker image" - ), + image: str = typer.Option(..., "--image", "-i", help="FastAPI Docker image"), mlflow_uri: Optional[str] = typer.Option( None, "--mlflow-uri", "-m", help="MLflow tracking URI (optional)" ), start_cluster: bool = typer.Option( - True, "--start-cluster/--no-start-cluster", - help="Start minikube cluster if not running" + True, + "--start-cluster/--no-start-cluster", + help="Start minikube cluster if not running", ), ): """ @@ -2338,33 +2706,39 @@ def minikube_init( raise typer.Exit(code=1) else: typer.echo("Minikube is already running") - + typer.echo(f"\nGenerating FastAPI Kubernetes manifests in {output_dir}...") generate_fastapi_manifests( - output_dir=output_dir, - image=image, - mlflow_tracking_uri=mlflow_uri + output_dir=output_dir, image=image, mlflow_tracking_uri=mlflow_uri ) - + typer.echo("\nSetup complete! Next steps:") typer.echo(f" 1. Edit the manifests in {output_dir} if needed") - typer.echo(f" 2. Deploy with: deployml minikube-deploy --manifest-dir {output_dir}") + typer.echo( + f" 2. Deploy with: deployml minikube-deploy --manifest-dir {output_dir}" + ) @cli.command() def minikube_deploy( manifest_dir: Path = typer.Option( - ..., "--manifest-dir", "-d", - help="Directory containing deployment.yaml and service.yaml" + ..., + "--manifest-dir", + "-d", + help="Directory containing deployment.yaml and service.yaml", ), image_name: Optional[str] = typer.Option( - None, "--image-name", "-i", - help="Docker image name to load into minikube (auto-detected from deployment.yaml if not provided)" + None, + "--image-name", + "-i", + help="Docker image name to load into minikube (auto-detected from deployment.yaml if not provided)", ), namespace: Optional[str] = typer.Option( - None, "--namespace", "-n", + None, + "--namespace", + "-n", help="Kubernetes namespace to deploy into. Defaults to the default namespace. " - "Use the same namespace for MLflow and FastAPI so service DNS resolves." + "Use the same namespace for MLflow and FastAPI so service DNS resolves.", ), ): """ @@ -2382,8 +2756,10 @@ def minikube_deploy( typer.echo(" deployml minikube-init --start-cluster") raise typer.Exit(code=1) - success = deploy_fastapi_to_minikube(manifest_dir, image_name=image_name, namespace=namespace) - + success = deploy_fastapi_to_minikube( + manifest_dir, image_name=image_name, namespace=namespace + ) + if not success: raise typer.Exit(code=1) @@ -2393,22 +2769,28 @@ def mlflow_init( output_dir: Path = typer.Option( ..., "--output-dir", "-o", help="Directory to create Kubernetes manifests" ), - image: str = typer.Option( - ..., "--image", "-i", help="MLflow Docker image" - ), + image: str = typer.Option(..., "--image", "-i", help="MLflow Docker image"), backend_store_uri: Optional[str] = typer.Option( - None, "--backend-store-uri", "-b", help="Backend store URI. Default sqlite on the mounted PVC." + None, + "--backend-store-uri", + "-b", + help="Backend store URI. Default sqlite on the mounted PVC.", ), artifact_root: Optional[str] = typer.Option( - None, "--artifact-root", "-a", help="Artifact root path (defaults to /mlflow-artifacts)" + None, + "--artifact-root", + "-a", + help="Artifact root path (defaults to /mlflow-artifacts)", ), start_cluster: bool = typer.Option( - True, "--start-cluster/--no-start-cluster", - help="Start minikube cluster if not running" + True, + "--start-cluster/--no-start-cluster", + help="Start minikube cluster if not running", ), persistent_storage: bool = typer.Option( - True, "--persistent-storage/--ephemeral-storage", - help="Mount a PersistentVolumeClaim so sqlite and artifacts survive pod restarts. Default on." + True, + "--persistent-storage/--ephemeral-storage", + help="Mount a PersistentVolumeClaim so sqlite and artifacts survive pod restarts. Default on.", ), pvc_size: str = typer.Option( "5Gi", "--pvc-size", help="PVC size when --persistent-storage is on." @@ -2428,7 +2810,7 @@ def mlflow_init( raise typer.Exit(code=1) else: typer.echo("Minikube is already running") - + typer.echo(f"\nGenerating MLflow Kubernetes manifests in {output_dir}...") generate_mlflow_manifests( output_dir=output_dir, @@ -2438,7 +2820,7 @@ def mlflow_init( use_pvc=persistent_storage, pvc_size=pvc_size, ) - + typer.echo("\nSetup complete! Next steps:") typer.echo(f" 1. Edit the manifests in {output_dir} if needed") typer.echo(f" 2. Deploy with: deployml mlflow-deploy --manifest-dir {output_dir}") @@ -2447,17 +2829,23 @@ def mlflow_init( @cli.command() def mlflow_deploy( manifest_dir: Path = typer.Option( - ..., "--manifest-dir", "-d", - help="Directory containing deployment.yaml and service.yaml" + ..., + "--manifest-dir", + "-d", + help="Directory containing deployment.yaml and service.yaml", ), image_name: Optional[str] = typer.Option( - None, "--image-name", "-i", - help="Docker image name to load into minikube (auto-detected from deployment.yaml if not provided)" + None, + "--image-name", + "-i", + help="Docker image name to load into minikube (auto-detected from deployment.yaml if not provided)", ), namespace: Optional[str] = typer.Option( - None, "--namespace", "-n", + None, + "--namespace", + "-n", help="Kubernetes namespace to deploy into. Defaults to the default namespace. " - "Use the same namespace for MLflow and FastAPI so service DNS resolves." + "Use the same namespace for MLflow and FastAPI so service DNS resolves.", ), ): """ @@ -2475,8 +2863,10 @@ def mlflow_deploy( typer.echo(" deployml mlflow-init --start-cluster") raise typer.Exit(code=1) - success = deploy_mlflow_to_minikube(manifest_dir, image_name=image_name, namespace=namespace) - + success = deploy_mlflow_to_minikube( + manifest_dir, image_name=image_name, namespace=namespace + ) + if not success: raise typer.Exit(code=1) @@ -2484,25 +2874,23 @@ def mlflow_deploy( @cli.command() def gke_deploy( manifest_dir: Path = typer.Option( - ..., "--manifest-dir", "-d", - help="Directory containing deployment.yaml and service.yaml" - ), - cluster: str = typer.Option( - ..., "--cluster", "-c", help="GKE cluster name" - ), - project: str = typer.Option( - ..., "--project", "-p", help="GCP project ID" - ), - zone: Optional[str] = typer.Option( - None, "--zone", "-z", help="GKE cluster zone" + ..., + "--manifest-dir", + "-d", + help="Directory containing deployment.yaml and service.yaml", ), + cluster: str = typer.Option(..., "--cluster", "-c", help="GKE cluster name"), + project: str = typer.Option(..., "--project", "-p", help="GCP project ID"), + zone: Optional[str] = typer.Option(None, "--zone", "-z", help="GKE cluster zone"), region: Optional[str] = typer.Option( None, "--region", "-r", help="GKE cluster region" ), namespace: Optional[str] = typer.Option( - None, "--namespace", "-n", + None, + "--namespace", + "-n", help="Kubernetes namespace to deploy into. Defaults to the default namespace. " - "Use the same namespace for MLflow and FastAPI so service DNS resolves." + "Use the same namespace for MLflow and FastAPI so service DNS resolves.", ), ): """ @@ -2534,9 +2922,12 @@ def gke_deploy( def gke_cluster_create( cluster: str = typer.Option(..., "--cluster", "-c", help="Cluster name"), project: str = typer.Option(..., "--project", "-p", help="GCP project ID"), - region: str = typer.Option("us-west1", "--region", "-r", help="Region for the cluster"), + region: str = typer.Option( + "us-west1", "--region", "-r", help="Region for the cluster" + ), autopilot: bool = typer.Option( - True, "--autopilot/--standard", + True, + "--autopilot/--standard", help="Use GKE Autopilot (default) or a standard zonal cluster.", ), ): @@ -2546,54 +2937,75 @@ def gke_cluster_create( """ if autopilot: cmd = [ - "gcloud", "container", "clusters", "create-auto", cluster, - "--region", region, "--project", project, + "gcloud", + "container", + "clusters", + "create-auto", + cluster, + "--region", + region, + "--project", + project, ] else: cmd = [ - "gcloud", "container", "clusters", "create", cluster, - "--region", region, "--project", project, - "--num-nodes", "1", "--machine-type", "e2-medium", + "gcloud", + "container", + "clusters", + "create", + cluster, + "--region", + region, + "--project", + project, + "--num-nodes", + "1", + "--machine-type", + "e2-medium", ] - typer.echo(f" Creating {'Autopilot' if autopilot else 'standard'} cluster {cluster}...") + typer.echo( + f" Creating {'Autopilot' if autopilot else 'standard'} cluster {cluster}..." + ) typer.echo(" This typically takes 5 to 10 minutes.") result = run_tool(cmd[0], cmd[1:], capture_output=False, text=True) if result.returncode != 0: raise typer.Exit(code=1) typer.secho(f" Cluster {cluster} created.", fg=typer.colors.GREEN) - typer.echo(f" Next: deployml gke-init --output-dir manifests --image gcr.io/{project}/... --project {project}") + typer.echo( + f" Next: deployml gke-init --output-dir manifests --image gcr.io/{project}/... --project {project}" + ) @cli.command("gke-destroy") def gke_destroy( manifest_dir: Path = typer.Option( - ..., "--manifest-dir", "-d", - help="Directory containing deployment.yaml and service.yaml that were applied" - ), - cluster: str = typer.Option( - ..., "--cluster", "-c", help="GKE cluster name" - ), - project: str = typer.Option( - ..., "--project", "-p", help="GCP project ID" - ), - zone: Optional[str] = typer.Option( - None, "--zone", "-z", help="GKE cluster zone" + ..., + "--manifest-dir", + "-d", + help="Directory containing deployment.yaml and service.yaml that were applied", ), + cluster: str = typer.Option(..., "--cluster", "-c", help="GKE cluster name"), + project: str = typer.Option(..., "--project", "-p", help="GCP project ID"), + zone: Optional[str] = typer.Option(None, "--zone", "-z", help="GKE cluster zone"), region: Optional[str] = typer.Option( None, "--region", "-r", help="GKE cluster region" ), namespace: Optional[str] = typer.Option( - None, "--namespace", "-n", - help="Namespace the manifests were applied to. Defaults to the default namespace." + None, + "--namespace", + "-n", + help="Namespace the manifests were applied to. Defaults to the default namespace.", ), delete_cluster: bool = typer.Option( - False, "--delete-cluster", - help="Also delete the GKE cluster after removing manifests." + False, + "--delete-cluster", + help="Also delete the GKE cluster after removing manifests.", ), keep_images: bool = typer.Option( - False, "--keep-images", + False, + "--keep-images", help="Keep the gcr.io image this workload used. By default it is deleted so " - "teardown is fully self-cleaning, matching the Cloud Run destroy behavior." + "teardown is fully self-cleaning, matching the Cloud Run destroy behavior.", ), ): """ @@ -2613,7 +3025,6 @@ def gke_destroy( raise typer.Exit(code=1) from deployml.utils.kubernetes_gke import ( - connect_to_gke_cluster, get_pvc_volume_handle, disk_ref_from_volume_handle, delete_gce_disk_if_exists, @@ -2647,8 +3058,10 @@ def gke_destroy( if f.exists(): typer.echo(f" Deleting {fname}...") result = run_tool( - "kubectl", ["delete", "-f", str(f), "--ignore-not-found"] + ns, - capture_output=True, text=True, + "kubectl", + ["delete", "-f", str(f), "--ignore-not-found"] + ns, + capture_output=True, + text=True, ) if result.returncode == 0: typer.echo(f" {result.stdout.strip() or 'deleted'}") @@ -2664,21 +3077,40 @@ def gke_destroy( if dep.exists(): try: doc = yaml.safe_load(dep.read_text()) - image = doc["spec"]["template"]["spec"]["containers"][0].get("image", "") + image = doc["spec"]["template"]["spec"]["containers"][0].get( + "image", "" + ) except Exception: image = "" if image.startswith("gcr.io/"): typer.echo(f" Removing image {image}...") run_tool( - "gcloud", ["container", "images", "delete", image, - "--force-delete-tags", "--quiet", "--project", project], + "gcloud", + [ + "container", + "images", + "delete", + image, + "--force-delete-tags", + "--quiet", + "--project", + project, + ], capture_output=True, ) if delete_cluster: typer.echo(f"\n Deleting cluster {cluster}...") - cmd = ["gcloud", "container", "clusters", "delete", cluster, - "--project", project, "--quiet"] + cmd = [ + "gcloud", + "container", + "clusters", + "delete", + cluster, + "--project", + project, + "--quiet", + ] if zone: cmd += ["--zone", zone] else: @@ -2724,17 +3156,20 @@ def gke_init( image: str = typer.Option( ..., "--image", "-i", help="Docker image name (local or GCR)" ), - project: str = typer.Option( - ..., "--project", "-p", help="GCP project ID" - ), + project: str = typer.Option(..., "--project", "-p", help="GCP project ID"), service: str = typer.Option( "mlflow", "--service", "-s", help="Service type: mlflow, fastapi, or all" ), mlflow_uri: Optional[str] = typer.Option( - None, "--mlflow-uri", "-m", help="MLflow URI (for FastAPI). Only used when service is fastapi." + None, + "--mlflow-uri", + "-m", + help="MLflow URI (for FastAPI). Only used when service is fastapi.", ), mlflow_image: Optional[str] = typer.Option( - None, "--mlflow-image", help="MLflow image. Required when --service all. Defaults to --image when not provided." + None, + "--mlflow-image", + help="MLflow image. Required when --service all. Defaults to --image when not provided.", ), ): """ @@ -2751,7 +3186,9 @@ def gke_init( project_id=project, push_image=not image.startswith("gcr.io/"), ) - typer.echo(f"\nNext: deployml gke-deploy -d {output_dir} -c CLUSTER -p {project} -z ZONE") + typer.echo( + f"\nNext: deployml gke-deploy -d {output_dir} -c CLUSTER -p {project} -z ZONE" + ) elif service == "fastapi": generate_fastapi_manifests_gke( output_dir=output_dir, @@ -2760,7 +3197,9 @@ def gke_init( mlflow_tracking_uri=mlflow_uri, push_image=not image.startswith("gcr.io/"), ) - typer.echo(f"\nNext: deployml gke-deploy -d {output_dir} -c CLUSTER -p {project} -z ZONE") + typer.echo( + f"\nNext: deployml gke-deploy -d {output_dir} -c CLUSTER -p {project} -z ZONE" + ) elif service == "all": ml_img = mlflow_image or image ml_dir = output_dir / "mlflow" @@ -2781,8 +3220,12 @@ def gke_init( push_image=not image.startswith("gcr.io/"), ) typer.echo("\nNext steps:") - typer.echo(f" 1. deployml gke-deploy -d {ml_dir} -c CLUSTER -p {project} -r REGION") - typer.echo(f" 2. deployml gke-deploy -d {fa_dir} -c CLUSTER -p {project} -r REGION") + typer.echo( + f" 1. deployml gke-deploy -d {ml_dir} -c CLUSTER -p {project} -r REGION" + ) + typer.echo( + f" 2. deployml gke-deploy -d {fa_dir} -c CLUSTER -p {project} -r REGION" + ) else: typer.echo(f"Unknown service: {service}. Use 'mlflow', 'fastapi', or 'all'") raise typer.Exit(code=1) @@ -2806,22 +3249,26 @@ def gke_apply( raise typer.Exit(code=1) config = _load_config_or_exit(config_path) - + # Validate deployment type deployment_type = config.get("deployment", {}).get("type") if deployment_type != "gke": - typer.echo(f"This command is only for GKE deployments. Found: {deployment_type}") + typer.echo( + f"This command is only for GKE deployments. Found: {deployment_type}" + ) raise typer.Exit(code=1) - + workspace_name = config.get("name") or "default" DEPLOYML_DIR = Path.cwd() / ".deployml" / workspace_name manifests_dir = DEPLOYML_DIR / "manifests" - + if not manifests_dir.exists(): typer.echo(f"Manifests directory not found: {manifests_dir}") - typer.echo(" Generate manifests first with: deployml deploy --config-path --generate-only") + typer.echo( + " Generate manifests first with: deployml deploy --config-path --generate-only" + ) raise typer.Exit(code=1) - + # Extract GKE-specific config project_id = config["provider"]["project_id"] gke_config = config.get("gke", {}) @@ -2834,40 +3281,39 @@ def gke_apply( if not cluster_name: typer.echo("GKE cluster_name must be specified in config.gke.cluster_name") raise typer.Exit(code=1) - + if not zone and not region_gke: typer.echo("Either config.gke.zone or config.gke.region must be specified") raise typer.Exit(code=1) - - typer.echo(f" Applying GKE manifests") + + typer.echo(" Applying GKE manifests") typer.echo(f" Cluster: {cluster_name}") typer.echo(f" Location: {zone or region_gke}") typer.echo(f" Manifests: {manifests_dir}") - + # Import deployment function from deployml.utils.kubernetes_gke import ( deploy_to_gke, - connect_to_gke_cluster, ) - + # Connect to GKE cluster if not connect_to_gke_cluster(project_id, cluster_name, zone, region_gke): raise typer.Exit(code=1) - + # Find and deploy all manifest directories mlflow_manifest_dir = manifests_dir / "mlflow" fastapi_manifest_dir = manifests_dir / "fastapi" - + if not yes: typer.echo("\n About to apply manifests to GKE cluster") if not typer.confirm("Continue?"): typer.echo("Deployment cancelled") return - + deployed_any = False - + if mlflow_manifest_dir.exists(): - typer.echo(f"\n Deploying MLflow to GKE...") + typer.echo("\n Deploying MLflow to GKE...") if deploy_to_gke( manifest_dir=mlflow_manifest_dir, cluster_name=cluster_name, @@ -2881,7 +3327,7 @@ def gke_apply( raise typer.Exit(code=1) if fastapi_manifest_dir.exists(): - typer.echo(f"\n Deploying FastAPI to GKE...") + typer.echo("\n Deploying FastAPI to GKE...") if deploy_to_gke( manifest_dir=fastapi_manifest_dir, cluster_name=cluster_name, @@ -3010,6 +3456,7 @@ def build_images_command( typer.secho(f"Error building images: {e}", fg=typer.colors.RED) raise typer.Exit(code=1) + def main(): """ Entry point for the DeployML CLI. diff --git a/src/deployml/diagnostics/__init__.py b/src/deployml/diagnostics/__init__.py index 3added1..1f16f5b 100644 --- a/src/deployml/diagnostics/__init__.py +++ b/src/deployml/diagnostics/__init__.py @@ -5,10 +5,4 @@ from .doctor import DeployMLDoctor, CheckStatus, CheckResult, run_doctor, check_system -__all__ = [ - 'DeployMLDoctor', - 'CheckStatus', - 'CheckResult', - 'run_doctor', - 'check_system' -] \ No newline at end of file +__all__ = ["DeployMLDoctor", "CheckStatus", "CheckResult", "run_doctor", "check_system"] diff --git a/src/deployml/diagnostics/doctor.py b/src/deployml/diagnostics/doctor.py index e04196c..398bd93 100644 --- a/src/deployml/diagnostics/doctor.py +++ b/src/deployml/diagnostics/doctor.py @@ -1,10 +1,8 @@ import shutil -import os import sys import platform from pathlib import Path -from typing import Dict, List, Optional, Tuple -import json +from typing import Dict, List, Optional import importlib from deployml.utils.platform_compat import run_tool @@ -19,7 +17,6 @@ get_package_version_metadata = None from dataclasses import dataclass from enum import Enum -import re try: import pandas as pd @@ -27,7 +24,8 @@ pd = None try: - from IPython.display import display, HTML + from IPython.display import display + IN_NOTEBOOK = True except ImportError: IN_NOTEBOOK = False @@ -35,6 +33,7 @@ class CheckStatus(Enum): """Status of a system check""" + PASS = "PASS" FAIL = "FAIL" WARNING = "WARNING" @@ -45,6 +44,7 @@ class CheckStatus(Enum): @dataclass class CheckResult: """Result of a system check""" + name: str status: CheckStatus message: str @@ -55,50 +55,50 @@ class CheckResult: class DeployMLDoctor: """System verification and dependency checker for DeployML""" - + def __init__(self, verbose: bool = False): self.verbose = verbose self.results: List[CheckResult] = [] self.system_info = self._gather_system_info() - + def _gather_system_info(self) -> Dict[str, str]: """Gather basic system information""" return { - 'os': platform.system(), - 'os_version': platform.version(), - 'architecture': platform.architecture()[0], - 'python_version': sys.version, - 'python_executable': sys.executable, - 'current_directory': str(Path.cwd()) + "os": platform.system(), + "os_version": platform.version(), + "architecture": platform.architecture()[0], + "python_version": sys.version, + "python_executable": sys.executable, + "current_directory": str(Path.cwd()), } - + def run_all_checks(self) -> List[CheckResult]: """Run all system checks and return results""" self.results.clear() - + # Core Python checks self._check_python_version() self._check_required_packages() self._check_optional_packages() - + # Infrastructure tools self._check_docker() self._check_terraform() self._check_cloud_cli_tools() - + # Development tools self._check_git() self._check_infracost() - + # Permissions and access self._check_docker_permissions() self._check_cloud_authentication() - + # Configuration self._check_deployml_config() - + return self.results - + def _add_result(self, result: CheckResult): """Add a check result""" self.results.append(result) @@ -108,94 +108,112 @@ def _add_result(self, result: CheckResult): CheckStatus.FAIL: "[FAIL]", CheckStatus.WARNING: "[WARN]", CheckStatus.INFO: "[INFO]", - CheckStatus.SKIP: "[SKIP]" + CheckStatus.SKIP: "[SKIP]", }[result.status] print(f"{status_symbol} {result.name}: {result.message}") - + def _check_python_version(self): """Check Python version compatibility""" version = sys.version_info - + if version >= (3, 11): - self._add_result(CheckResult( - name="Python Version", - status=CheckStatus.PASS, - message=f"Python {version.major}.{version.minor}.{version.micro} (>= 3.11)" - )) + self._add_result( + CheckResult( + name="Python Version", + status=CheckStatus.PASS, + message=f"Python {version.major}.{version.minor}.{version.micro} (>= 3.11)", + ) + ) elif version >= (3, 8): - self._add_result(CheckResult( - name="Python Version", - status=CheckStatus.WARNING, - message=f"Python {version.major}.{version.minor}.{version.micro} (Minimum supported, recommend >= 3.11)", - fix_command="Consider upgrading: pyenv install 3.11 && pyenv global 3.11" - )) + self._add_result( + CheckResult( + name="Python Version", + status=CheckStatus.WARNING, + message=f"Python {version.major}.{version.minor}.{version.micro} (Minimum supported, recommend >= 3.11)", + fix_command="Consider upgrading: pyenv install 3.11 && pyenv global 3.11", + ) + ) else: - self._add_result(CheckResult( - name="Python Version", - status=CheckStatus.FAIL, - message=f"Python {version.major}.{version.minor}.{version.micro} (Too old, requires >= 3.8)", - fix_command="Upgrade Python: pyenv install 3.11 && pyenv global 3.11" - )) - + self._add_result( + CheckResult( + name="Python Version", + status=CheckStatus.FAIL, + message=f"Python {version.major}.{version.minor}.{version.micro} (Too old, requires >= 3.8)", + fix_command="Upgrade Python: pyenv install 3.11 && pyenv global 3.11", + ) + ) + def _check_required_packages(self): """Check required Python packages""" required_packages = [ - ('typer', 'typer', 'CLI framework'), - ('pyyaml', 'yaml', 'YAML configuration parsing'), - ('jinja2', 'jinja2', 'Template rendering'), - ('pandas', 'pandas', 'Data manipulation'), - ('requests', 'requests', 'HTTP client'), - ('ipython', 'IPython', 'Interactive Python'), - ('jupyter', 'jupyter', 'Notebook support') + ("typer", "typer", "CLI framework"), + ("pyyaml", "yaml", "YAML configuration parsing"), + ("jinja2", "jinja2", "Template rendering"), + ("pandas", "pandas", "Data manipulation"), + ("requests", "requests", "HTTP client"), + ("ipython", "IPython", "Interactive Python"), + ("jupyter", "jupyter", "Notebook support"), ] - + for pip_name, import_name, description in required_packages: try: importlib.import_module(import_name) version = self._get_package_version(pip_name) - self._add_result(CheckResult( - name=f"Package: {pip_name}", - status=CheckStatus.PASS, - message=f"{description} - v{version}" if version else f"{description} - installed" - )) + self._add_result( + CheckResult( + name=f"Package: {pip_name}", + status=CheckStatus.PASS, + message=f"{description} - v{version}" + if version + else f"{description} - installed", + ) + ) except ImportError: - self._add_result(CheckResult( - name=f"Package: {pip_name}", - status=CheckStatus.FAIL, - message=f"Missing required package: {pip_name} ({description})", - fix_command=f"pip install {pip_name}", - required=True - )) - + self._add_result( + CheckResult( + name=f"Package: {pip_name}", + status=CheckStatus.FAIL, + message=f"Missing required package: {pip_name} ({description})", + fix_command=f"pip install {pip_name}", + required=True, + ) + ) + def _check_optional_packages(self): """Check optional packages that enhance functionality""" optional_packages = [ - ('mlflow', 'ML experiment tracking'), - ('google-cloud-storage', 'GCP storage integration'), - ('scikit-learn', 'Machine learning'), - ('matplotlib', 'Plotting'), - ('seaborn', 'Statistical visualization') + ("mlflow", "ML experiment tracking"), + ("google-cloud-storage", "GCP storage integration"), + ("scikit-learn", "Machine learning"), + ("matplotlib", "Plotting"), + ("seaborn", "Statistical visualization"), ] - + for package, description in optional_packages: try: importlib.import_module(package) version = self._get_package_version(package) - self._add_result(CheckResult( - name=f"Optional: {package}", - status=CheckStatus.PASS, - message=f"{description} - v{version}" if version else f"{description} - installed", - required=False - )) + self._add_result( + CheckResult( + name=f"Optional: {package}", + status=CheckStatus.PASS, + message=f"{description} - v{version}" + if version + else f"{description} - installed", + required=False, + ) + ) except ImportError: - self._add_result(CheckResult( - name=f"Optional: {package}", - status=CheckStatus.INFO, - message=f"Optional package not installed: {package} ({description})", - fix_command=f"pip install {package}", - required=False - )) - + self._add_result( + CheckResult( + name=f"Optional: {package}", + status=CheckStatus.INFO, + message=f"Optional package not installed: {package} ({description})", + fix_command=f"pip install {package}", + required=False, + ) + ) + def _get_package_version(self, package_name: str) -> Optional[str]: """Get version of installed package""" try: @@ -205,298 +223,362 @@ def _get_package_version(self, package_name: str) -> Optional[str]: return None except Exception: return None - + def _check_docker(self): """Check Docker installation and version""" - if not shutil.which('docker'): - self._add_result(CheckResult( - name="Docker", - status=CheckStatus.FAIL, - message="Docker not found in PATH", - fix_command="Install Docker: https://docs.docker.com/get-docker/" - )) + if not shutil.which("docker"): + self._add_result( + CheckResult( + name="Docker", + status=CheckStatus.FAIL, + message="Docker not found in PATH", + fix_command="Install Docker: https://docs.docker.com/get-docker/", + ) + ) return - + try: - result = run_tool('docker', ['--version'], capture_output=True, text=True) + result = run_tool("docker", ["--version"], capture_output=True, text=True) if result.returncode == 0: version = result.stdout.strip() - self._add_result(CheckResult( - name="Docker", - status=CheckStatus.PASS, - message=version - )) + self._add_result( + CheckResult(name="Docker", status=CheckStatus.PASS, message=version) + ) else: - self._add_result(CheckResult( + self._add_result( + CheckResult( + name="Docker", + status=CheckStatus.FAIL, + message="Docker command failed", + details=result.stderr, + ) + ) + except Exception as e: + self._add_result( + CheckResult( name="Docker", status=CheckStatus.FAIL, - message="Docker command failed", - details=result.stderr - )) - except Exception as e: - self._add_result(CheckResult( - name="Docker", - status=CheckStatus.FAIL, - message="Error checking Docker", - details=str(e) - )) - + message="Error checking Docker", + details=str(e), + ) + ) + def _check_terraform(self): """Check Terraform installation""" - if not shutil.which('terraform'): - self._add_result(CheckResult( - name="Terraform", - status=CheckStatus.FAIL, - message="Terraform not found in PATH", - fix_command="Install Terraform: https://developer.hashicorp.com/terraform/install" - )) + if not shutil.which("terraform"): + self._add_result( + CheckResult( + name="Terraform", + status=CheckStatus.FAIL, + message="Terraform not found in PATH", + fix_command="Install Terraform: https://developer.hashicorp.com/terraform/install", + ) + ) return - + try: - result = run_tool('terraform', ['version'], capture_output=True, text=True) + result = run_tool("terraform", ["version"], capture_output=True, text=True) if result.returncode == 0: - version_line = result.stdout.split('\n')[0] - self._add_result(CheckResult( - name="Terraform", - status=CheckStatus.PASS, - message=version_line - )) + version_line = result.stdout.split("\n")[0] + self._add_result( + CheckResult( + name="Terraform", status=CheckStatus.PASS, message=version_line + ) + ) else: - self._add_result(CheckResult( + self._add_result( + CheckResult( + name="Terraform", + status=CheckStatus.FAIL, + message="Terraform command failed", + details=result.stderr, + ) + ) + except Exception as e: + self._add_result( + CheckResult( name="Terraform", status=CheckStatus.FAIL, - message="Terraform command failed", - details=result.stderr - )) - except Exception as e: - self._add_result(CheckResult( - name="Terraform", - status=CheckStatus.FAIL, - message="Error checking Terraform", - details=str(e) - )) - + message="Error checking Terraform", + details=str(e), + ) + ) + def _check_cloud_cli_tools(self): """Check cloud CLI tools""" cloud_tools = [ - ('gcloud', 'Google Cloud CLI', 'https://cloud.google.com/sdk/docs/install'), - ('aws', 'AWS CLI', 'https://aws.amazon.com/cli/'), - ('az', 'Azure CLI', 'https://docs.microsoft.com/en-us/cli/azure/install-azure-cli') + ("gcloud", "Google Cloud CLI", "https://cloud.google.com/sdk/docs/install"), + ("aws", "AWS CLI", "https://aws.amazon.com/cli/"), + ( + "az", + "Azure CLI", + "https://docs.microsoft.com/en-us/cli/azure/install-azure-cli", + ), ] - + for tool, description, install_url in cloud_tools: if shutil.which(tool): try: - if tool == 'gcloud': - result = run_tool('gcloud', ['version'], capture_output=True, text=True) + if tool == "gcloud": + result = run_tool( + "gcloud", ["version"], capture_output=True, text=True + ) if result.returncode == 0: - version = result.stdout.split('\n')[0] - self._add_result(CheckResult( - name=description, - status=CheckStatus.PASS, - message=version, - required=False - )) + version = result.stdout.split("\n")[0] + self._add_result( + CheckResult( + name=description, + status=CheckStatus.PASS, + message=version, + required=False, + ) + ) else: - self._add_result(CheckResult( - name=description, - status=CheckStatus.WARNING, - message="Installed but not properly configured", - required=False - )) + self._add_result( + CheckResult( + name=description, + status=CheckStatus.WARNING, + message="Installed but not properly configured", + required=False, + ) + ) else: # For AWS and Azure CLI - result = run_tool(tool, ['--version'], capture_output=True, text=True) + result = run_tool( + tool, ["--version"], capture_output=True, text=True + ) if result.returncode == 0: version = result.stdout.strip() - self._add_result(CheckResult( - name=description, - status=CheckStatus.PASS, - message=version, - required=False - )) + self._add_result( + CheckResult( + name=description, + status=CheckStatus.PASS, + message=version, + required=False, + ) + ) except Exception: - self._add_result(CheckResult( - name=description, - status=CheckStatus.WARNING, - message="Installed but version check failed", - required=False - )) + self._add_result( + CheckResult( + name=description, + status=CheckStatus.WARNING, + message="Installed but version check failed", + required=False, + ) + ) else: - self._add_result(CheckResult( - name=description, - status=CheckStatus.INFO, - message=f"Not installed (optional for cloud deployments)", - fix_command=f"Install from: {install_url}", - required=False - )) - + self._add_result( + CheckResult( + name=description, + status=CheckStatus.INFO, + message="Not installed (optional for cloud deployments)", + fix_command=f"Install from: {install_url}", + required=False, + ) + ) + def _check_git(self): """Check Git installation""" - if not shutil.which('git'): - self._add_result(CheckResult( - name="Git", - status=CheckStatus.WARNING, - message="Git not found (recommended for version control)", - fix_command="Install Git: https://git-scm.com/downloads", - required=False - )) + if not shutil.which("git"): + self._add_result( + CheckResult( + name="Git", + status=CheckStatus.WARNING, + message="Git not found (recommended for version control)", + fix_command="Install Git: https://git-scm.com/downloads", + required=False, + ) + ) return - + try: - result = run_tool('git', ['--version'], capture_output=True, text=True) + result = run_tool("git", ["--version"], capture_output=True, text=True) if result.returncode == 0: version = result.stdout.strip() - self._add_result(CheckResult( - name="Git", - status=CheckStatus.PASS, - message=version, - required=False - )) + self._add_result( + CheckResult( + name="Git", + status=CheckStatus.PASS, + message=version, + required=False, + ) + ) except Exception: - self._add_result(CheckResult( - name="Git", - status=CheckStatus.WARNING, - message="Git installed but version check failed", - required=False - )) - + self._add_result( + CheckResult( + name="Git", + status=CheckStatus.WARNING, + message="Git installed but version check failed", + required=False, + ) + ) + def _check_infracost(self): """Check Infracost installation for cost analysis""" - if not shutil.which('infracost'): - self._add_result(CheckResult( - name="Infracost", - status=CheckStatus.INFO, - message="Not installed (optional for cost analysis)", - fix_command="Install: brew install infracost/tap/infracost", - required=False - )) + if not shutil.which("infracost"): + self._add_result( + CheckResult( + name="Infracost", + status=CheckStatus.INFO, + message="Not installed (optional for cost analysis)", + fix_command="Install: brew install infracost/tap/infracost", + required=False, + ) + ) return - + try: - result = run_tool('infracost', ['--version'], capture_output=True, text=True) + result = run_tool( + "infracost", ["--version"], capture_output=True, text=True + ) if result.returncode == 0: version = result.stdout.strip() - self._add_result(CheckResult( - name="Infracost", - status=CheckStatus.PASS, - message=version, - required=False - )) + self._add_result( + CheckResult( + name="Infracost", + status=CheckStatus.PASS, + message=version, + required=False, + ) + ) except Exception: - self._add_result(CheckResult( - name="Infracost", - status=CheckStatus.WARNING, - message="Installed but version check failed", - required=False - )) - + self._add_result( + CheckResult( + name="Infracost", + status=CheckStatus.WARNING, + message="Installed but version check failed", + required=False, + ) + ) + def _check_docker_permissions(self): """Check Docker permissions""" - if not shutil.which('docker'): + if not shutil.which("docker"): # _check_docker already reports the missing binary. Skip here rather # than emit a misleading "cannot run docker" permission failure. - self._add_result(CheckResult( - name="Docker Permissions", - status=CheckStatus.SKIP, - message="Docker not installed, skipping permission check", - required=False - )) + self._add_result( + CheckResult( + name="Docker Permissions", + status=CheckStatus.SKIP, + message="Docker not installed, skipping permission check", + required=False, + ) + ) return try: - result = run_tool('docker', ['ps'], capture_output=True, text=True) + result = run_tool("docker", ["ps"], capture_output=True, text=True) if result.returncode == 0: - self._add_result(CheckResult( - name="Docker Permissions", - status=CheckStatus.PASS, - message="Can run Docker commands" - )) + self._add_result( + CheckResult( + name="Docker Permissions", + status=CheckStatus.PASS, + message="Can run Docker commands", + ) + ) else: if "permission denied" in result.stderr.lower(): - self._add_result(CheckResult( - name="Docker Permissions", - status=CheckStatus.FAIL, - message="Permission denied - user not in docker group", - fix_command="sudo usermod -aG docker $USER && newgrp docker" - )) + self._add_result( + CheckResult( + name="Docker Permissions", + status=CheckStatus.FAIL, + message="Permission denied - user not in docker group", + fix_command="sudo usermod -aG docker $USER && newgrp docker", + ) + ) else: - self._add_result(CheckResult( - name="Docker Permissions", - status=CheckStatus.FAIL, - message="Cannot run Docker commands", - details=result.stderr - )) + self._add_result( + CheckResult( + name="Docker Permissions", + status=CheckStatus.FAIL, + message="Cannot run Docker commands", + details=result.stderr, + ) + ) except Exception as e: - self._add_result(CheckResult( - name="Docker Permissions", - status=CheckStatus.FAIL, - message="Error testing Docker permissions", - details=str(e) - )) - + self._add_result( + CheckResult( + name="Docker Permissions", + status=CheckStatus.FAIL, + message="Error testing Docker permissions", + details=str(e), + ) + ) + def _check_cloud_authentication(self): """Check cloud authentication status""" # Check GCP authentication - if shutil.which('gcloud'): + if shutil.which("gcloud"): try: - result = run_tool('gcloud', ['auth', 'list'], capture_output=True, text=True) + result = run_tool( + "gcloud", ["auth", "list"], capture_output=True, text=True + ) if result.returncode == 0 and "ACTIVE" in result.stdout: - self._add_result(CheckResult( - name="GCP Authentication", - status=CheckStatus.PASS, - message="Authenticated with Google Cloud", - required=False - )) + self._add_result( + CheckResult( + name="GCP Authentication", + status=CheckStatus.PASS, + message="Authenticated with Google Cloud", + required=False, + ) + ) else: - self._add_result(CheckResult( + self._add_result( + CheckResult( + name="GCP Authentication", + status=CheckStatus.INFO, + message="Not authenticated (required for GCP deployments)", + fix_command="gcloud auth login", + required=False, + ) + ) + except Exception: + self._add_result( + CheckResult( name="GCP Authentication", status=CheckStatus.INFO, - message="Not authenticated (required for GCP deployments)", - fix_command="gcloud auth login", - required=False - )) - except Exception: - self._add_result(CheckResult( - name="GCP Authentication", - status=CheckStatus.INFO, - message="Cannot check authentication status", - required=False - )) - + message="Cannot check authentication status", + required=False, + ) + ) + def _check_deployml_config(self): """Check DeployML configuration""" config_locations = [ - Path.home() / '.deployml' / 'config.yaml', - Path.cwd() / '.deployml' / 'config.yaml', - Path.cwd() / 'deployml.yaml' + Path.home() / ".deployml" / "config.yaml", + Path.cwd() / ".deployml" / "config.yaml", + Path.cwd() / "deployml.yaml", ] - + config_found = any(path.exists() for path in config_locations) - + if config_found: - self._add_result(CheckResult( - name="DeployML Config", - status=CheckStatus.PASS, - message="Configuration file found", - required=False - )) + self._add_result( + CheckResult( + name="DeployML Config", + status=CheckStatus.PASS, + message="Configuration file found", + required=False, + ) + ) else: - self._add_result(CheckResult( - name="DeployML Config", - status=CheckStatus.INFO, - message="No configuration file found (will use defaults)", - details=f"Looked in: {', '.join(str(p) for p in config_locations)}", - required=False - )) - + self._add_result( + CheckResult( + name="DeployML Config", + status=CheckStatus.INFO, + message="No configuration file found (will use defaults)", + details=f"Looked in: {', '.join(str(p) for p in config_locations)}", + required=False, + ) + ) + def print_results(self, show_all: bool = False): """Print results in a formatted way""" if IN_NOTEBOOK: self._print_notebook_results(show_all) else: self._print_cli_results(show_all) - + def _print_notebook_results(self, show_all: bool): """Print results formatted for Jupyter notebooks""" # System info @@ -506,39 +588,39 @@ def _print_notebook_results(self, show_all: bool): print(f"Python: {self.system_info['python_version'].split()[0]}") print(f"Location: {self.system_info['current_directory']}") print() - + # Create HTML table for better notebook display if pd is not None: df = self.to_dataframe() if not show_all: - df = df[(df['status'] != 'INFO') | (df['required'] == True)] - + df = df[(df["status"] != "INFO") | df["required"]] + # Style the DataFrame def color_status(val): - if val == 'PASS': - return 'color: green; font-weight: bold' - elif val == 'FAIL': - return 'color: red; font-weight: bold' - elif val == 'WARNING': - return 'color: orange; font-weight: bold' + if val == "PASS": + return "color: green; font-weight: bold" + elif val == "FAIL": + return "color: red; font-weight: bold" + elif val == "WARNING": + return "color: orange; font-weight: bold" else: - return 'color: blue' - - styled_df = df[['name', 'status', 'message', 'fix_command']].style.map( - color_status, subset=['status'] + return "color: blue" + + styled_df = df[["name", "status", "message", "fix_command"]].style.map( + color_status, subset=["status"] ) - + print("DEPLOYML DOCTOR RESULTS") print("=" * 80) display(styled_df) else: self._print_simple_table(show_all) - + # Summary summary = self._get_summary_text() print("\n" + "=" * 80) print(summary) - + def _print_cli_results(self, show_all: bool): """Print results for CLI""" print("\nSYSTEM INFORMATION") @@ -546,42 +628,50 @@ def _print_cli_results(self, show_all: bool): print(f"OS: {self.system_info['os']} ({self.system_info['architecture']})") print(f"Python: {self.system_info['python_version'].split()[0]}") print(f"Location: {self.system_info['current_directory']}") - + print("\nDEPLOYML DOCTOR RESULTS") print("=" * 80) - + self._print_simple_table(show_all) - + print("\n" + "=" * 80) summary = self._get_summary_text() print(summary) - + def _print_simple_table(self, show_all: bool): """Print a simple text table""" # Calculate column widths max_name_width = max(len(r.name) for r in self.results) max_status_width = 8 - + # Header print(f"{'STATUS':<{max_status_width}} {'COMPONENT':<{max_name_width}} RESULT") print("-" * (max_status_width + max_name_width + 50)) - + for result in self.results: - if not show_all and result.status == CheckStatus.INFO and not result.required: + if ( + not show_all + and result.status == CheckStatus.INFO + and not result.required + ): continue - + status_text = f"[{result.status.value}]" - print(f"{status_text:<{max_status_width}} {result.name:<{max_name_width}} {result.message}") - + print( + f"{status_text:<{max_status_width}} {result.name:<{max_name_width}} {result.message}" + ) + if result.fix_command: - print(f"{'':<{max_status_width}} {'':<{max_name_width}} Fix: {result.fix_command}") - + print( + f"{'':<{max_status_width}} {'':<{max_name_width}} Fix: {result.fix_command}" + ) + def _get_summary_text(self) -> str: """Get summary text""" summary = self.get_summary() - failed_count = summary['failed'] - warning_count = summary['warnings'] - + failed_count = summary["failed"] + warning_count = summary["warnings"] + if failed_count == 0: if warning_count == 0: return "All checks passed! DeployML is ready to use." @@ -589,43 +679,47 @@ def _get_summary_text(self) -> str: return f"Ready with {warning_count} warnings. Consider addressing them for optimal experience." else: return f"{failed_count} critical issues found. DeployML may not work properly. Run the suggested fix commands above." - + def get_summary(self) -> Dict[str, int]: """Get a summary of check results""" return { - 'total': len(self.results), - 'passed': len([r for r in self.results if r.status == CheckStatus.PASS]), - 'failed': len([r for r in self.results if r.status == CheckStatus.FAIL]), - 'warnings': len([r for r in self.results if r.status == CheckStatus.WARNING]), - 'info': len([r for r in self.results if r.status == CheckStatus.INFO]) + "total": len(self.results), + "passed": len([r for r in self.results if r.status == CheckStatus.PASS]), + "failed": len([r for r in self.results if r.status == CheckStatus.FAIL]), + "warnings": len( + [r for r in self.results if r.status == CheckStatus.WARNING] + ), + "info": len([r for r in self.results if r.status == CheckStatus.INFO]), } - - def to_dataframe(self) -> 'pd.DataFrame': + + def to_dataframe(self) -> "pd.DataFrame": """Convert results to pandas DataFrame""" if pd is None: raise ImportError("pandas not available") - + data = [] for result in self.results: - data.append({ - 'name': result.name, - 'status': result.status.value, - 'message': result.message, - 'required': result.required, - 'fix_command': result.fix_command or '', - 'details': result.details or '' - }) - + data.append( + { + "name": result.name, + "status": result.status.value, + "message": result.message, + "required": result.required, + "fix_command": result.fix_command or "", + "details": result.details or "", + } + ) + return pd.DataFrame(data) def run_doctor(verbose: bool = False, show_all: bool = False) -> DeployMLDoctor: """Run DeployML system verification - + Args: verbose: Show progress as checks run show_all: Show all results including optional info - + Returns: DeployMLDoctor instance with results """ @@ -643,10 +737,10 @@ def check_system() -> DeployMLDoctor: # CLI integration - can be called directly if __name__ == "__main__": import argparse - + parser = argparse.ArgumentParser(description="DeployML System Doctor") parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output") parser.add_argument("-a", "--all", action="store_true", help="Show all results") - + args = parser.parse_args() - run_doctor(verbose=args.verbose, show_all=args.all) \ No newline at end of file + run_doctor(verbose=args.verbose, show_all=args.all) diff --git a/src/deployml/docker/fastapi/main.py b/src/deployml/docker/fastapi/main.py index 2ecea2b..9183977 100644 --- a/src/deployml/docker/fastapi/main.py +++ b/src/deployml/docker/fastapi/main.py @@ -12,7 +12,7 @@ app = FastAPI( title="FastAPI Demo", description="Simple FastAPI application for Kubernetes deployment demo", - version="1.0.0" + version="1.0.0", ) MLFLOW_TRACKING_URI = os.getenv("MLFLOW_TRACKING_URI", "http://mlflow-service:5000") @@ -23,7 +23,15 @@ model = None bq_client = None -FEATURE_ORDER = ['bedrooms', 'bathrooms', 'area_sqft', 'lot_size', 'year_built', 'city', 'state'] +FEATURE_ORDER = [ + "bedrooms", + "bathrooms", + "area_sqft", + "lot_size", + "year_built", + "city", + "state", +] class PredictionRequest(BaseModel): @@ -65,6 +73,7 @@ def init_bigquery(): return try: from google.cloud import bigquery + bq_client = bigquery.Client(project=BIGQUERY_PROJECT) print(f"āœ“ BigQuery client initialized for project {BIGQUERY_PROJECT}") except Exception as e: @@ -72,7 +81,9 @@ def init_bigquery(): bq_client = None -def log_prediction_to_bigquery(entity_id: str, predicted_value: float, model_version: str): +def log_prediction_to_bigquery( + entity_id: str, predicted_value: float, model_version: str +): if bq_client is None: return try: @@ -95,6 +106,7 @@ async def _model_load_retry_loop(): the load once at startup; if MLflow was unavailable then, the model stayed None forever even after MLflow recovered.""" import asyncio + retry_interval = 30 while model is None: ok = await asyncio.to_thread(load_model_from_mlflow) @@ -110,6 +122,7 @@ async def startup_event(): # Load the model in a background task with retry so startup completes fast # AND we keep trying if MLflow was unreachable at first. import asyncio + asyncio.create_task(_model_load_retry_loop()) @@ -121,7 +134,7 @@ async def root(): "model_loaded": model is not None, "model_name": MODEL_NAME, "mlflow_uri": MLFLOW_TRACKING_URI, - "endpoints": {"health": "/health", "predict": "/predict", "docs": "/docs"} + "endpoints": {"health": "/health", "predict": "/predict", "docs": "/docs"}, } @@ -133,6 +146,7 @@ async def health(): mlflow_ok = False try: import requests as _rq + resp = _rq.get(f"{MLFLOW_TRACKING_URI.rstrip('/')}/health", timeout=2) mlflow_ok = resp.status_code == 200 except Exception: @@ -142,7 +156,7 @@ async def health(): timestamp=datetime.now(timezone.utc).isoformat(), port=port, mlflow_connected=mlflow_ok, - model_loaded=model is not None + model_loaded=model is not None, ) @@ -179,7 +193,7 @@ async def predict(request: PredictionRequest): prediction=prediction, timestamp=datetime.now(timezone.utc).isoformat(), model_used=model_version, - entity_id=entity_id + entity_id=entity_id, ) except Exception as e: # Earlier code returned prediction=-1.0 with HTTP 200 and no message, @@ -196,5 +210,6 @@ async def predict(request: PredictionRequest): if __name__ == "__main__": import uvicorn + port = int(os.getenv("PORT", "8000")) uvicorn.run(app, host="0.0.0.0", port=port) diff --git a/src/deployml/notebook/__init__.py b/src/deployml/notebook/__init__.py index f9ce343..1333af2 100644 --- a/src/deployml/notebook/__init__.py +++ b/src/deployml/notebook/__init__.py @@ -6,16 +6,16 @@ Example usage: import deployml.notebook as nb - + # Deploy a new stack stack = nb.deploy('config.yaml') - + # Load existing stack stack = nb.load('my-stack') - + # Display service URLs stack.show_urls() - + # Access MLflow client mlflow_client = stack.mlflow """ @@ -24,9 +24,4 @@ from .stack import DeploymentStack from .urls import ServiceURLs -__all__ = [ - 'deploy', - 'load', - 'DeploymentStack', - 'ServiceURLs' -] \ No newline at end of file +__all__ = ["deploy", "load", "DeploymentStack", "ServiceURLs"] diff --git a/src/deployml/notebook/deployment.py b/src/deployml/notebook/deployment.py index 3dd6457..e3bcc2c 100644 --- a/src/deployml/notebook/deployment.py +++ b/src/deployml/notebook/deployment.py @@ -3,7 +3,6 @@ import shutil import subprocess from pathlib import Path -from typing import Dict, Any from .stack import DeploymentStack @@ -11,16 +10,16 @@ def deploy(config_path: str, show_progress: bool = True) -> DeploymentStack: """ Deploy MLOps stack from YAML configuration - + Args: config_path: Path to YAML configuration file show_progress: Show deployment progress - + Returns: DeploymentStack: Configured stack with service access """ config_file = Path(config_path) - + # If relative path doesn't exist, try common locations if not config_file.exists(): # Try in current directory @@ -35,7 +34,9 @@ def deploy(config_path: str, show_progress: bool = True) -> DeploymentStack: example_path = Path("example/config") / config_file.name if example_path.exists(): config_file = example_path - print(f"ā„¹ļø Found config file in example/config/ directory: {config_file}") + print( + f"ā„¹ļø Found config file in example/config/ directory: {config_file}" + ) else: raise FileNotFoundError( f"Configuration file not found: {config_path}\n" @@ -44,72 +45,78 @@ def deploy(config_path: str, show_progress: bool = True) -> DeploymentStack: f" - {Path('demo') / Path(config_path).name}\n" f" - {Path('example/config') / Path(config_path).name}" ) - + # Load configuration - with open(config_file, 'r') as f: + with open(config_file, "r") as f: config = yaml.safe_load(f) - + # Pretty header - print("\n" + "="*60) + print("\n" + "=" * 60) print("šŸš€ DEPLOYML STACK DEPLOYMENT") - print("="*60) + print("=" * 60) print(f"šŸ“‹ Stack Name: {config.get('name', 'unknown')}") - print(f"ā˜ļø Provider: {config.get('provider', {}).get('name', 'unknown')} ({config.get('provider', {}).get('project_id', 'unknown')})") + print( + f"ā˜ļø Provider: {config.get('provider', {}).get('name', 'unknown')} ({config.get('provider', {}).get('project_id', 'unknown')})" + ) print(f"šŸŒ Region: {config.get('provider', {}).get('region', 'unknown')}") print(f"šŸ“„ Configuration: {config_path}") - + if show_progress: - print(f"\nā³ Initializing deployment...") - + print("\nā³ Initializing deployment...") + # Setup workspace - workspace_name = config.get('name', 'default') + workspace_name = config.get("name", "default") workspace_dir = Path.cwd() / ".deployml" / workspace_name - + # Run deployment using CLI command with logs (use resolved config_file path) _deploy_with_cli(str(config_file), workspace_dir) - + # Create and return deployment stack stack = DeploymentStack(config, workspace_dir) - - print("\n" + "="*60) + + print("\n" + "=" * 60) print("šŸŽ‰ DEPLOYMENT COMPLETED SUCCESSFULLY!") - print("="*60) + print("=" * 60) print("āœ… All MLOps services are deployed and ready to use") print("šŸ“Š Check the URLs below to access your services") - print("="*60) - + print("=" * 60) + return stack def load(stack_name: str) -> DeploymentStack: """ Load an existing deployed stack - + Args: stack_name: Name of the deployed stack - + Returns: DeploymentStack: Loaded stack configuration """ workspace_dir = Path.cwd() / ".deployml" / stack_name - + if not workspace_dir.exists(): - raise FileNotFoundError(f"Stack '{stack_name}' not found. Deploy it first with deployml.deploy()") - + raise FileNotFoundError( + f"Stack '{stack_name}' not found. Deploy it first with deployml.deploy()" + ) + # Try to find the original config file - config_files = list(workspace_dir.glob("*.yaml")) + list(workspace_dir.glob("*.yml")) - + config_files = list(workspace_dir.glob("*.yaml")) + list( + workspace_dir.glob("*.yml") + ) + if not config_files: # Fallback: create minimal config from terraform state config = { - 'name': stack_name, - 'provider': {'name': 'gcp'}, # Default assumption - 'stack': [] + "name": stack_name, + "provider": {"name": "gcp"}, # Default assumption + "stack": [], } else: - with open(config_files[0], 'r') as f: + with open(config_files[0], "r") as f: config = yaml.safe_load(f) - + return DeploymentStack(config, workspace_dir) @@ -117,105 +124,130 @@ def _deploy_with_logs(config_path: str, workspace_dir: Path) -> None: """Run deployment using CLI command and show logs in notebook""" import subprocess import sys - + # Run the actual CLI command that works cmd = [sys.executable, "-m", "deployml.cli.cli", "deploy", "-c", config_path, "-y"] - + print(f"Running: {' '.join(cmd)}") print("=" * 60) - + # Run with live output process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, - bufsize=1 + bufsize=1, ) - + # Stream output line by line - for line in iter(process.stdout.readline, ''): + for line in iter(process.stdout.readline, ""): print(line.rstrip()) sys.stdout.flush() - + process.wait() - + if process.returncode != 0: raise RuntimeError(f"Deployment failed with exit code {process.returncode}") - + + def _format_deployment_line(line: str) -> str: """Format deployment log lines for better readability""" line = line.rstrip() - + # Skip empty lines and some control characters but preserve progress indicators - if not line or (line.startswith('\\x1b') and '[2K' not in line): + if not line or (line.startswith("\\x1b") and "[2K" not in line): return None - + # Clean up terminal control sequences but keep progress content - if '[2K' in line: - line = line.replace('\x1b[2K', '').strip() + if "[2K" in line: + line = line.replace("\x1b[2K", "").strip() if not line: return None - + # Fix literal \n strings that should be actual newlines - if '\\n' in line: - line = line.replace('\\n', '\n') - + if "\\n" in line: + line = line.replace("\\n", "\n") + # Format different types of lines - if line.startswith('šŸ“¦ DeployML Outputs:'): - return f"\n{'='*60}\nšŸŽÆ DEPLOYMENT OUTPUTS\n{'='*60}" - elif line.startswith('šŸ’° COST ANALYSIS'): - return f"\n{'='*60}\nšŸ’° COST ANALYSIS\n{'='*60}" - elif line.startswith('āœ… Deployment complete!'): - return f"\nāœ… INFRASTRUCTURE DEPLOYMENT COMPLETE!" - elif 'DeployML: Creating resources' in line: + if line.startswith("šŸ“¦ DeployML Outputs:"): + return f"\n{'=' * 60}\nšŸŽÆ DEPLOYMENT OUTPUTS\n{'=' * 60}" + elif line.startswith("šŸ’° COST ANALYSIS"): + return f"\n{'=' * 60}\nšŸ’° COST ANALYSIS\n{'=' * 60}" + elif line.startswith("āœ… Deployment complete!"): + return "\nāœ… INFRASTRUCTURE DEPLOYMENT COMPLETE!" + elif "DeployML: Creating resources" in line: # Show progress lines for better user feedback - if any(char in line for char in ['ā ', 'ā ™', 'ā ¹', 'ā ø', 'ā ¼', 'ā “', 'ā ¦', 'ā §', 'ā ‡', 'ā ‹']): + if any( + char in line for char in ["ā ", "ā ™", "ā ¹", "ā ø", "ā ¼", "ā “", "ā ¦", "ā §", "ā ‡", "ā ‹"] + ): # This is a progress line with spinner return f"\r{line}" # Use carriage return to update same line - elif '%' in line: + elif "%" in line: # This is a progress line with percentage return f"\r{line}" # Use carriage return to update same line - elif '100%' in line: + elif "100%" in line: return "\nāœ… Infrastructure deployment completed!" else: return line - elif line.startswith('šŸš€ Deploying'): - return f"\nšŸš€ STARTING DEPLOYMENT\n{'-'*40}\n{line}" - elif line.startswith('šŸ“‹ Initializing Terraform'): - return f"\nšŸ”§ INFRASTRUCTURE SETUP\n{'-'*40}\n{line}" - elif line.startswith('Monthly Cost:') or line.startswith('Hourly Cost:'): + elif line.startswith("šŸš€ Deploying"): + return f"\nšŸš€ STARTING DEPLOYMENT\n{'-' * 40}\n{line}" + elif line.startswith("šŸ“‹ Initializing Terraform"): + return f"\nšŸ”§ INFRASTRUCTURE SETUP\n{'-' * 40}\n{line}" + elif line.startswith("Monthly Cost:") or line.startswith("Hourly Cost:"): return f"šŸ’µ {line}" - elif line.startswith('ā€¢ module.') or line.startswith(' Type:') or line.startswith(' Monthly Cost:'): + elif ( + line.startswith("ā€¢ module.") + or line.startswith(" Type:") + or line.startswith(" Monthly Cost:") + ): return f" {line}" - elif '_url:' in line and 'https://' in line: + elif "_url:" in line and "https://" in line: # Format service URLs nicely - parts = line.split(': ') + parts = line.split(": ") if len(parts) == 2: - service = parts[0].replace('_', ' ').title().replace('Mlflow', 'MLflow').replace('Fastapi', 'FastAPI') + service = ( + parts[0] + .replace("_", " ") + .title() + .replace("Mlflow", "MLflow") + .replace("Fastapi", "FastAPI") + ) url = parts[1] - if '[No value]' not in url: + if "[No value]" not in url: return f"šŸŒ {service}: {url}" return None # Skip [No value] entries - elif line.startswith(' ') and not line.startswith(' '): + elif line.startswith(" ") and not line.startswith(" "): # Indent sub-items return f" {line.strip()}" - elif 'šŸ—ļø Applying changes...' in line: - return f"\nšŸ—ļø DEPLOYING INFRASTRUCTURE\n{'-'*40}\n{line}" - elif 'Estimated time:' in line: + elif "šŸ—ļø Applying changes..." in line: + return f"\nšŸ—ļø DEPLOYING INFRASTRUCTURE\n{'-' * 40}\n{line}" + elif "Estimated time:" in line: return f"ā±ļø {line}" - elif any(char in line for char in ['ā ', 'ā ™', 'ā ¹', 'ā ø', 'ā ¼', 'ā “', 'ā ¦', 'ā §', 'ā ‡', 'ā ‹']) and ('terraform' in line.lower() or 'applying' in line.lower() or 'creating' in line.lower()): + elif any( + char in line for char in ["ā ", "ā ™", "ā ¹", "ā ø", "ā ¼", "ā “", "ā ¦", "ā §", "ā ‡", "ā ‹"] + ) and ( + "terraform" in line.lower() + or "applying" in line.lower() + or "creating" in line.lower() + ): # Progress indicators for terraform operations return f"\r{line}" - elif line.startswith('ā ') or line.startswith('ā ™') or line.startswith('ā ¹') or line.startswith('ā ø'): + elif ( + line.startswith("ā ") + or line.startswith("ā ™") + or line.startswith("ā ¹") + or line.startswith("ā ø") + ): # Direct progress spinner lines return f"\r{line}" - + return line + def _deploy_with_cli(config_path: str, workspace_dir: Path) -> None: """Run deployment using deployml CLI command with formatted logs""" - + # Try direct deployml command first (pip installed) if shutil.which("deployml"): cmd = ["deployml", "deploy", "-c", config_path, "-y"] @@ -226,44 +258,52 @@ def _deploy_with_cli(config_path: str, workspace_dir: Path) -> None: print(f"šŸ”§ Command: {' '.join(cmd)} (development mode)") # Last resort: direct python module else: - cmd = [sys.executable, "-m", "deployml.cli.cli", "deploy", "-c", config_path, "-y"] + cmd = [ + sys.executable, + "-m", + "deployml.cli.cli", + "deploy", + "-c", + config_path, + "-y", + ] print(f"šŸ”§ Command: {' '.join(cmd)} (fallback)") - - print("\n" + "="*60) + + print("\n" + "=" * 60) print("šŸ“ DEPLOYMENT LOG") - print("="*60) - + print("=" * 60) + # Store output lines for error reporting output_lines = [] - + # Run with live output process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, - bufsize=1 + bufsize=1, ) - + # Stream and format output line by line - for line in iter(process.stdout.readline, ''): + for line in iter(process.stdout.readline, ""): output_lines.append(line) formatted_line = _format_deployment_line(line) if formatted_line is not None: print(formatted_line) sys.stdout.flush() - + process.wait() - + if process.returncode != 0: # Force flush any pending output sys.stdout.flush() sys.stderr.flush() - - print(f"\n" + "="*60, flush=True) + + print("\n" + "=" * 60, flush=True) print(f"āŒ DEPLOYMENT FAILED (Exit Code: {process.returncode})", flush=True) - print("="*60, flush=True) - + print("=" * 60, flush=True) + # Show last 50 lines of output to help debug print("\nšŸ“‹ Last 50 lines of output:", flush=True) print("-" * 60, flush=True) @@ -272,9 +312,16 @@ def _deploy_with_cli(config_path: str, workspace_dir: Path) -> None: clean_line = line.rstrip() if clean_line: print(clean_line, flush=True) - + # Try to extract error message - error_lines = [line for line in output_lines if any(keyword in line.lower() for keyword in ['error', 'failed', 'fatal', 'exception', 'traceback'])] + error_lines = [ + line + for line in output_lines + if any( + keyword in line.lower() + for keyword in ["error", "failed", "fatal", "exception", "traceback"] + ) + ] if error_lines: print("\nšŸ” Error Summary:", flush=True) print("-" * 60, flush=True) @@ -290,14 +337,14 @@ def _deploy_with_cli(config_path: str, workspace_dir: Path) -> None: clean_line = line.rstrip() if clean_line: print(clean_line, flush=True) - - print("="*60, flush=True) + + print("=" * 60, flush=True) sys.stdout.flush() sys.stderr.flush() - + raise RuntimeError( f"Deployment failed with exit code {process.returncode}. " f"Check the output above for details. " f"Total output lines: {len(output_lines)}, " - f"Error-related lines: {len([l for l in output_lines if any(k in l.lower() for k in ['error', 'failed', 'fatal'])])}" - ) \ No newline at end of file + f"Error-related lines: {len([line for line in output_lines if any(k in line.lower() for k in ['error', 'failed', 'fatal'])])}" + ) diff --git a/src/deployml/notebook/display.py b/src/deployml/notebook/display.py index 33842a8..7a82cf5 100644 --- a/src/deployml/notebook/display.py +++ b/src/deployml/notebook/display.py @@ -4,7 +4,7 @@ def display_services_table(df: pd.DataFrame): """Create a professional HTML table with clickable links""" - html_content = ''' + html_content = """
@@ -15,18 +15,18 @@ def display_services_table(df: pd.DataFrame): - ''' - + """ + for _, row in df.iterrows(): - status_color = "#28a745" if row['Status'] == 'Ready' else "#dc3545" - status_bg = "#d4edda" if row['Status'] == 'Ready' else "#f8d7da" - - html_content += f''' + status_color = "#28a745" if row["Status"] == "Ready" else "#dc3545" + status_bg = "#d4edda" if row["Status"] == "Ready" else "#f8d7da" + + html_content += f""" - ''' - - if row['URL'].startswith('http'): + """ + + if row["URL"].startswith("http"): html_content += f''' ''' - elif 'PostgreSQL Database' in row['Service'] and row['Status'] == 'Ready': + elif "PostgreSQL Database" in row["Service"] and row["Status"] == "Ready": # Style PostgreSQL connection info differently - html_content += f''' + html_content += f""" - ''' - elif 'Cron Job:' in row['Service'] and row['URL'].startswith('https://console.cloud.google.com'): + """ + elif "Cron Job:" in row["Service"] and row["URL"].startswith( + "https://console.cloud.google.com" + ): # Style cron job links with special GCP console styling html_content += f''' ' - - html_content += f''' + + html_content += f""" - ''' - - html_content += ''' + """ + + html_content += """
{row["Service"]} {row["URL"]} @@ -59,8 +61,8 @@ def display_services_table(df: pd.DataFrame): ''' else: html_content += f'{row["URL"]} @@ -68,18 +70,18 @@ def display_services_table(df: pd.DataFrame):
- ''' - + """ + try: display(HTML(html_content)) - except: + except Exception: # Fallback to simple print if HTML display fails for _, row in df.iterrows(): - status = "[READY]" if row['Status'] == 'Ready' else "[MISSING]" - print(f"{status:10} {row['Service']:35} {row['URL']}") \ No newline at end of file + status = "[READY]" if row["Status"] == "Ready" else "[MISSING]" + print(f"{status:10} {row['Service']:35} {row['URL']}") diff --git a/src/deployml/notebook/docker.py b/src/deployml/notebook/docker.py index 9d02f0f..34d6c91 100644 --- a/src/deployml/notebook/docker.py +++ b/src/deployml/notebook/docker.py @@ -5,9 +5,11 @@ from deployml.utils.helpers import check_docker_daemon from deployml.utils.platform_compat import run_tool + class ImageBuildError(Exception): pass + def build_images( docker_root: Path, gcp_project_id: Optional[str] = None, @@ -59,8 +61,7 @@ def build_images( # Discover services services = [ - d for d in docker_root.iterdir() - if d.is_dir() and (d / "Dockerfile").exists() + d for d in docker_root.iterdir() if d.is_dir() and (d / "Dockerfile").exists() ] if not services: @@ -77,16 +78,21 @@ def build_images( # GCP MODE # ---------------------------------------- if gcp_project_id: - image_base = f"{region}-docker.pkg.dev/{gcp_project_id}/{repository}" # Create Artifact Registry repo if requested if create_repo: create_cmd = [ - "gcloud", "artifacts", "repositories", "create", repository, + "gcloud", + "artifacts", + "repositories", + "create", + repository, "--repository-format=docker", - "--location", region, - "--project", gcp_project_id, + "--location", + region, + "--project", + gcp_project_id, ] if dry_run: @@ -96,13 +102,18 @@ def build_images( else: print("Ensuring Artifact Registry repository exists...") create_proc = run_tool( - create_cmd[0], create_cmd[1:], check=False, - capture_output=True, text=True, + create_cmd[0], + create_cmd[1:], + check=False, + capture_output=True, + text=True, ) stderr_lower = (create_proc.stderr or "").lower() if create_proc.returncode == 0: print(f"Created repository: {repository}") - elif "already exists" in stderr_lower or "alreadyexists" in stderr_lower: + elif ( + "already exists" in stderr_lower or "alreadyexists" in stderr_lower + ): print(f"Repository {repository} already exists, reusing.") else: raise ImageBuildError( @@ -116,10 +127,14 @@ def build_images( image_uri = f"{image_base}/{service_name}:{tag}" build_cmd = [ - "gcloud", "builds", "submit", + "gcloud", + "builds", + "submit", str(service_dir), - "--tag", image_uri, - "--project", gcp_project_id, + "--tag", + image_uri, + "--project", + gcp_project_id, ] if dry_run: @@ -163,14 +178,17 @@ def build_images( if dry_run: print("Dry run complete. No commands were executed.") + # ----------------------------- # Local Docker Build # ----------------------------- + def _validate_docker(): try: run_tool( - "docker", ["--version"], + "docker", + ["--version"], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -206,10 +224,12 @@ def _build_locally(service_dirs: list[Path], tag: str): # GCP Cloud Build # ----------------------------- + def _validate_gcloud(): try: run_tool( - "gcloud", ["--version"], + "gcloud", + ["--version"], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -253,4 +273,4 @@ def _build_with_cloud_build( check=True, ) - print(f"Successfully built and pushed {image_uri}\n") \ No newline at end of file + print(f"Successfully built and pushed {image_uri}\n") diff --git a/src/deployml/notebook/stack.py b/src/deployml/notebook/stack.py index e6967a1..2f35112 100644 --- a/src/deployml/notebook/stack.py +++ b/src/deployml/notebook/stack.py @@ -17,53 +17,58 @@ class DeploymentStack: Main interface for a deployed MLOps stack Provides easy access to services and URLs """ - + def __init__(self, config: Dict[str, Any], workspace_dir: Path): self.config = config self.workspace_dir = workspace_dir - self.name = config.get('name', 'unknown') - self.provider = config.get('provider', {}) + self.name = config.get("name", "unknown") + self.provider = config.get("provider", {}) self._urls = None self._mlflow_client = None - + @property def urls(self) -> ServiceURLs: """Get all service URLs""" if self._urls is None: self._urls = self._extract_urls() return self._urls - + def _extract_urls(self) -> ServiceURLs: """Extract URLs from Terraform outputs""" try: terraform_dir = self.workspace_dir / "terraform" result = run_tool( - "terraform", ["output", "-json"], + "terraform", + ["output", "-json"], cwd=terraform_dir, capture_output=True, text=True, - check=True + check=True, ) - + outputs = json.loads(result.stdout) urls = ServiceURLs() - + # Extract URLs from Terraform outputs for key, value in outputs.items(): - output_val = value.get('value', '') - if 'mlflow' in key.lower() and output_val.startswith('http'): + output_val = value.get("value", "") + if "mlflow" in key.lower() and output_val.startswith("http"): urls.mlflow = output_val - elif 'feast' in key.lower() and output_val.startswith('http'): + elif "feast" in key.lower() and output_val.startswith("http"): urls.feast = output_val - elif 'fastapi' in key.lower() or 'serving' in key.lower() and output_val.startswith('http'): + elif ( + "fastapi" in key.lower() + or "serving" in key.lower() + and output_val.startswith("http") + ): urls.serving = output_val - elif 'grafana' in key.lower() and output_val.startswith('http'): + elif "grafana" in key.lower() and output_val.startswith("http"): urls.grafana = output_val - elif 'instance_connection_name' in key.lower(): + elif "instance_connection_name" in key.lower(): # Format PostgreSQL connection info for display - if output_val and ':' in output_val: + if output_val and ":" in output_val: # Parse the instance connection name: project:region:instance - parts = output_val.split(':') + parts = output_val.split(":") if len(parts) == 3: project, region, instance = parts urls.postgresql = f"Instance: {instance} (Project: {project}, Region: {region})" @@ -71,22 +76,24 @@ def _extract_urls(self) -> ServiceURLs: urls.postgresql = f"Connection: {output_val}" elif output_val: urls.postgresql = str(output_val) - elif 'workflow_orchestration_cron_jobs_summary' in key.lower(): + elif "workflow_orchestration_cron_jobs_summary" in key.lower(): # Extract cron job URLs from the summary if isinstance(output_val, dict): for job_index, job_info in output_val.items(): - if isinstance(job_info, dict) and 'job_url' in job_info: - job_name = job_info.get('service_name', f'job-{job_index}') - urls.cron_jobs[job_name] = job_info['job_url'] - + if isinstance(job_info, dict) and "job_url" in job_info: + job_name = job_info.get( + "service_name", f"job-{job_index}" + ) + urls.cron_jobs[job_name] = job_info["job_url"] + return urls - + except Exception as e: print(f"Warning: Could not extract URLs: {e}") return ServiceURLs() - - @property - def mlflow(self) -> 'MlflowClient': + + @property + def mlflow(self) -> "MlflowClient": """Get pre-configured MLflow client (lazy import)""" if self._mlflow_client is None: try: @@ -103,183 +110,204 @@ def mlflow(self) -> 'MlflowClient': else: raise RuntimeError("MLflow URL not available. Check deployment status.") return self._mlflow_client - + def get_urls_dataframe(self) -> pd.DataFrame: """Get service URLs as a pandas DataFrame""" return self.urls.to_dataframe() - + def show_urls(self) -> pd.DataFrame: """Display service URLs as professional DataFrame with clickable links""" df = self.get_urls_dataframe() - - print("\n" + "="*80) + + print("\n" + "=" * 80) print("DEPLOYED MLOPS SERVICES") - print("="*80) - + print("=" * 80) + # Create professional HTML table with clickable links display_services_table(df) - - print("="*80) + + print("=" * 80) return df - + def get_postgresql_info(self, show_credentials: bool = False) -> Dict[str, str]: """Get PostgreSQL connection information for development use - + Args: show_credentials: If True, attempts to retrieve sensitive credentials """ try: terraform_dir = self.workspace_dir / "terraform" result = run_tool( - "terraform", ["output", "-json"], + "terraform", + ["output", "-json"], cwd=terraform_dir, capture_output=True, text=True, - check=True + check=True, ) - + outputs = json.loads(result.stdout) - + postgresql_info = {} for key, value in outputs.items(): - output_val = value.get('value', '') - if 'instance_connection_name' in key.lower(): - postgresql_info['connection_name'] = output_val - elif 'postgresql_credentials' in key.lower() or 'db_password' in key.lower(): + output_val = value.get("value", "") + if "instance_connection_name" in key.lower(): + postgresql_info["connection_name"] = output_val + elif ( + "postgresql_credentials" in key.lower() + or "db_password" in key.lower() + ): if show_credentials: # Try to get the actual sensitive value try: sensitive_result = run_tool( - "terraform", ["output", "-raw", key], + "terraform", + ["output", "-raw", key], cwd=terraform_dir, capture_output=True, text=True, - check=True + check=True, + ) + postgresql_info["password"] = ( + sensitive_result.stdout.strip() + ) + except Exception: + postgresql_info["password"] = ( + "[SENSITIVE - Run show_postgresql_credentials()]" ) - postgresql_info['password'] = sensitive_result.stdout.strip() - except: - postgresql_info['password'] = "[SENSITIVE - Run show_postgresql_credentials()]" else: - postgresql_info['credentials'] = "[SENSITIVE - Use show_credentials=True]" - elif 'db_user' in key.lower() or 'postgresql_user' in key.lower(): - postgresql_info['username'] = output_val - elif 'db_name' in key.lower() or 'database_name' in key.lower(): - postgresql_info['database'] = output_val - elif 'public_ip' in key.lower() and 'postgresql' in key.lower(): - postgresql_info['public_ip'] = output_val - - if 'connection_name' in postgresql_info: - parts = postgresql_info['connection_name'].split(':') + postgresql_info["credentials"] = ( + "[SENSITIVE - Use show_credentials=True]" + ) + elif "db_user" in key.lower() or "postgresql_user" in key.lower(): + postgresql_info["username"] = output_val + elif "db_name" in key.lower() or "database_name" in key.lower(): + postgresql_info["database"] = output_val + elif "public_ip" in key.lower() and "postgresql" in key.lower(): + postgresql_info["public_ip"] = output_val + + if "connection_name" in postgresql_info: + parts = postgresql_info["connection_name"].split(":") if len(parts) == 3: - postgresql_info['project'] = parts[0] - postgresql_info['region'] = parts[1] - postgresql_info['instance'] = parts[2] - + postgresql_info["project"] = parts[0] + postgresql_info["region"] = parts[1] + postgresql_info["instance"] = parts[2] + # Generate connection strings with actual credentials if available - username = postgresql_info.get('username', 'postgres') - password = postgresql_info.get('password', 'PASSWORD') - database = postgresql_info.get('database', 'postgres') - - postgresql_info['cloud_sql_proxy'] = f"cloud_sql_proxy -instances={postgresql_info['connection_name']}=tcp:5432" - postgresql_info['connection_proxy'] = f"postgresql://{username}:{password}@127.0.0.1:5432/{database}" - - if 'public_ip' in postgresql_info: - postgresql_info['connection_direct'] = f"postgresql://{username}:{password}@{postgresql_info['public_ip']}:5432/{database}" - + username = postgresql_info.get("username", "postgres") + password = postgresql_info.get("password", "PASSWORD") + database = postgresql_info.get("database", "postgres") + + postgresql_info["cloud_sql_proxy"] = ( + f"cloud_sql_proxy -instances={postgresql_info['connection_name']}=tcp:5432" + ) + postgresql_info["connection_proxy"] = ( + f"postgresql://{username}:{password}@127.0.0.1:5432/{database}" + ) + + if "public_ip" in postgresql_info: + postgresql_info["connection_direct"] = ( + f"postgresql://{username}:{password}@{postgresql_info['public_ip']}:5432/{database}" + ) + return postgresql_info - + except Exception as e: return {"error": f"Could not extract PostgreSQL info: {e}"} - + def show_postgresql_connection(self, show_credentials: bool = False): """Display detailed PostgreSQL connection information - + Args: show_credentials: If True, displays actual passwords and connection strings """ info = self.get_postgresql_info(show_credentials=show_credentials) - - if 'error' in info: + + if "error" in info: print(f"Error: {info['error']}") return - - print("\\n" + "="*80) + + print("\\n" + "=" * 80) print("POSTGRESQL CONNECTION INFORMATION") - print("="*80) - - if 'connection_name' in info: + print("=" * 80) + + if "connection_name" in info: print(f"Instance Connection Name: {info['connection_name']}") print(f"Project: {info.get('project', 'N/A')}") print(f"Region: {info.get('region', 'N/A')}") print(f"Instance: {info.get('instance', 'N/A')}") - + if show_credentials: print() print("Credentials:") print(f" Username: {info.get('username', 'postgres')}") print(f" Password: {info.get('password', '[NOT FOUND]')}") print(f" Database: {info.get('database', 'postgres')}") - - if 'public_ip' in info: + + if "public_ip" in info: print(f" Public IP: {info['public_ip']}") - + print() print("Connection Options:") print("1. Using Cloud SQL Proxy:") print(f" {info.get('cloud_sql_proxy', 'N/A')}") - - if show_credentials and 'connection_proxy' in info: + + if show_credentials and "connection_proxy" in info: print(f" Connection String: {info['connection_proxy']}") else: - print(" Then connect to: postgresql://username:password@127.0.0.1:5432/database") - + print( + " Then connect to: postgresql://username:password@127.0.0.1:5432/database" + ) + print() print("2. Direct connection (if public IP enabled):") - if show_credentials and 'connection_direct' in info: + if show_credentials and "connection_direct" in info: print(f" Connection String: {info['connection_direct']}") else: print(" postgresql://username:password@PUBLIC_IP:5432/database") - + if not show_credentials: print() - print("šŸ’” To see actual credentials, use: stack.show_postgresql_connection(show_credentials=True)") + print( + "šŸ’” To see actual credentials, use: stack.show_postgresql_connection(show_credentials=True)" + ) print(" Or use: stack.show_postgresql_credentials()") - + else: print("No PostgreSQL instance found in deployment") - - print("="*80) - + + print("=" * 80) + def show_postgresql_credentials(self): """Display PostgreSQL credentials for easy copy-paste into pgAdmin""" info = self.get_postgresql_info(show_credentials=True) - - if 'error' in info: + + if "error" in info: print(f"Error: {info['error']}") return - - if 'connection_name' not in info: + + if "connection_name" not in info: print("No PostgreSQL instance found in deployment") return - - print("\\n" + "="*80) + + print("\\n" + "=" * 80) print("šŸ” POSTGRESQL CREDENTIALS FOR PGADMIN") - print("="*80) - - username = info.get('username', 'postgres') - password = info.get('password', '[NOT FOUND]') - database = info.get('database', 'postgres') + print("=" * 80) + + username = info.get("username", "postgres") + password = info.get("password", "[NOT FOUND]") + database = info.get("database", "postgres") host_proxy = "127.0.0.1" port = "5432" - + print("For pgAdmin connection setup:") print(f" Host: {host_proxy} (after running Cloud SQL Proxy)") print(f" Port: {port}") print(f" Username: {username}") print(f" Password: {password}") print(f" Database: {database}") - - if 'public_ip' in info: + + if "public_ip" in info: print() print("Direct connection (if public IP enabled):") print(f" Host: {info['public_ip']}") @@ -287,120 +315,126 @@ def show_postgresql_credentials(self): print(f" Username: {username}") print(f" Password: {password}") print(f" Database: {database}") - + print() print("Connection Strings:") - if 'connection_proxy' in info: + if "connection_proxy" in info: print(f" Proxy: {info['connection_proxy']}") - if 'connection_direct' in info: + if "connection_direct" in info: print(f" Direct: {info['connection_direct']}") - + print() print("šŸš€ Steps to connect:") print("1. Start Cloud SQL Proxy:") print(f" {info.get('cloud_sql_proxy', 'N/A')}") print("2. Open pgAdmin and create new server connection") print("3. Use the credentials above") - - print("="*80) - + + print("=" * 80) + def get_postgresql_password(self) -> str: """Get just the PostgreSQL password for programmatic use""" info = self.get_postgresql_info(show_credentials=True) - return info.get('password', '[NOT FOUND]') - + return info.get("password", "[NOT FOUND]") + def get_postgresql_connection_string(self, use_proxy: bool = True) -> str: """Get PostgreSQL connection string - + Args: use_proxy: If True, returns proxy connection (127.0.0.1), else direct IP """ info = self.get_postgresql_info(show_credentials=True) - + if use_proxy: - return info.get('connection_proxy', '[CONNECTION STRING NOT AVAILABLE]') + return info.get("connection_proxy", "[CONNECTION STRING NOT AVAILABLE]") else: - return info.get('connection_direct', '[DIRECT CONNECTION NOT AVAILABLE]') - + return info.get("connection_direct", "[DIRECT CONNECTION NOT AVAILABLE]") + def get_cron_jobs_info(self) -> Dict[str, Any]: """Get detailed cron job information""" try: terraform_dir = self.workspace_dir / "terraform" result = run_tool( - "terraform", ["output", "-json"], + "terraform", + ["output", "-json"], cwd=terraform_dir, capture_output=True, text=True, - check=True + check=True, ) - + outputs = json.loads(result.stdout) - + cron_info = {} for key, value in outputs.items(): - if 'workflow_orchestration_cron_jobs_summary' in key.lower(): - cron_info['jobs_summary'] = value.get('value', {}) - elif 'workflow_orchestration_cron_job_names' in key.lower(): - cron_info['job_names'] = value.get('value', []) - elif 'workflow_orchestration_cron_scheduler_jobs' in key.lower(): - cron_info['scheduler_jobs'] = value.get('value', []) - + if "workflow_orchestration_cron_jobs_summary" in key.lower(): + cron_info["jobs_summary"] = value.get("value", {}) + elif "workflow_orchestration_cron_job_names" in key.lower(): + cron_info["job_names"] = value.get("value", []) + elif "workflow_orchestration_cron_scheduler_jobs" in key.lower(): + cron_info["scheduler_jobs"] = value.get("value", []) + return cron_info - + except Exception as e: return {"error": f"Could not extract cron job info: {e}"} - + def show_cron_jobs(self): """Display detailed cron job information""" info = self.get_cron_jobs_info() - - if 'error' in info: + + if "error" in info: print(f"Error: {info['error']}") return - - print("\n" + "="*80) + + print("\n" + "=" * 80) print("WORKFLOW ORCHESTRATION - CRON JOBS") - print("="*80) - - jobs_summary = info.get('jobs_summary', {}) + print("=" * 80) + + jobs_summary = info.get("jobs_summary", {}) if jobs_summary: for job_index, job_info in jobs_summary.items(): if isinstance(job_info, dict): - print(f"\nJob {int(job_index) + 1}: {job_info.get('service_name', 'Unknown')}") + print( + f"\nJob {int(job_index) + 1}: {job_info.get('service_name', 'Unknown')}" + ) print(f" Schedule: {job_info.get('cron_schedule', 'N/A')}") print(f" Image: {job_info.get('image', 'N/A')}") - if job_info.get('bigquery_dataset'): + if job_info.get("bigquery_dataset"): print(f" BigQuery Dataset: {job_info.get('bigquery_dataset')}") print(f" Console URL: {job_info.get('job_url', 'N/A')}") - if job_info.get('scheduler_name'): + if job_info.get("scheduler_name"): print(f" Scheduler: {job_info.get('scheduler_name')}") else: print("No cron jobs found in deployment") - - print("="*80) - + + print("=" * 80) + def show_status(self) -> None: """Show deployment status in professional format""" - print("\n" + "="*80) + print("\n" + "=" * 80) print("DEPLOYMENT STATUS") - print("="*80) - + print("=" * 80) + status_data = [ ["Stack Name", self.name], - ["Cloud Provider", f"{self.provider.get('name', 'unknown')} ({self.provider.get('project_id', 'unknown')})"], - ["Region", self.provider.get('region', 'unknown')], - ["Workspace", str(self.workspace_dir)] + [ + "Cloud Provider", + f"{self.provider.get('name', 'unknown')} ({self.provider.get('project_id', 'unknown')})", + ], + ["Region", self.provider.get("region", "unknown")], + ["Workspace", str(self.workspace_dir)], ] - + for label, value in status_data: print(f"{label:20}: {value}") - - print("="*80) - + + print("=" * 80) + def get_teardown_status(self) -> Dict[str, Any]: """ Get teardown status for this deployment stack. - + Returns: Dictionary with status information including: - exists: bool - Whether the scheduler job exists @@ -412,36 +446,34 @@ def get_teardown_status(self) -> Dict[str, Any]: - metadata: Optional[Dict] - Local metadata if available """ from deployml.api import get_teardown_status as _get_teardown_status - - project_id = self.provider.get('project_id') - region = self.provider.get('region') + + project_id = self.provider.get("project_id") + region = self.provider.get("region") workspace_name = self.name - + if not project_id or not region: return { "exists": False, - "error": "Missing project_id or region in configuration" + "error": "Missing project_id or region in configuration", } - + return _get_teardown_status( project_id=project_id, region=region, workspace_name=workspace_name, - deployml_dir=self.workspace_dir + deployml_dir=self.workspace_dir, ) - + def update_teardown_schedule( - self, - duration_hours: int, - time_zone: str = "UTC" + self, duration_hours: int, time_zone: str = "UTC" ) -> Dict[str, Any]: """ Update teardown schedule for this deployment stack. - + Args: duration_hours: Hours until teardown time_zone: Timezone (default: UTC) - + Returns: Dictionary with update result: - success: bool - Whether update succeeded @@ -450,81 +482,89 @@ def update_teardown_schedule( - error: Optional[str] - Error message if failed """ from deployml.api import update_teardown_schedule as _update_teardown_schedule - - project_id = self.provider.get('project_id') - region = self.provider.get('region') + + project_id = self.provider.get("project_id") + region = self.provider.get("region") workspace_name = self.name - + if not project_id or not region: return { "success": False, - "error": "Missing project_id or region in configuration" + "error": "Missing project_id or region in configuration", } - + return _update_teardown_schedule( project_id=project_id, region=region, workspace_name=workspace_name, duration_hours=duration_hours, deployml_dir=self.workspace_dir, - time_zone=time_zone + time_zone=time_zone, ) - + def cancel_teardown(self) -> Dict[str, Any]: """ Cancel scheduled teardown for this deployment stack. - + Returns: Dictionary with cancellation result: - success: bool - Whether cancellation succeeded - error: Optional[str] - Error message if failed """ from deployml.api import cancel_teardown as _cancel_teardown - - project_id = self.provider.get('project_id') - region = self.provider.get('region') + + project_id = self.provider.get("project_id") + region = self.provider.get("region") workspace_name = self.name - + if not project_id or not region: return { "success": False, - "error": "Missing project_id or region in configuration" + "error": "Missing project_id or region in configuration", } - + return _cancel_teardown( project_id=project_id, region=region, workspace_name=workspace_name, - deployml_dir=self.workspace_dir + deployml_dir=self.workspace_dir, ) - + def show_teardown_status(self): """Display teardown status in a formatted way""" status = self.get_teardown_status() - - print("\n" + "="*80) + + print("\n" + "=" * 80) print("AUTO-TEARDOWN STATUS") - print("="*80) - + print("=" * 80) + if status.get("error"): print(f"āŒ Error: {status['error']}") - print("="*80) + print("=" * 80) return - + if not status.get("exists"): print("ā„¹ļø Auto-teardown is not scheduled for this workspace") - print("="*80) + print("=" * 80) return - - state_emoji = "āœ…" if status.get("state") == "ENABLED" else "āøļø" if status.get("state") == "PAUSED" else "āŒ" + + state_emoji = ( + "āœ…" + if status.get("state") == "ENABLED" + else "āøļø" + if status.get("state") == "PAUSED" + else "āŒ" + ) print(f"{state_emoji} Status: {status.get('state', 'UNKNOWN')}") print(f"šŸ“… Cron Schedule: {status.get('schedule', 'N/A')}") print(f"šŸŒ Timezone: {status.get('time_zone', 'UTC')}") - + schedule_time = status.get("schedule_time") if schedule_time: - print(f"ā° Next Execution: {schedule_time.strftime('%Y-%m-%d %H:%M:%S UTC')}") - + print( + f"ā° Next Execution: {schedule_time.strftime('%Y-%m-%d %H:%M:%S UTC')}" + ) + now = datetime.now(timezone.utc) if now < schedule_time: time_remaining = schedule_time - now @@ -535,17 +575,21 @@ def show_teardown_status(self): time_passed = now - schedule_time hours_passed = int(time_passed.total_seconds() // 3600) minutes_passed = int((time_passed.total_seconds() % 3600) // 60) - print(f" āš ļø Scheduled time passed {hours_passed}h {minutes_passed}m ago") - + print( + f" āš ļø Scheduled time passed {hours_passed}h {minutes_passed}m ago" + ) + last_attempt = status.get("last_attempt_time") if last_attempt: - print(f"šŸ• Last Execution: {last_attempt.strftime('%Y-%m-%d %H:%M:%S UTC')}") + print( + f"šŸ• Last Execution: {last_attempt.strftime('%Y-%m-%d %H:%M:%S UTC')}" + ) else: print("šŸ• Last Execution: Never") - + print(f"\nšŸ“Œ Job Name: {status.get('scheduler_job_name')}") print("\nšŸ’” Actions:") - print(f" Update: stack.update_teardown_schedule(duration_hours=6)") - print(f" Cancel: stack.cancel_teardown()") - - print("="*80) \ No newline at end of file + print(" Update: stack.update_teardown_schedule(duration_hours=6)") + print(" Cancel: stack.cancel_teardown()") + + print("=" * 80) diff --git a/src/deployml/notebook/urls.py b/src/deployml/notebook/urls.py index a5fedea..d3d4e36 100644 --- a/src/deployml/notebook/urls.py +++ b/src/deployml/notebook/urls.py @@ -4,6 +4,7 @@ class ServiceURLs: """Container for all service URLs""" + def __init__(self): self.mlflow: Optional[str] = None self.feast: Optional[str] = None @@ -11,48 +12,54 @@ def __init__(self): self.grafana: Optional[str] = None self.postgresql: Optional[str] = None self.cron_jobs: Dict[str, str] = {} # job_name -> job_url - + def to_dict(self) -> Dict[str, str]: """Convert to dictionary for easy iteration""" base_services = { - 'mlflow': self.mlflow, - 'feast': self.feast, - 'serving': self.serving, - 'grafana': self.grafana, - 'postgresql': self.postgresql + "mlflow": self.mlflow, + "feast": self.feast, + "serving": self.serving, + "grafana": self.grafana, + "postgresql": self.postgresql, } # Add cron jobs with prefixed keys for job_name, job_url in self.cron_jobs.items(): - base_services[f'cron_{job_name}'] = job_url + base_services[f"cron_{job_name}"] = job_url return base_services - + def to_dataframe(self) -> pd.DataFrame: """Convert to pandas DataFrame for notebook display""" data = [] service_names = { - 'mlflow': 'MLflow Experiment Tracking', - 'feast': 'Feast Feature Store', - 'serving': 'Model Serving API', - 'grafana': 'Grafana Monitoring Dashboard', - 'postgresql': 'PostgreSQL Database' + "mlflow": "MLflow Experiment Tracking", + "feast": "Feast Feature Store", + "serving": "Model Serving API", + "grafana": "Grafana Monitoring Dashboard", + "postgresql": "PostgreSQL Database", } - + # Add cron job service names for job_name in self.cron_jobs.keys(): - service_names[f'cron_{job_name}'] = f'Cron Job: {job_name.replace("-", " ").title()}' - + service_names[f"cron_{job_name}"] = ( + f"Cron Job: {job_name.replace('-', ' ').title()}" + ) + for key, url in self.to_dict().items(): if url: - data.append({ - 'Service': service_names.get(key, key), - 'URL': url, - 'Status': 'Ready' - }) + data.append( + { + "Service": service_names.get(key, key), + "URL": url, + "Status": "Ready", + } + ) else: - data.append({ - 'Service': service_names.get(key, key), - 'URL': 'Not deployed', - 'Status': 'Missing' - }) - - return pd.DataFrame(data) \ No newline at end of file + data.append( + { + "Service": service_names.get(key, key), + "URL": "Not deployed", + "Status": "Missing", + } + ) + + return pd.DataFrame(data) diff --git a/src/deployml/terraform/modules/teardown/cloud/gcp/cloud_function/main.py b/src/deployml/terraform/modules/teardown/cloud/gcp/cloud_function/main.py index 87aad00..025dee1 100644 --- a/src/deployml/terraform/modules/teardown/cloud/gcp/cloud_function/main.py +++ b/src/deployml/terraform/modules/teardown/cloud/gcp/cloud_function/main.py @@ -5,11 +5,9 @@ This function uses the DeployML CLI approach: it calls terraform destroy on the workspace directory stored in GCS, or uses terraform remote state. """ -import os -import json + import subprocess import tempfile -import shutil from pathlib import Path from google.cloud import storage as gcs import logging @@ -31,116 +29,124 @@ def teardown_infrastructure(request): # Parse request request_json = request.get_json(silent=True) if not request_json: - return {'error': 'No JSON payload provided'}, 400 - - workspace_name = request_json.get('workspace_name') - project_id = request_json.get('project_id') - terraform_files_bucket = request_json.get('terraform_files_bucket', '') - terraform_state_bucket = request_json.get('terraform_state_bucket', '') - + return {"error": "No JSON payload provided"}, 400 + + workspace_name = request_json.get("workspace_name") + project_id = request_json.get("project_id") + terraform_files_bucket = request_json.get("terraform_files_bucket", "") + terraform_state_bucket = request_json.get("terraform_state_bucket", "") + if not workspace_name or not project_id: - return {'error': 'Missing required fields: workspace_name, project_id'}, 400 - + return {"error": "Missing required fields: workspace_name, project_id"}, 400 + logger.info(f"Starting teardown for workspace: {workspace_name}") - + # Create temporary directory for terraform with tempfile.TemporaryDirectory() as tmpdir: terraform_dir = Path(tmpdir) / "terraform" terraform_dir.mkdir(parents=True, exist_ok=True) - + # Download terraform files from GCS if provided if terraform_files_bucket: - download_terraform_files(terraform_files_bucket, workspace_name, terraform_dir) - + download_terraform_files( + terraform_files_bucket, workspace_name, terraform_dir + ) + # Download terraform state if stored in GCS if terraform_state_bucket: - download_terraform_state(terraform_state_bucket, workspace_name, terraform_dir) - + download_terraform_state( + terraform_state_bucket, workspace_name, terraform_dir + ) + # Install terraform if not available (Cloud Functions don't have it by default) # We'll use a workaround: call terraform via a container or use gcloud commands # For now, we'll assume terraform is available or use alternative method - + # Set GCP project try: subprocess.run( - ['gcloud', 'config', 'set', 'project', project_id], + ["gcloud", "config", "set", "project", project_id], cwd=terraform_dir, check=True, capture_output=True, - timeout=30 + timeout=30, ) except Exception as e: logger.warning(f"Could not set gcloud project: {e}") - + # Try to initialize terraform (if files exist) if (terraform_dir / "main.tf").exists(): # Initialize terraform init_result = subprocess.run( - ['terraform', 'init'], + ["terraform", "init"], cwd=terraform_dir, check=False, capture_output=True, text=True, - timeout=120 + timeout=120, ) - + if init_result.returncode != 0: logger.warning(f"Terraform init failed: {init_result.stderr}") # Try alternative: use terraform with remote state only - return destroy_via_remote_state(project_id, workspace_name, terraform_state_bucket) - + return destroy_via_remote_state( + project_id, workspace_name, terraform_state_bucket + ) + # Run terraform destroy destroy_result = subprocess.run( - ['terraform', 'destroy', '-auto-approve'], + ["terraform", "destroy", "-auto-approve"], cwd=terraform_dir, capture_output=True, text=True, - timeout=600 # 10 minutes timeout + timeout=600, # 10 minutes timeout ) - + if destroy_result.returncode == 0: - logger.info(f"Successfully destroyed infrastructure for {workspace_name}") + logger.info( + f"Successfully destroyed infrastructure for {workspace_name}" + ) return { - 'status': 'success', - 'message': f'Infrastructure destroyed for workspace: {workspace_name}', - 'workspace': workspace_name + "status": "success", + "message": f"Infrastructure destroyed for workspace: {workspace_name}", + "workspace": workspace_name, }, 200 else: logger.error(f"Terraform destroy failed: {destroy_result.stderr}") return { - 'status': 'error', - 'message': f'Destroy failed: {destroy_result.stderr}', - 'workspace': workspace_name + "status": "error", + "message": f"Destroy failed: {destroy_result.stderr}", + "workspace": workspace_name, }, 500 else: # No terraform files found, try alternative approach - logger.info("No terraform files found, attempting alternative teardown method") - return destroy_via_remote_state(project_id, workspace_name, terraform_state_bucket) - + logger.info( + "No terraform files found, attempting alternative teardown method" + ) + return destroy_via_remote_state( + project_id, workspace_name, terraform_state_bucket + ) + except subprocess.TimeoutExpired: logger.error("Terraform operation timed out") - return { - 'status': 'error', - 'message': 'Teardown operation timed out' - }, 500 + return {"status": "error", "message": "Teardown operation timed out"}, 500 except Exception as e: logger.error(f"Error in teardown function: {str(e)}") - return { - 'status': 'error', - 'message': str(e) - }, 500 + return {"status": "error", "message": str(e)}, 500 -def download_terraform_files(bucket_name: str, workspace_name: str, terraform_dir: Path): +def download_terraform_files( + bucket_name: str, workspace_name: str, terraform_dir: Path +): """Download Terraform files from GCS bucket.""" try: storage_client = gcs.Client() bucket = storage_client.bucket(bucket_name) - + # List and download all terraform files prefix = f"{workspace_name}/terraform/" blobs = bucket.list_blobs(prefix=prefix) - + for blob in blobs: # Get relative path relative_path = blob.name.replace(prefix, "") @@ -153,12 +159,14 @@ def download_terraform_files(bucket_name: str, workspace_name: str, terraform_di logger.warning(f"Could not download terraform files: {e}") -def download_terraform_state(bucket_name: str, workspace_name: str, terraform_dir: Path): +def download_terraform_state( + bucket_name: str, workspace_name: str, terraform_dir: Path +): """Download Terraform state files from GCS bucket.""" try: storage_client = gcs.Client() bucket = storage_client.bucket(bucket_name) - + # Download terraform.tfstate if exists blob_name = f"{workspace_name}/terraform/terraform.tfstate" blob = bucket.blob(blob_name) @@ -179,13 +187,10 @@ def destroy_via_remote_state(project_id: str, workspace_name: str, state_bucket: # This would require creating a backend config and running terraform destroy # For now, return an error suggesting manual teardown return { - 'status': 'error', - 'message': f'Could not automatically teardown. Please run: deployml destroy --config-path ', - 'workspace': workspace_name + "status": "error", + "message": "Could not automatically teardown. Please run: deployml destroy --config-path ", + "workspace": workspace_name, }, 500 except Exception as e: logger.error(f"Remote state teardown failed: {e}") - return { - 'status': 'error', - 'message': str(e) - }, 500 \ No newline at end of file + return {"status": "error", "message": str(e)}, 500 diff --git a/src/deployml/utils/constants.py b/src/deployml/utils/constants.py index 643d4f1..0f359e8 100644 --- a/src/deployml/utils/constants.py +++ b/src/deployml/utils/constants.py @@ -7,30 +7,114 @@ "mlflow": [ {"name": "project_id", "type": "string", "description": "GCP project ID"}, {"name": "region", "type": "string", "description": "Deployment region"}, - {"name": "artifact_bucket", "type": "string", "description": "Bucket for MLflow artifacts"}, - {"name": "backend_store_uri", "type": "string", "description": "URI for MLflow backend store"}, + { + "name": "artifact_bucket", + "type": "string", + "description": "Bucket for MLflow artifacts", + }, + { + "name": "backend_store_uri", + "type": "string", + "description": "URI for MLflow backend store", + }, {"name": "image", "type": "string", "description": "MLflow Docker image"}, ], "wandb": [ {"name": "project_id", "type": "string", "description": "GCP project ID"}, {"name": "region", "type": "string", "description": "Deployment region"}, - {"name": "artifact_bucket", "type": "string", "description": "Bucket for wandb artifacts"}, - {"name": "wandb_port", "type": "number", "description": "Port for wandb server (default 8080)"}, - {"name": "image", "type": "string", "description": "wandb Docker image (optional)"}, + { + "name": "artifact_bucket", + "type": "string", + "description": "Bucket for wandb artifacts", + }, + { + "name": "wandb_port", + "type": "number", + "description": "Port for wandb server (default 8080)", + }, + { + "name": "image", + "type": "string", + "description": "wandb Docker image (optional)", + }, ], "fastapi": [ {"name": "project_id", "type": "string", "description": "GCP project ID"}, {"name": "region", "type": "string", "description": "Deployment region"}, {"name": "image", "type": "string", "description": "FastAPI Docker image"}, - ] + ], } ANIMAL_NAMES = [ - "antelope", "badger", "beaver", "bison", "buffalo", "camel", "cheetah", "cougar", "coyote", "deer", "dingo", "elephant", "elk", "ferret", "fox", "gazelle", "giraffe", "gnu", "goat", "hippo", "hyena", "ibex", "jaguar", "kangaroo", "koala", "leopard", "lion", "llama", "lynx", "mink", "moose", "otter", "panda", "panther", "pig", "platypus", "porcupine", "puma", "rabbit", "raccoon", "ram", "rat", "reindeer", "rhinoceros", "sheep", "skunk", "sloth", "squirrel", "tiger", "walrus", "weasel", "wolf", "wombat", "yak", "zebra" + "antelope", + "badger", + "beaver", + "bison", + "buffalo", + "camel", + "cheetah", + "cougar", + "coyote", + "deer", + "dingo", + "elephant", + "elk", + "ferret", + "fox", + "gazelle", + "giraffe", + "gnu", + "goat", + "hippo", + "hyena", + "ibex", + "jaguar", + "kangaroo", + "koala", + "leopard", + "lion", + "llama", + "lynx", + "mink", + "moose", + "otter", + "panda", + "panther", + "pig", + "platypus", + "porcupine", + "puma", + "rabbit", + "raccoon", + "ram", + "rat", + "reindeer", + "rhinoceros", + "sheep", + "skunk", + "sloth", + "squirrel", + "tiger", + "walrus", + "weasel", + "wolf", + "wombat", + "yak", + "zebra", ] FALLBACK_WORDS = [ - "mlflow", "model", "artifact", "experiment", "data", "pipeline", "deploy", "track", "ai", "ml", "cloud" + "mlflow", + "model", + "artifact", + "experiment", + "data", + "pipeline", + "deploy", + "track", + "ai", + "ml", + "cloud", ] REQUIRED_GCP_APIS = [ @@ -63,4 +147,4 @@ "roles/bigquery.admin", "roles/iam.serviceAccountAdmin", "roles/iam.serviceAccountUser", -] \ No newline at end of file +] diff --git a/src/deployml/utils/helpers.py b/src/deployml/utils/helpers.py index 6c5d69a..614aeac 100644 --- a/src/deployml/utils/helpers.py +++ b/src/deployml/utils/helpers.py @@ -7,7 +7,7 @@ from google.cloud import storage import random import string -from deployml.utils.constants import ANIMAL_NAMES, FALLBACK_WORDS, TERRAFORM_DIR +from deployml.utils.constants import TERRAFORM_DIR from deployml.utils.platform_compat import run_tool, resolve_tool, terraform_env import time from rich.progress import ( @@ -47,9 +47,7 @@ def check_gcp_auth() -> bool: bool: True if authenticated, False otherwise. """ try: - result = run_tool( - "gcloud", ["auth", "list"], capture_output=True, text=True - ) + result = run_tool("gcloud", ["auth", "list"], capture_output=True, text=True) return "ACTIVE" in result.stdout except Exception: return False @@ -59,8 +57,10 @@ def check_gcp_adc() -> bool: """Application Default Credentials are required by Terraform and client libs.""" try: result = run_tool( - "gcloud", ["auth", "application-default", "print-access-token"], - capture_output=True, text=True, + "gcloud", + ["auth", "application-default", "print-access-token"], + capture_output=True, + text=True, ) return result.returncode == 0 except Exception: @@ -72,7 +72,10 @@ def check_bq() -> bool: return False try: result = run_tool( - "bq", ["version"], capture_output=True, text=True, + "bq", + ["version"], + capture_output=True, + text=True, ) return result.returncode == 0 except Exception: @@ -85,9 +88,12 @@ def get_terraform_version() -> Optional[tuple]: return None try: import json as _json + result = run_tool( - "terraform", ["version", "-json"], - capture_output=True, text=True, + "terraform", + ["version", "-json"], + capture_output=True, + text=True, ) if result.returncode != 0: return None @@ -102,9 +108,10 @@ def validate_gcp_project(project_id: str) -> bool: """Verify project exists and active gcloud account can access it.""" try: result = run_tool( - "gcloud", ["projects", "describe", project_id, - "--format=value(projectId)"], - capture_output=True, text=True, + "gcloud", + ["projects", "describe", project_id, "--format=value(projectId)"], + capture_output=True, + text=True, ) return result.returncode == 0 and result.stdout.strip() == project_id except Exception: @@ -146,17 +153,22 @@ def get_missing_iam_roles(project_id: str, required_roles: list) -> list: """Return roles the active account lacks. roles/owner short-circuits to empty.""" try: import json as _json + account_result = run_tool( - "gcloud", ["config", "get-value", "account"], - capture_output=True, text=True, + "gcloud", + ["config", "get-value", "account"], + capture_output=True, + text=True, ) account = account_result.stdout.strip() if not account: return list(required_roles) result = run_tool( - "gcloud", ["projects", "get-iam-policy", project_id, "--format=json"], - capture_output=True, text=True, + "gcloud", + ["projects", "get-iam-policy", project_id, "--format=json"], + capture_output=True, + text=True, ) if result.returncode != 0: return list(required_roles) @@ -181,7 +193,10 @@ def check_docker_daemon() -> bool: return False try: result = run_tool( - "docker", ["info"], capture_output=True, text=True, + "docker", + ["info"], + capture_output=True, + text=True, ) return result.returncode == 0 except Exception: @@ -228,7 +243,7 @@ def copy_modules_to_workspace( tool_name = tool.get("name") if tool_name: used_modules.add(tool_name) - + # Add teardown module to used_modules if teardown is enabled if teardown_enabled: used_modules.add("teardown") @@ -268,9 +283,7 @@ def copy_modules_to_workspace( # Copy only the specific deployment type if specified if deployment_type: - deployment_source = ( - module_path / "cloud" / cloud / deployment_type - ) + deployment_source = module_path / "cloud" / cloud / deployment_type if deployment_source.exists(): deployment_dest = ( dest_module_path / "cloud" / cloud / deployment_type @@ -279,14 +292,10 @@ def copy_modules_to_workspace( shutil.copytree(deployment_source, deployment_dest) else: # Fallback: copy entire module if specific deployment type doesn't exist - shutil.copytree( - module_path, dest_module_path, dirs_exist_ok=True - ) + shutil.copytree(module_path, dest_module_path, dirs_exist_ok=True) else: # Copy entire module if no deployment type specified - shutil.copytree( - module_path, dest_module_path, dirs_exist_ok=True - ) + shutil.copytree(module_path, dest_module_path, dirs_exist_ok=True) def bucket_exists(bucket_name: str, project_id: str) -> bool: @@ -320,9 +329,7 @@ def generate_unique_bucket_name(base_name: str, project_id: str) -> str: str: A unique bucket name. """ while True: - suffix = "".join( - random.choices(string.ascii_lowercase + string.digits, k=6) - ) + suffix = "".join(random.choices(string.ascii_lowercase + string.digits, k=6)) new_name = f"{base_name}-{suffix}" if not bucket_exists(new_name, project_id): return new_name @@ -403,7 +410,8 @@ def cleanup_cloud_sql_resources(terraform_dir: Path, project_id: str): try: result = run_tool( - "terraform", ["output", "-raw", "instance_connection_name"], + "terraform", + ["output", "-raw", "instance_connection_name"], cwd=terraform_dir, capture_output=True, text=True, @@ -415,10 +423,20 @@ def cleanup_cloud_sql_resources(terraform_dir: Path, project_id: str): parts = instance_connection_name.split(":") instance_name = parts[2] if len(parts) == 3 else instance_connection_name - print(f"šŸ—„ļø Restarting Cloud SQL instance to close active connections: {instance_name}") + print( + f"šŸ—„ļø Restarting Cloud SQL instance to close active connections: {instance_name}" + ) run_tool( - "gcloud", ["sql", "instances", "restart", instance_name, - "--project", project_id, "--quiet"], + "gcloud", + [ + "sql", + "instances", + "restart", + instance_name, + "--project", + project_id, + "--quiet", + ], capture_output=True, text=True, ) @@ -453,7 +471,9 @@ def cleanup_terraform_files(terraform_dir: Path): print("āœ… Cleanup completed") -def run_terraform_with_loading_bar(cmd, cwd, estimated_minutes, stack=None, verbose=False): +def run_terraform_with_loading_bar( + cmd, cwd, estimated_minutes, stack=None, verbose=False +): """ Run a subprocess command with a loading bar using rich.progress. Progress messages are based on the stack/resources from the YAML config if provided. @@ -504,8 +524,13 @@ def run_terraform_with_loading_bar(cmd, cwd, estimated_minutes, stack=None, verb if verbose: with open(log_file, "w", encoding="utf-8", errors="replace") as f: process = subprocess.Popen( - cmd, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, - encoding="utf-8", errors="replace", + cmd, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + errors="replace", env=tf_env, ) for line in iter(process.stdout.readline, ""): @@ -540,16 +565,14 @@ def run_terraform_with_loading_bar(cmd, cwd, estimated_minutes, stack=None, verb else: # If we exceed estimated time, slowly approach 95% excess_time = elapsed - estimated_seconds - progress_percent = min(95, 85 + int(excess_time / 30)) # +1% per 30 seconds + progress_percent = min( + 95, 85 + int(excess_time / 30) + ) # +1% per 30 seconds # Choose message based on progress - msg_idx = min( - int(progress_percent / (100 / (n_msgs - 1))), n_msgs - 2 - ) + msg_idx = min(int(progress_percent / (100 / (n_msgs - 1))), n_msgs - 2) message = resource_msgs[msg_idx] - progress.update( - task, completed=progress_percent, description=message - ) + progress.update(task, completed=progress_percent, description=message) time.sleep(1) # Wait for process to fully complete and flush all output @@ -560,7 +583,11 @@ def run_terraform_with_loading_bar(cmd, cwd, estimated_minutes, stack=None, verb if returncode == 0: progress.update(task, completed=100, description=resource_msgs[-1]) else: - progress.update(task, completed=progress_percent, description=f"āš ļø Terraform apply returned code {returncode}") + progress.update( + task, + completed=progress_percent, + description=f"āš ļø Terraform apply returned code {returncode}", + ) return returncode finally: diff --git a/src/deployml/utils/infracost.py b/src/deployml/utils/infracost.py index ef9f1e5..cf5d96a 100644 --- a/src/deployml/utils/infracost.py +++ b/src/deployml/utils/infracost.py @@ -51,7 +51,8 @@ def check_infracost_available() -> bool: """ try: result = run_tool( - "infracost", ["--version"], + "infracost", + ["--version"], capture_output=True, text=True, timeout=10, @@ -65,7 +66,9 @@ def check_infracost_available() -> bool: return False -def run_infracost_breakdown(terraform_dir: Path, usage_file: Optional[Path] = None) -> Optional[Dict]: +def run_infracost_breakdown( + terraform_dir: Path, usage_file: Optional[Path] = None +) -> Optional[Dict]: """ Run infracost breakdown analysis on the terraform directory. @@ -92,7 +95,8 @@ def run_infracost_breakdown(terraform_dir: Path, usage_file: Optional[Path] = No cmd.extend(["--usage-file", str(usage_file)]) result = run_tool( - cmd[0], cmd[1:], + cmd[0], + cmd[1:], cwd=terraform_dir, capture_output=True, text=True, @@ -167,9 +171,7 @@ def parse_infracost_data(data: Dict) -> Optional[CostAnalysis]: total_hourly_cost=float(data.get("totalHourlyCost", 0)), currency=data.get("currency", "USD"), resources=resources, - detected_resources=data.get("summary", {}).get( - "totalDetectedResources", 0 - ), + detected_resources=data.get("summary", {}).get("totalDetectedResources", 0), supported_resources=data.get("summary", {}).get( "totalSupportedResources", 0 ), @@ -205,9 +207,7 @@ def display_cost_breakdown( ), bold=True, ) - typer.echo( - f"Hourly Cost: ${analysis.total_hourly_cost:.4f} {analysis.currency}" - ) + typer.echo(f"Hourly Cost: ${analysis.total_hourly_cost:.4f} {analysis.currency}") typer.echo( f"Resources: {analysis.supported_resources} supported, {analysis.detected_resources} total" ) @@ -242,14 +242,10 @@ def display_cost_breakdown( # Show top cost components if resource.components: - for component in resource.components[ - :3 - ]: # Show top 3 components + for component in resource.components[:3]: # Show top 3 components if component.monthly_cost > 0: usage_note = ( - " (usage-based)" - if component.usage_based - else "" + " (usage-based)" if component.usage_based else "" ) typer.echo( f" ā””ā”€ {component.name}: ${component.monthly_cost:.2f}{usage_note}" @@ -257,9 +253,7 @@ def display_cost_breakdown( # Usage-based resources note usage_based_resources = [ - r - for r in analysis.resources - if any(c.usage_based for c in r.components) + r for r in analysis.resources if any(c.usage_based for c in r.components) ] if usage_based_resources: @@ -287,7 +281,9 @@ def format_cost_for_confirmation(monthly_cost: float, currency: str) -> str: def run_infracost_analysis( - terraform_dir: Path, warning_threshold: float = 100.0, usage_file: Optional[Path] = None + terraform_dir: Path, + warning_threshold: float = 100.0, + usage_file: Optional[Path] = None, ) -> Optional[CostAnalysis]: """ Run complete infracost analysis workflow. @@ -301,9 +297,7 @@ def run_infracost_analysis( """ # Check if infracost is available if not check_infracost_available(): - typer.echo( - "šŸ’” Tip: Install infracost CLI for cost analysis before deployment" - ) + typer.echo("šŸ’” Tip: Install infracost CLI for cost analysis before deployment") typer.echo(" Visit: https://www.infracost.io/docs/#quick-start") return None diff --git a/src/deployml/utils/kubernetes_gke.py b/src/deployml/utils/kubernetes_gke.py index 72ec15f..2f4d235 100644 --- a/src/deployml/utils/kubernetes_gke.py +++ b/src/deployml/utils/kubernetes_gke.py @@ -8,6 +8,7 @@ try: from importlib.metadata import version as _pkg_version + _DEPLOYML_VERSION = _pkg_version("deployml-core") except Exception: _DEPLOYML_VERSION = "0.0.42" @@ -17,7 +18,9 @@ from deployml.utils.platform_compat import run_tool -def check_gke_cluster_connection(cluster_name: str, zone: Optional[str] = None, region: Optional[str] = None) -> bool: +def check_gke_cluster_connection( + cluster_name: str, zone: Optional[str] = None, region: Optional[str] = None +) -> bool: """Check if kubectl is connected to THIS specific GKE cluster. Earlier the function returned True if kubectl was connected to any GKE @@ -25,16 +28,10 @@ def check_gke_cluster_connection(cluster_name: str, zone: Optional[str] = None, only return True if the current context contains the exact cluster name. """ try: - result = run_tool( - "kubectl", ["cluster-info"], - capture_output=True, - text=True - ) + result = run_tool("kubectl", ["cluster-info"], capture_output=True, text=True) if result.returncode == 0: context_result = run_tool( - "kubectl", ["config", "current-context"], - capture_output=True, - text=True + "kubectl", ["config", "current-context"], capture_output=True, text=True ) return cluster_name in context_result.stdout return False @@ -97,7 +94,8 @@ def get_pvc_volume_handle(pvc_name, namespace=None): pv = run_tool( "kubectl", ["get", "pvc", pvc_name, "-o", "jsonpath={.spec.volumeName}"] + ns, - capture_output=True, text=True, + capture_output=True, + text=True, ) pv_name = (pv.stdout or "").strip() if pv.returncode != 0 or not pv_name: @@ -105,7 +103,8 @@ def get_pvc_volume_handle(pvc_name, namespace=None): handle = run_tool( "kubectl", ["get", "pv", pv_name, "-o", "jsonpath={.spec.csi.volumeHandle}"], - capture_output=True, text=True, + capture_output=True, + text=True, ) vh = (handle.stdout or "").strip() return vh or None @@ -123,24 +122,54 @@ def delete_gce_disk_if_exists(project, disk_ref) -> bool: disk_name, loc_flag, loc = disk_ref describe = run_tool( "gcloud", - ["compute", "disks", "describe", disk_name, loc_flag, loc, - "--project", project, "--format=value(name)"], - capture_output=True, text=True, + [ + "compute", + "disks", + "describe", + disk_name, + loc_flag, + loc, + "--project", + project, + "--format=value(name)", + ], + capture_output=True, + text=True, ) if describe.returncode != 0: return True typer.echo(f" Removing orphaned persistent disk {disk_name}...") run_tool( "gcloud", - ["compute", "disks", "delete", disk_name, loc_flag, loc, - "--project", project, "--quiet"], - capture_output=True, text=True, + [ + "compute", + "disks", + "delete", + disk_name, + loc_flag, + loc, + "--project", + project, + "--quiet", + ], + capture_output=True, + text=True, ) verify = run_tool( "gcloud", - ["compute", "disks", "describe", disk_name, loc_flag, loc, - "--project", project, "--format=value(name)"], - capture_output=True, text=True, + [ + "compute", + "disks", + "describe", + disk_name, + loc_flag, + loc, + "--project", + project, + "--format=value(name)", + ], + capture_output=True, + text=True, ) return verify.returncode != 0 @@ -149,36 +178,41 @@ def connect_to_gke_cluster( project_id: str, cluster_name: str, zone: Optional[str] = None, - region: Optional[str] = None + region: Optional[str] = None, ) -> bool: """Connect kubectl to a GKE cluster.""" typer.echo(f"Connecting to GKE cluster: {cluster_name}...") - + try: if zone: cmd = [ - "gcloud", "container", "clusters", "get-credentials", + "gcloud", + "container", + "clusters", + "get-credentials", cluster_name, - "--zone", zone, - "--project", project_id + "--zone", + zone, + "--project", + project_id, ] elif region: cmd = [ - "gcloud", "container", "clusters", "get-credentials", + "gcloud", + "container", + "clusters", + "get-credentials", cluster_name, - "--region", region, - "--project", project_id + "--region", + region, + "--project", + project_id, ] else: typer.echo("Either zone or region must be provided") return False - - result = run_tool( - cmd[0], cmd[1:], - check=True, - capture_output=True, - text=True - ) + + run_tool(cmd[0], cmd[1:], check=True, capture_output=True, text=True) typer.echo(f"Connected to cluster: {cluster_name}") # kubectl will now need the GKE auth plugin; warn early if it is missing. warn_if_gke_auth_plugin_missing() @@ -194,24 +228,22 @@ def connect_to_gke_cluster( def push_image_to_gcr(image_name: str, gcr_image: str, project_id: str) -> bool: """Tag and push Docker image to Google Container Registry.""" typer.echo(f"šŸ“¦ Pushing image to GCR: {gcr_image}...") - + try: # Tag image - tag_result = run_tool( - "docker", ["tag", image_name, gcr_image], + run_tool( + "docker", + ["tag", image_name, gcr_image], check=True, capture_output=True, - text=True + text=True, ) # Push image - push_result = run_tool( - "docker", ["push", gcr_image], - check=True, - capture_output=True, - text=True + run_tool( + "docker", ["push", gcr_image], check=True, capture_output=True, text=True ) - + typer.echo(f"Image pushed successfully: {gcr_image}") return True except subprocess.CalledProcessError as e: @@ -232,7 +264,7 @@ def generate_fastapi_manifests_gke( ) -> None: """ Generate deployment.yaml and service.yaml for FastAPI on GKE. - + Args: output_dir: Directory where manifests will be created image: Docker image for FastAPI (local name) @@ -242,7 +274,7 @@ def generate_fastapi_manifests_gke( push_image: Whether to push image to GCR """ output_dir.mkdir(parents=True, exist_ok=True) - + # Convert local image to GCR format. Pin tag to the deployml version to # avoid the :latest drift bug that bites the Cloud Run path the same way. if not image.startswith("gcr.io/"): @@ -265,14 +297,13 @@ def generate_fastapi_manifests_gke( cpu_limit = "500m" memory_limit = "1Gi" service_name = "fastapi-service" - + # Load templates from files (reuse kubernetes_local templates) template_dir = TEMPLATE_DIR / "kubernetes_local" env = Environment(loader=FileSystemLoader(str(template_dir))) - + deployment_template = env.get_template("deployment.yaml.j2") - service_template = env.get_template("service.yaml.j2") - + # Render deployment template deployment_yaml = deployment_template.render( image=gcr_image, @@ -282,12 +313,14 @@ def generate_fastapi_manifests_gke( memory_request=memory_request, cpu_limit=cpu_limit, memory_limit=memory_limit, - mlflow_tracking_uri=mlflow_tracking_uri + mlflow_tracking_uri=mlflow_tracking_uri, ) - + # Update imagePullPolicy for GCR images - deployment_yaml = deployment_yaml.replace("imagePullPolicy: Never", "imagePullPolicy: IfNotPresent") - + deployment_yaml = deployment_yaml.replace( + "imagePullPolicy: Never", "imagePullPolicy: IfNotPresent" + ) + # Render service template with LoadBalancer service_yaml = f"""apiVersion: v1 kind: Service @@ -304,14 +337,14 @@ def generate_fastapi_manifests_gke( targetPort: {port} protocol: TCP """ - + # Write files deployment_file = output_dir / "deployment.yaml" service_file = output_dir / "service.yaml" - + deployment_file.write_text(deployment_yaml) service_file.write_text(service_yaml) - + typer.echo(f"Generated GKE manifests in {output_dir}") typer.echo(f" - {deployment_file}") typer.echo(f" - {service_file}") @@ -345,7 +378,7 @@ def generate_mlflow_manifests_gke( pvc_size: PVC size when use_pvc=True. """ output_dir.mkdir(parents=True, exist_ok=True) - + # Convert local image to GCR format. Pin tag to the deployml version. if not image.startswith("gcr.io/"): gcr_image = f"gcr.io/{project_id}/mlflow/mlflow:v{_DEPLOYML_VERSION}" @@ -367,24 +400,26 @@ def generate_mlflow_manifests_gke( cpu_limit = "500m" memory_limit = "2Gi" # Increased for GKE service_name = "mlflow-service" - + # Defaults if not provided. Put sqlite on the mounted volume (4 slashes = # absolute /mlflow-artifacts/mlflow.db) so the backend store persists with # use_pvc. A relative sqlite:///mlflow.db would sit in the ephemeral # container filesystem and be lost on restart. if not backend_store_uri: backend_store_uri = ( - "sqlite:////mlflow-artifacts/mlflow.db" if use_pvc else "sqlite:///mlflow.db" + "sqlite:////mlflow-artifacts/mlflow.db" + if use_pvc + else "sqlite:///mlflow.db" ) if not artifact_root: artifact_root = "/mlflow-artifacts" - + # Load templates from files template_dir = TEMPLATE_DIR / "kubernetes_local" env = Environment(loader=FileSystemLoader(str(template_dir))) - + deployment_template = env.get_template("mlflow-deployment.yaml.j2") - + # Render deployment template deployment_yaml = deployment_template.render( image=gcr_image, @@ -400,8 +435,10 @@ def generate_mlflow_manifests_gke( ) # Update imagePullPolicy for GCR images - deployment_yaml = deployment_yaml.replace("imagePullPolicy: Never", "imagePullPolicy: IfNotPresent") - + deployment_yaml = deployment_yaml.replace( + "imagePullPolicy: Never", "imagePullPolicy: IfNotPresent" + ) + # Render service template with LoadBalancer service_yaml = f"""apiVersion: v1 kind: Service @@ -418,11 +455,11 @@ def generate_mlflow_manifests_gke( targetPort: {port} protocol: TCP """ - + # Write files deployment_file = output_dir / "deployment.yaml" service_file = output_dir / "service.yaml" - + deployment_file.write_text(deployment_yaml) service_file.write_text(service_yaml) @@ -462,19 +499,19 @@ def deploy_to_gke( if not manifest_dir.exists(): typer.echo(f"Directory not found: {manifest_dir}") return False - + deployment_file = manifest_dir / "deployment.yaml" service_file = manifest_dir / "service.yaml" - + if not deployment_file.exists() or not service_file.exists(): typer.echo(f"Required manifest files not found in {manifest_dir}") return False - + # Connect to cluster if not already connected if not check_gke_cluster_connection(cluster_name, zone, region): if not connect_to_gke_cluster(project_id, cluster_name, zone, region): return False - + typer.echo("šŸš€ Applying Kubernetes manifests to GKE...") ensure_namespace(namespace) ns = ns_args(namespace) @@ -485,39 +522,44 @@ def deploy_to_gke( if pvc_file.exists(): typer.echo(f" Applying {pvc_file.name}...") result = run_tool( - "kubectl", ["apply", "-f", str(pvc_file)] + ns, + "kubectl", + ["apply", "-f", str(pvc_file)] + ns, check=True, capture_output=True, - text=True + text=True, ) typer.echo(f" {result.stdout.strip()}") typer.echo(f" Applying {deployment_file.name}...") result = run_tool( - "kubectl", ["apply", "-f", str(deployment_file)] + ns, + "kubectl", + ["apply", "-f", str(deployment_file)] + ns, check=True, capture_output=True, - text=True + text=True, ) typer.echo(f" {result.stdout.strip()}") typer.echo(f" Applying {service_file.name}...") result = run_tool( - "kubectl", ["apply", "-f", str(service_file)] + ns, + "kubectl", + ["apply", "-f", str(service_file)] + ns, check=True, capture_output=True, - text=True + text=True, ) typer.echo(f" {result.stdout.strip()}") - + # Get service URL (LoadBalancer) typer.echo("\nā³ Waiting for LoadBalancer IP...") typer.echo(" (This may take a few minutes)") - + # Wait for external IP. Earlier code did service_file.stem.replace("service", "service") # which is a no-op and then queried kubectl for service "service" which is wrong. # Read the actual service name from the rendered manifest instead. - import time, yaml as _yaml + import time + import yaml as _yaml + max_wait = 300 waited = 0 try: @@ -530,22 +572,38 @@ def deploy_to_gke( # Querying every service and guessing an IP risks reporting the wrong # endpoint, so bail out and let the user inspect manually instead. if not service_name: - typer.echo(" Could not read the service name from service.yaml; " - "skipping IP wait. Run: kubectl get svc") + typer.echo( + " Could not read the service name from service.yaml; " + "skipping IP wait. Run: kubectl get svc" + ) waited = max_wait while waited < max_wait: ip_query = "{.status.loadBalancer.ingress[0].ip}" - cmd = ["kubectl", "get", "svc", service_name, - "-o", f"jsonpath={ip_query}"] + ns + cmd = [ + "kubectl", + "get", + "svc", + service_name, + "-o", + f"jsonpath={ip_query}", + ] + ns result = run_tool(cmd[0], cmd[1:], capture_output=True, text=True) external_ip = result.stdout.strip().strip("'") if result.returncode == 0 and external_ip and external_ip != "": port_query = "{.spec.ports[0].port}" - port_cmd = ["kubectl", "get", "svc", service_name, - "-o", f"jsonpath={port_query}"] + ns - port_result = run_tool(port_cmd[0], port_cmd[1:], capture_output=True, text=True) + port_cmd = [ + "kubectl", + "get", + "svc", + service_name, + "-o", + f"jsonpath={port_query}", + ] + ns + port_result = run_tool( + port_cmd[0], port_cmd[1:], capture_output=True, text=True + ) port = port_result.stdout.strip().strip("'") or "5000" typer.echo(f"\n Service is available at: http://{external_ip}:{port}") break @@ -554,13 +612,13 @@ def deploy_to_gke( waited += 5 if waited % 30 == 0: typer.echo(f" Still waiting... ({waited}s)") - + typer.echo("\n Deployment status:") run_tool("kubectl", ["get", "pods"] + ns) run_tool("kubectl", ["get", "svc"] + ns) return True - + except subprocess.CalledProcessError as e: typer.echo(f"Deployment failed: {e.stderr}") return False diff --git a/src/deployml/utils/kubernetes_local.py b/src/deployml/utils/kubernetes_local.py index b102b1b..9d0309f 100644 --- a/src/deployml/utils/kubernetes_local.py +++ b/src/deployml/utils/kubernetes_local.py @@ -1,7 +1,7 @@ import subprocess import typer from pathlib import Path -from typing import Optional, Dict +from typing import Optional from jinja2 import Environment, FileSystemLoader from deployml.utils.constants import TEMPLATE_DIR from deployml.utils.platform_compat import run_tool @@ -19,23 +19,26 @@ def ensure_namespace(namespace: Optional[str]) -> None: if not namespace or namespace == "default": return rendered = run_tool( - "kubectl", ["create", "namespace", namespace, "--dry-run=client", "-o", "yaml"], - capture_output=True, text=True, + "kubectl", + ["create", "namespace", namespace, "--dry-run=client", "-o", "yaml"], + capture_output=True, + text=True, ) if rendered.returncode == 0: - run_tool("kubectl", ["apply", "-f", "-"], input=rendered.stdout, - capture_output=True, text=True) + run_tool( + "kubectl", + ["apply", "-f", "-"], + input=rendered.stdout, + capture_output=True, + text=True, + ) typer.echo(f" Using namespace: {namespace}") def check_minikube_running() -> bool: """Check if minikube is currently running.""" try: - result = run_tool( - "minikube", ["status"], - capture_output=True, - text=True - ) + result = run_tool("minikube", ["status"], capture_output=True, text=True) return "Running" in result.stdout except Exception: return False @@ -45,12 +48,7 @@ def start_minikube() -> bool: """Start minikube cluster.""" typer.echo("Starting minikube...") try: - result = run_tool( - "minikube", ["start"], - check=True, - capture_output=True, - text=True - ) + run_tool("minikube", ["start"], check=True, capture_output=True, text=True) typer.echo("Minikube started successfully!") return True except subprocess.CalledProcessError as e: @@ -69,7 +67,7 @@ def generate_fastapi_manifests( ) -> None: """ Generate deployment.yaml and service.yaml for FastAPI in the specified directory. - + Args: output_dir: Directory where manifests will be created image: Docker image for FastAPI @@ -77,11 +75,11 @@ def generate_fastapi_manifests( load_image: Whether to automatically load image into minikube (default: True) """ output_dir.mkdir(parents=True, exist_ok=True) - + # Load image into minikube if requested if load_image: load_image_to_minikube(image) - + # Default values port = 8000 node_port = 30080 @@ -91,14 +89,14 @@ def generate_fastapi_manifests( cpu_limit = "500m" memory_limit = "1Gi" service_name = "fastapi-service" - + # Load templates from files template_dir = TEMPLATE_DIR / "kubernetes_local" env = Environment(loader=FileSystemLoader(str(template_dir))) - + deployment_template = env.get_template("deployment.yaml.j2") service_template = env.get_template("service.yaml.j2") - + # Render templates deployment_yaml = deployment_template.render( image=image, @@ -108,22 +106,20 @@ def generate_fastapi_manifests( memory_request=memory_request, cpu_limit=cpu_limit, memory_limit=memory_limit, - mlflow_tracking_uri=mlflow_tracking_uri + mlflow_tracking_uri=mlflow_tracking_uri, ) - + service_yaml = service_template.render( - service_name=service_name, - port=port, - node_port=node_port + service_name=service_name, port=port, node_port=node_port ) - + # Write files deployment_file = output_dir / "deployment.yaml" service_file = output_dir / "service.yaml" - + deployment_file.write_text(deployment_yaml) service_file.write_text(service_yaml) - + typer.echo(f"Generated manifests in {output_dir}") typer.echo(f" - {deployment_file}") typer.echo(f" - {service_file}") @@ -196,9 +192,7 @@ def generate_mlflow_manifests( ) service_yaml = service_template.render( - service_name=service_name, - port=port, - node_port=node_port + service_name=service_name, port=port, node_port=node_port ) deployment_file = output_dir / "deployment.yaml" @@ -219,8 +213,11 @@ def generate_mlflow_manifests( typer.echo(f" - {pvc_file} (PersistentVolumeClaim, {pvc_size})") -def deploy_mlflow_to_minikube(manifest_dir: Path, image_name: Optional[str] = None, - namespace: Optional[str] = None) -> bool: +def deploy_mlflow_to_minikube( + manifest_dir: Path, + image_name: Optional[str] = None, + namespace: Optional[str] = None, +) -> bool: """ Deploy MLflow to minikube using kubectl apply. @@ -233,29 +230,30 @@ def deploy_mlflow_to_minikube(manifest_dir: Path, image_name: Optional[str] = No if not manifest_dir.exists(): typer.echo(f"Directory not found: {manifest_dir}") return False - + deployment_file = manifest_dir / "deployment.yaml" service_file = manifest_dir / "service.yaml" - + if not deployment_file.exists() or not service_file.exists(): typer.echo(f"Required manifest files not found in {manifest_dir}") return False - + # Extract image name from deployment if not provided if not image_name: try: content = deployment_file.read_text() import re - match = re.search(r'image:\s*([^\s]+)', content) + + match = re.search(r"image:\s*([^\s]+)", content) if match: image_name = match.group(1) except Exception: pass - + # Load image if provided if image_name: load_image_to_minikube(image_name) - + typer.echo("Applying Kubernetes manifests...") ensure_namespace(namespace) ns = ns_args(namespace) @@ -266,27 +264,32 @@ def deploy_mlflow_to_minikube(manifest_dir: Path, image_name: Optional[str] = No if pvc_file.exists(): typer.echo(f" Applying {pvc_file.name}...") result = run_tool( - "kubectl", ["apply", "-f", str(pvc_file)] + ns, - check=True, capture_output=True, text=True, + "kubectl", + ["apply", "-f", str(pvc_file)] + ns, + check=True, + capture_output=True, + text=True, ) typer.echo(f"{result.stdout.strip()}") typer.echo(f" Applying {deployment_file.name}...") result = run_tool( - "kubectl", ["apply", "-f", str(deployment_file)] + ns, + "kubectl", + ["apply", "-f", str(deployment_file)] + ns, check=True, capture_output=True, - text=True + text=True, ) typer.echo(f"{result.stdout.strip()}") # Apply service typer.echo(f" Applying {service_file.name}...") result = run_tool( - "kubectl", ["apply", "-f", str(service_file)] + ns, + "kubectl", + ["apply", "-f", str(service_file)] + ns, check=True, capture_output=True, - text=True + text=True, ) typer.echo(f"{result.stdout.strip()}") @@ -296,10 +299,11 @@ def deploy_mlflow_to_minikube(manifest_dir: Path, image_name: Optional[str] = No # Try minikube service --url with timeout (can hang) try: result = run_tool( - "minikube", ["service", "mlflow-service", "--url"] + ns, + "minikube", + ["service", "mlflow-service", "--url"] + ns, capture_output=True, text=True, - timeout=5 # 5 second timeout to prevent hanging + timeout=5, # 5 second timeout to prevent hanging ) if result.returncode == 0 and result.stdout.strip(): @@ -311,16 +315,22 @@ def deploy_mlflow_to_minikube(manifest_dir: Path, image_name: Optional[str] = No # Fallback: get NodePort manually (more reliable) typer.echo(" Getting NodePort...") result = run_tool( - "kubectl", ["get", "svc", "mlflow-service", "-o", "jsonpath='{.spec.ports[0].nodePort}'"] + ns, + "kubectl", + [ + "get", + "svc", + "mlflow-service", + "-o", + "jsonpath='{.spec.ports[0].nodePort}'", + ] + + ns, capture_output=True, - text=True + text=True, ) if result.returncode == 0: node_port = result.stdout.strip().strip("'") minikube_ip_result = run_tool( - "minikube", ["ip"], - capture_output=True, - text=True + "minikube", ["ip"], capture_output=True, text=True ) if minikube_ip_result.returncode == 0: minikube_ip = minikube_ip_result.stdout.strip() @@ -328,14 +338,16 @@ def deploy_mlflow_to_minikube(manifest_dir: Path, image_name: Optional[str] = No else: typer.echo(" Could not determine minikube IP") else: - typer.echo(" Could not determine service URL. Check with: kubectl get svc mlflow-service") + typer.echo( + " Could not determine service URL. Check with: kubectl get svc mlflow-service" + ) typer.echo("\n Deployment status:") run_tool("kubectl", ["get", "pods", "-l", "app=mlflow"] + ns) run_tool("kubectl", ["get", "svc", "-l", "app=mlflow"] + ns) - + return True - + except subprocess.CalledProcessError as e: typer.echo(f"Deployment failed: {e.stderr}") return False @@ -344,8 +356,11 @@ def deploy_mlflow_to_minikube(manifest_dir: Path, image_name: Optional[str] = No return False -def deploy_fastapi_to_minikube(manifest_dir: Path, image_name: Optional[str] = None, - namespace: Optional[str] = None) -> bool: +def deploy_fastapi_to_minikube( + manifest_dir: Path, + image_name: Optional[str] = None, + namespace: Optional[str] = None, +) -> bool: """ Deploy FastAPI to minikube using kubectl apply. @@ -359,29 +374,30 @@ def deploy_fastapi_to_minikube(manifest_dir: Path, image_name: Optional[str] = N if not manifest_dir.exists(): typer.echo(f"Directory not found: {manifest_dir}") return False - + deployment_file = manifest_dir / "deployment.yaml" service_file = manifest_dir / "service.yaml" - + if not deployment_file.exists() or not service_file.exists(): typer.echo(f"Required manifest files not found in {manifest_dir}") return False - + # Extract image name from deployment if not provided if not image_name: try: content = deployment_file.read_text() import re - match = re.search(r'image:\s*([^\s]+)', content) + + match = re.search(r"image:\s*([^\s]+)", content) if match: image_name = match.group(1) except Exception: pass - + # Load image if provided if image_name: load_image_to_minikube(image_name) - + typer.echo("Applying Kubernetes manifests...") ensure_namespace(namespace) ns = ns_args(namespace) @@ -389,20 +405,22 @@ def deploy_fastapi_to_minikube(manifest_dir: Path, image_name: Optional[str] = N try: typer.echo(f" Applying {deployment_file.name}...") result = run_tool( - "kubectl", ["apply", "-f", str(deployment_file)] + ns, + "kubectl", + ["apply", "-f", str(deployment_file)] + ns, check=True, capture_output=True, - text=True + text=True, ) typer.echo(f"{result.stdout.strip()}") # Apply service typer.echo(f" Applying {service_file.name}...") result = run_tool( - "kubectl", ["apply", "-f", str(service_file)] + ns, + "kubectl", + ["apply", "-f", str(service_file)] + ns, check=True, capture_output=True, - text=True + text=True, ) typer.echo(f"{result.stdout.strip()}") @@ -412,10 +430,11 @@ def deploy_fastapi_to_minikube(manifest_dir: Path, image_name: Optional[str] = N # Try minikube service --url with timeout (can hang) try: result = run_tool( - "minikube", ["service", "fastapi-service", "--url"] + ns, + "minikube", + ["service", "fastapi-service", "--url"] + ns, capture_output=True, text=True, - timeout=5 # 5 second timeout to prevent hanging + timeout=5, # 5 second timeout to prevent hanging ) if result.returncode == 0 and result.stdout.strip(): @@ -427,16 +446,22 @@ def deploy_fastapi_to_minikube(manifest_dir: Path, image_name: Optional[str] = N # Fallback: get NodePort manually (more reliable) typer.echo(" Getting NodePort...") result = run_tool( - "kubectl", ["get", "svc", "fastapi-service", "-o", "jsonpath='{.spec.ports[0].nodePort}'"] + ns, + "kubectl", + [ + "get", + "svc", + "fastapi-service", + "-o", + "jsonpath='{.spec.ports[0].nodePort}'", + ] + + ns, capture_output=True, - text=True + text=True, ) if result.returncode == 0: node_port = result.stdout.strip().strip("'") minikube_ip_result = run_tool( - "minikube", ["ip"], - capture_output=True, - text=True + "minikube", ["ip"], capture_output=True, text=True ) if minikube_ip_result.returncode == 0: minikube_ip = minikube_ip_result.stdout.strip() @@ -444,14 +469,16 @@ def deploy_fastapi_to_minikube(manifest_dir: Path, image_name: Optional[str] = N else: typer.echo(" Could not determine minikube IP") else: - typer.echo(" Could not determine service URL. Check with: kubectl get svc fastapi-service") + typer.echo( + " Could not determine service URL. Check with: kubectl get svc fastapi-service" + ) typer.echo("\n Deployment status:") run_tool("kubectl", ["get", "pods", "-l", "app=fastapi"] + ns) run_tool("kubectl", ["get", "svc", "-l", "app=fastapi"] + ns) - + return True - + except subprocess.CalledProcessError as e: typer.echo(f"Deployment failed: {e.stderr}") return False @@ -463,48 +490,43 @@ def deploy_fastapi_to_minikube(manifest_dir: Path, image_name: Optional[str] = N def load_image_to_minikube(image_name: str) -> bool: """ Load a Docker image into minikube if it exists locally. - + Args: image_name: Name of the Docker image to load - + Returns: True if image was loaded or already exists, False otherwise """ # Check if image exists locally result = run_tool( - "docker", ["images", "-q", image_name], - capture_output=True, - text=True + "docker", ["images", "-q", image_name], capture_output=True, text=True ) - + if not result.stdout.strip(): typer.echo(f"āš ļø Image '{image_name}' not found locally") typer.echo(f" Build it first: docker build -t {image_name} .") return False - + # Check if image is already in minikube - result = run_tool( - "minikube", ["image", "ls"], - capture_output=True, - text=True - ) - + result = run_tool("minikube", ["image", "ls"], capture_output=True, text=True) + if image_name in result.stdout: typer.echo(f"āœ… Image '{image_name}' already in minikube") return True - + # Load image into minikube typer.echo(f"šŸ“¦ Loading image '{image_name}' into minikube...") result = run_tool( - "minikube", ["image", "load", image_name], + "minikube", + ["image", "load", image_name], check=True, capture_output=True, - text=True + text=True, ) - + if result.returncode == 0: typer.echo(f"āœ… Image '{image_name}' loaded into minikube") return True else: typer.echo(f"āŒ Failed to load image: {result.stderr}") - return False \ No newline at end of file + return False diff --git a/src/deployml/utils/platform_compat.py b/src/deployml/utils/platform_compat.py index 8feddb1..787a629 100644 --- a/src/deployml/utils/platform_compat.py +++ b/src/deployml/utils/platform_compat.py @@ -120,7 +120,9 @@ def find_windows_bash() -> "str | None": for base in (program_files, program_files_x86): candidates.append(os.path.join(base, "Git", "bin", "bash.exe")) if local_appdata: - candidates.append(os.path.join(local_appdata, "Programs", "Git", "bin", "bash.exe")) + candidates.append( + os.path.join(local_appdata, "Programs", "Git", "bin", "bash.exe") + ) for path in candidates: if path and os.path.isfile(path): return path @@ -176,6 +178,7 @@ def robust_rmtree(path) -> None: raises PermissionError. The error handler clears the read only bit and retries, then pauses briefly and retries once more before giving up. """ + def _handle(func, target, _exc): try: os.chmod(target, stat.S_IWRITE) diff --git a/src/deployml/utils/teardown.py b/src/deployml/utils/teardown.py index b730d61..ca40688 100644 --- a/src/deployml/utils/teardown.py +++ b/src/deployml/utils/teardown.py @@ -1,4 +1,5 @@ """Utilities for managing auto-teardown functionality.""" + import json from pathlib import Path from datetime import datetime, timedelta @@ -12,8 +13,8 @@ def save_deployment_metadata(workspace_dir: Path, metadata: Dict[str, Any]): """Save deployment metadata including teardown schedule.""" metadata_path = workspace_dir / METADATA_FILE metadata_path.parent.mkdir(parents=True, exist_ok=True) - - with open(metadata_path, 'w') as f: + + with open(metadata_path, "w") as f: json.dump(metadata, f, indent=2) @@ -22,8 +23,8 @@ def load_deployment_metadata(workspace_dir: Path) -> Optional[Dict[str, Any]]: metadata_path = workspace_dir / METADATA_FILE if not metadata_path.exists(): return None - - with open(metadata_path, 'r') as f: + + with open(metadata_path, "r") as f: return json.load(f) @@ -33,13 +34,13 @@ def calculate_teardown_schedule(deployed_at: datetime, duration_hours: int) -> s Returns cron string like "0 2 15 1 *" for Jan 15 at 2:00 AM UTC """ teardown_time = deployed_at + timedelta(hours=duration_hours) - + # Format: minute hour day month day-of-week minute = teardown_time.minute hour = teardown_time.hour day = teardown_time.day month = teardown_time.month - + return f"{minute} {hour} {day} {month} *" @@ -47,6 +48,6 @@ def calculate_cron_from_timestamp(timestamp: int) -> str: """Convert Unix timestamp to cron expression.""" # Use UTC timezone-aware datetime to ensure correct conversion from datetime import timezone + dt = datetime.fromtimestamp(timestamp, tz=timezone.utc) return f"{dt.minute} {dt.hour} {dt.day} {dt.month} *" - diff --git a/tests/test_doctor.py b/tests/test_doctor.py index 6596ff3..da6c688 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -1,4 +1,5 @@ """Unit tests for the deployml doctor checks. No real Docker daemon or GCP calls.""" + from types import SimpleNamespace from unittest.mock import patch @@ -12,8 +13,10 @@ def test_docker_permissions_skips_when_docker_missing(): missing binary, so a second 'cannot run docker' failure is noise.""" d = DeployMLDoctor() d.results.clear() - with patch.object(doctor_mod.shutil, "which", return_value=None), \ - patch.object(doctor_mod, "run_tool", side_effect=FileNotFoundError("docker")): + with ( + patch.object(doctor_mod.shutil, "which", return_value=None), + patch.object(doctor_mod, "run_tool", side_effect=FileNotFoundError("docker")), + ): d._check_docker_permissions() assert len(d.results) == 1 result = d.results[0] @@ -27,7 +30,9 @@ def test_docker_permissions_pass_when_docker_runs(): d = DeployMLDoctor() d.results.clear() ok = SimpleNamespace(returncode=0, stdout="", stderr="") - with patch.object(doctor_mod.shutil, "which", return_value="/usr/local/bin/docker"), \ - patch.object(doctor_mod, "run_tool", return_value=ok): + with ( + patch.object(doctor_mod.shutil, "which", return_value="/usr/local/bin/docker"), + patch.object(doctor_mod, "run_tool", return_value=ok), + ): d._check_docker_permissions() assert d.results[0].status == CheckStatus.PASS diff --git a/tests/test_gke_destroy.py b/tests/test_gke_destroy.py index 94c76b7..b48213f 100644 --- a/tests/test_gke_destroy.py +++ b/tests/test_gke_destroy.py @@ -5,6 +5,7 @@ asynchronously, and deleting the cluster too soon orphans a billing disk. No real GCP or kubectl calls; run_tool is mocked at the boundary. """ + from types import SimpleNamespace from unittest.mock import patch @@ -13,6 +14,7 @@ # ---------- disk_ref_from_volume_handle (pure parsing) ---------- + def test_disk_ref_from_zonal_volume_handle(): h = "projects/my-proj/zones/us-west1-a/disks/pvc-1234" assert gke.disk_ref_from_volume_handle(h) == ("pvc-1234", "--zone", "us-west1-a") @@ -31,6 +33,7 @@ def test_disk_ref_from_volume_handle_none_for_garbage(): # ---------- get_pvc_volume_handle (boundary mocked) ---------- + def test_get_pvc_volume_handle_returns_handle_when_bound(): def fake_run_tool(name, args, **kw): if args[:2] == ["get", "pvc"]: @@ -42,17 +45,23 @@ def fake_run_tool(name, args, **kw): raise AssertionError(f"unexpected call {args}") with patch.object(gke, "run_tool", side_effect=fake_run_tool): - assert gke.get_pvc_volume_handle("mlflow-pvc") == "projects/p/zones/z/disks/pvc-1" + assert ( + gke.get_pvc_volume_handle("mlflow-pvc") == "projects/p/zones/z/disks/pvc-1" + ) def test_get_pvc_volume_handle_none_when_unbound(): - with patch.object(gke, "run_tool", - return_value=SimpleNamespace(returncode=0, stdout="", stderr="")): + with patch.object( + gke, + "run_tool", + return_value=SimpleNamespace(returncode=0, stdout="", stderr=""), + ): assert gke.get_pvc_volume_handle("mlflow-pvc") is None # ---------- delete_gce_disk_if_exists (boundary mocked) ---------- + def test_delete_gce_disk_skips_when_already_gone(): """If the CSI driver already reclaimed the disk, describe fails and no delete must be issued.""" @@ -70,8 +79,8 @@ def fake_run_tool(name, args, **kw): def test_delete_gce_disk_deletes_when_present(): seq = [ - SimpleNamespace(returncode=0, stdout="pvc-1\n", stderr=""), # describe: found - SimpleNamespace(returncode=0, stdout="", stderr=""), # delete + SimpleNamespace(returncode=0, stdout="pvc-1\n", stderr=""), # describe: found + SimpleNamespace(returncode=0, stdout="", stderr=""), # delete SimpleNamespace(returncode=1, stdout="", stderr="NOT_FOUND"), # describe: gone ] calls = [] diff --git a/tests/test_helpers.py b/tests/test_helpers.py index fcc2f41..f45baaf 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,4 +1,5 @@ """Unit tests for deployml helpers and config validators. No GCP calls.""" + from unittest.mock import patch, MagicMock import pytest @@ -23,6 +24,7 @@ # ---------- _load_config_or_exit ---------- + def test_load_config_valid_mapping(tmp_path): f = tmp_path / "c.yaml" f.write_text("foo: bar\nnested:\n key: 1\n") @@ -59,6 +61,7 @@ def test_load_config_scalar_top_level_exits(tmp_path): # ---------- _validate_deploy_config_or_exit ---------- + def test_validate_deploy_full_gcp(): cfg = { "provider": {"name": "gcp", "project_id": "test-123"}, @@ -82,7 +85,9 @@ def test_validate_deploy_missing_provider_exits(): def test_validate_deploy_provider_not_a_dict_exits(): with pytest.raises(typer.Exit): - _validate_deploy_config_or_exit({"provider": "gcp", "deployment": {"type": "x"}}) + _validate_deploy_config_or_exit( + {"provider": "gcp", "deployment": {"type": "x"}} + ) def test_validate_deploy_bad_provider_name_exits(): @@ -111,6 +116,7 @@ def test_validate_deploy_missing_deployment_type_exits(): # ---------- stack validation (#53) ---------- + def test_validate_deploy_valid_stack_ok(): cfg = { "provider": {"name": "gcp", "project_id": "x"}, @@ -157,12 +163,16 @@ def test_validate_deploy_stage_not_a_dict_exits(): def test_validate_deploy_no_stack_key_still_ok(): # Stack is validated only when present, so configs without it still pass. - cfg = {"provider": {"name": "gcp", "project_id": "x"}, "deployment": {"type": "cloud_run"}} + cfg = { + "provider": {"name": "gcp", "project_id": "x"}, + "deployment": {"type": "cloud_run"}, + } _validate_deploy_config_or_exit(cfg) # ---------- _gcp_credentials_preflight_or_exit (#54) ---------- + @patch("deployml.cli.cli.check_gcp_adc", return_value=True) @patch("deployml.cli.cli.check_gcp_auth", return_value=True) def test_gcp_preflight_passes_with_auth_and_adc(mock_auth, mock_adc): @@ -187,6 +197,7 @@ def test_gcp_preflight_exits_when_adc_missing(mock_auth, mock_adc): # ---------- check_gcp_adc ---------- + @patch("deployml.utils.helpers.run_tool") def test_check_gcp_adc_returncode_zero_true(mock_run): mock_run.return_value = MagicMock(returncode=0) @@ -207,6 +218,7 @@ def test_check_gcp_adc_exception_returns_false(mock_run): # ---------- check_bq ---------- + @patch("deployml.utils.helpers.shutil.which") def test_check_bq_binary_missing_false(mock_which): mock_which.return_value = None @@ -231,6 +243,7 @@ def test_check_bq_binary_present_but_errors(mock_run, mock_which): # ---------- get_terraform_version ---------- + @patch("deployml.utils.helpers.shutil.which") def test_get_terraform_version_binary_missing(mock_which): mock_which.return_value = None @@ -258,6 +271,7 @@ def test_get_terraform_version_bad_json_returns_none(mock_run, mock_which): # ---------- validate_gcp_project ---------- + @patch("deployml.utils.helpers.run_tool") def test_validate_gcp_project_exists(mock_run): mock_run.return_value = MagicMock(returncode=0, stdout="my-project\n") @@ -279,6 +293,7 @@ def test_validate_gcp_project_stdout_mismatch(mock_run): # ---------- validate_gcp_region ---------- + @patch("deployml.utils.helpers.run_tool") def test_validate_gcp_region_in_list(mock_run): helpers_mod._GCP_REGIONS_CACHE = None @@ -308,6 +323,7 @@ def test_validate_gcp_region_lookup_failure_does_not_block(mock_run): # ---------- check_docker_daemon ---------- + @patch("deployml.utils.helpers.shutil.which") def test_check_docker_daemon_binary_missing(mock_which): mock_which.return_value = None @@ -332,6 +348,7 @@ def test_check_docker_daemon_down(mock_run, mock_which): # ---------- get_missing_iam_roles ---------- + @patch("deployml.utils.helpers.run_tool") def test_get_missing_iam_roles_owner_short_circuits(mock_run): mock_run.side_effect = [ @@ -376,6 +393,7 @@ def test_get_missing_iam_roles_policy_query_fails_returns_all(mock_run): # ---------- teardown cron timezone correctness ---------- + def test_calculate_cron_from_timestamp_round_trip(): """The cron string must reflect the UTC time of the timestamp, with no TZ skew. Regression for the datetime.utcnow() bug that off-set teardown diff --git a/tests/test_platform_compat.py b/tests/test_platform_compat.py index 9dee129..cc77fd9 100644 --- a/tests/test_platform_compat.py +++ b/tests/test_platform_compat.py @@ -5,6 +5,7 @@ run_tool's kwarg passthrough were never exercised. No real external tools are launched; the only real side effect is rmtree on a pytest tmp_path. """ + import os import stat from types import SimpleNamespace @@ -17,6 +18,7 @@ # ---------- resolve_tool ---------- + def test_resolve_tool_returns_path_when_found(): with patch.object(pc.shutil, "which", return_value="/usr/local/bin/gcloud"): assert pc.resolve_tool("gcloud") == "/usr/local/bin/gcloud" @@ -31,18 +33,23 @@ def test_resolve_tool_raises_filenotfound_when_missing(): # ---------- run_tool ---------- + def test_run_tool_forwards_resolved_path_args_and_kwargs(): """run_tool must prepend the resolved path and pass every kwarg straight through to subprocess.run. The whole blocker 1 refactor depends on this.""" completed = SimpleNamespace(returncode=0) - with patch.object(pc, "resolve_tool", return_value="/usr/local/bin/gcloud"), \ - patch.object(pc.subprocess, "run", return_value=completed) as mock_run: - result = pc.run_tool("gcloud", ["version", "--quiet"], - capture_output=True, check=True) + with ( + patch.object(pc, "resolve_tool", return_value="/usr/local/bin/gcloud"), + patch.object(pc.subprocess, "run", return_value=completed) as mock_run, + ): + result = pc.run_tool( + "gcloud", ["version", "--quiet"], capture_output=True, check=True + ) assert result is completed mock_run.assert_called_once_with( ["/usr/local/bin/gcloud", "version", "--quiet"], - capture_output=True, check=True, + capture_output=True, + check=True, ) @@ -50,9 +57,11 @@ def test_run_tool_decodes_utf8_in_windows_text_mode(): """On Windows, text-mode captures must be decoded as UTF-8 with replacement to avoid the cp1252 UnicodeDecodeError. Off Windows this branch is skipped.""" completed = SimpleNamespace(returncode=0) - with patch.object(pc, "IS_WINDOWS", True), \ - patch.object(pc, "resolve_tool", return_value=r"C:\sdk\bin\gcloud.cmd"), \ - patch.object(pc.subprocess, "run", return_value=completed) as mock_run: + with ( + patch.object(pc, "IS_WINDOWS", True), + patch.object(pc, "resolve_tool", return_value=r"C:\sdk\bin\gcloud.cmd"), + patch.object(pc.subprocess, "run", return_value=completed) as mock_run, + ): pc.run_tool("gcloud", ["version"], text=True) _, kwargs = mock_run.call_args assert kwargs.get("encoding") == "utf-8" @@ -61,6 +70,7 @@ def test_run_tool_decodes_utf8_in_windows_text_mode(): # ---------- robust_rmtree ---------- + def test_robust_rmtree_removes_directory_tree(tmp_path): target = tmp_path / "ws" (target / "sub").mkdir(parents=True) @@ -87,6 +97,7 @@ def test_robust_rmtree_removes_tree_with_readonly_file(tmp_path): # ---------- configure_console_encoding ---------- + def test_configure_console_encoding_is_noop_off_windows(): fake_sys = SimpleNamespace(stdout=MagicMock(), stderr=MagicMock()) with patch.object(pc, "IS_WINDOWS", False), patch.object(pc, "sys", fake_sys): @@ -99,5 +110,9 @@ def test_configure_console_encoding_forces_utf8_on_windows(): fake_sys = SimpleNamespace(stdout=MagicMock(), stderr=MagicMock()) with patch.object(pc, "IS_WINDOWS", True), patch.object(pc, "sys", fake_sys): pc.configure_console_encoding() - fake_sys.stdout.reconfigure.assert_called_once_with(encoding="utf-8", errors="replace") - fake_sys.stderr.reconfigure.assert_called_once_with(encoding="utf-8", errors="replace") + fake_sys.stdout.reconfigure.assert_called_once_with( + encoding="utf-8", errors="replace" + ) + fake_sys.stderr.reconfigure.assert_called_once_with( + encoding="utf-8", errors="replace" + ) diff --git a/uv.lock b/uv.lock index bac416b..436cdf7 100644 --- a/uv.lock +++ b/uv.lock @@ -257,6 +257,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, ] +[[package]] +name = "cfgv" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/b5/721b8799b04bf9afe054a3899c6cf4e880fcf8563cc71c15610242490a0c/cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132", size = 7334, upload-time = "2025-11-19T20:55:51.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" }, +] + [[package]] name = "charset-normalizer" version = "3.4.8" @@ -490,8 +499,10 @@ dependencies = [ [package.dev-dependencies] dev = [ + { name = "pre-commit" }, { name = "pytest" }, { name = "python-semantic-release" }, + { name = "ruff" }, ] [package.metadata] @@ -509,8 +520,10 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ + { name = "pre-commit", specifier = ">=4" }, { name = "pytest", specifier = ">=8" }, { name = "python-semantic-release", specifier = ">=10,<11" }, + { name = "ruff", specifier = "==0.15.21" }, ] [[package]] @@ -525,6 +538,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, ] +[[package]] +name = "distlib" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/02/bd72be9134d25ed783ecbbc38a539ffaefbf90c78418c7fb7229600dbac7/distlib-0.4.3.tar.gz", hash = "sha256:f152097224a0ae24be5a0f6bae1b9359af82133bce63f98a95f86cae1aede9ed", size = 615141, upload-time = "2026-06-12T08:04:52.847Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/08/9c41fb51ab5b43eb21674aff13df270e8ba6c4b29c8624e328dc7a9482af/distlib-0.4.3-py2.py3-none-any.whl", hash = "sha256:4b0ce306c966eb73bc3a7b6abad017c556dadd92c44701562cd528ac7fde4d5b", size = 470628, upload-time = "2026-06-12T08:04:50.506Z" }, +] + [[package]] name = "dotty-dict" version = "1.3.1" @@ -552,6 +574,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/a8/20d0723294217e47de6d9e2e40fd4a9d2f7c4b6ef974babd482a59743694/fastjsonschema-2.21.2-py3-none-any.whl", hash = "sha256:1c797122d0a86c5cace2e54bf4e819c36223b552017172f32c5c024a6b77e463", size = 24024, upload-time = "2025-08-14T18:49:34.776Z" }, ] +[[package]] +name = "filelock" +version = "3.32.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c0/80/8232b582c4b318b817cf1274ba74976b07b34d35ef439b3eb948f98645a1/filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402", size = 213757, upload-time = "2026-07-21T13:17:42.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/79/b4c714bef36bc4ec2beeae1e0c124f0223888cd8c6feb1cdc56038116920/filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3", size = 97732, upload-time = "2026-07-21T13:17:41.55Z" }, +] + [[package]] name = "fqdn" version = "1.5.1" @@ -735,6 +766,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "identify" +version = "2.6.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/63/51723b5f116cc04b061cb6f5a561790abf249d25931d515cd375e063e0f4/identify-2.6.19.tar.gz", hash = "sha256:6be5020c38fcb07da56c53733538a3081ea5aa70d36a156f83044bfbf9173842", size = 99567, upload-time = "2026-04-17T18:39:50.265Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/84/d9273cd09688070a6523c4aee4663a8538721b2b755c4962aafae0011e72/identify-2.6.19-py2.py3-none-any.whl", hash = "sha256:20e6a87f786f768c092a721ad107fc9df0eb89347be9396cadf3f4abbd1fb78a", size = 99397, upload-time = "2026-04-17T18:39:49.221Z" }, +] + [[package]] name = "idna" version = "3.18" @@ -1333,6 +1373,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c5/3c/3179b85b0e1c3659f0369940200cd6d0fa900e6cefcc7ea0bc6dd0e29ffb/nest_asyncio2-1.7.2-py3-none-any.whl", hash = "sha256:f5dfa702f3f81f6a03857e9a19e2ba578c0946a4ad417b4c50a24d7ba641fe01", size = 7843, upload-time = "2026-02-13T00:34:02.691Z" }, ] +[[package]] +name = "nodeenv" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, +] + [[package]] name = "notebook" version = "7.6.0" @@ -1635,6 +1684,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "pre-commit" +version = "4.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cfgv" }, + { name = "identify" }, + { name = "nodeenv" }, + { name = "pyyaml" }, + { name = "virtualenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/3a/ddb78f32a0814e66b18a099377a106a2dcdce92d86a034d69d65df9b256e/pre_commit-4.6.1.tar.gz", hash = "sha256:03e809865c7d178b9979d06c761fcbfe6808fdaded8581a745bb110e52050421", size = 198646, upload-time = "2026-07-21T20:56:58.225Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/49/bc925106abcdac498074f2cbe6137e94e09f418dd2b7775df5b577dc0313/pre_commit-4.6.1-py2.py3-none-any.whl", hash = "sha256:0e3b2942510d1fb34eec167a3ec57331bf8442122f1153a9fb8b58f5c49b2717", size = 226186, upload-time = "2026-07-21T20:56:57.064Z" }, +] + [[package]] name = "prometheus-client" version = "0.25.0" @@ -1913,6 +1978,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] +[[package]] +name = "python-discovery" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "platformdirs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f1/51/276f964496a5714ab9f320896195639086881c2b39c03b5ad13de84acbb8/python_discovery-1.5.0.tar.gz", hash = "sha256:3e014c6327154d3dda27939a9a0dc9c5c000439f1906d3f303b48f984bd2ecef", size = 72483, upload-time = "2026-07-21T13:14:14.641Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/7b/14882602ddee241d7984a742fcb423cb4a30fb0d6efc546ac3129fba475a/python_discovery-1.5.0-py3-none-any.whl", hash = "sha256:70c4fc61b4e7404e44f01d6fc44a715c4d685ca6cea83d295922f05891877c98", size = 34205, upload-time = "2026-07-21T13:14:13.398Z" }, +] + [[package]] name = "python-gitlab" version = "8.4.0" @@ -2302,6 +2380,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/85/48f0abdcef5cce4e034c7a5b0ceeceba0b01bf0d942824f4bb720afe2dec/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826", size = 586486, upload-time = "2026-06-30T07:17:51.141Z" }, ] +[[package]] +name = "ruff" +version = "0.15.21" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/36/6f65aa9989acdec45d417192d8f4e7921931d8a6cf87ac74bce3eed98a8e/ruff-0.15.21.tar.gz", hash = "sha256:d0cfc841c572283c36548f82664a54ce6565567f1b0d5b4cf2caac693d8b7500", size = 4769401, upload-time = "2026-07-09T20:01:34.005Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/c6/ede15cac6839f3dbce52565c8f5164a8210e669c7bc4decb03e5bdf47d0d/ruff-0.15.21-py3-none-linux_armv6l.whl", hash = "sha256:63ea0e965e5d73c90e95b2434beeafc70820536717f561b32ab6e777cb9bdf5d", size = 10854342, upload-time = "2026-07-09T20:00:53.998Z" }, + { url = "https://files.pythonhosted.org/packages/28/9d/d825b07ee7ea9e2d61df92a860033c94e06e7300d50a1c2653aac27d24fe/ruff-0.15.21-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0f212c5d7d54c01bbfe6dcab02b724a39300f3e34ed7acbe995ccb320a2c58bd", size = 11139539, upload-time = "2026-07-09T20:00:57.809Z" }, + { url = "https://files.pythonhosted.org/packages/f5/de/3b107712e642f063c7a9e0887c427b22cb44097de5aab36c05f2e280670c/ruff-0.15.21-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e6312e41bc96791299614995ea3a977c5857c3b5662b1ecef6755b02b87cb646", size = 10595437, upload-time = "2026-07-09T20:01:00.006Z" }, + { url = "https://files.pythonhosted.org/packages/9a/6f/b4523cc90ba239ede441447a19d0c968846a3012e5a0b0c5b62831a3d5e3/ruff-0.15.21-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01d65b4831c6b2a4ba8ee6faa84049d44d982b7a706e622c4094c509e51673be", size = 10990053, upload-time = "2026-07-09T20:01:02.187Z" }, + { url = "https://files.pythonhosted.org/packages/92/cc/c6a9872a5375f0628875481cf2f66b13d7d865bf3ca2e57f91c7e762d976/ruff-0.15.21-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2c5a913a589120ce67933d5d05fd6ddbcc2481c6a054980ee767f7414c72b4fd", size = 10666096, upload-time = "2026-07-09T20:01:04.299Z" }, + { url = "https://files.pythonhosted.org/packages/ab/97/c621f7a17e097f1790fa3af6374138823b330b2d03fc38337945daca212c/ruff-0.15.21-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef04b681d02ad4dc9620f00f83ac5c22f652d0e9a9cfe431d219b16ad5ccc41", size = 11537011, upload-time = "2026-07-09T20:01:06.771Z" }, + { url = "https://files.pythonhosted.org/packages/ea/51/d928727e476e25ccc57c6f449ffd80241a651a973ad949d39cfb2a771d28/ruff-0.15.21-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16d090c0740916594157e75b80d666eab8e78083b39b3b0e1d698f4670a17b86", size = 12347101, upload-time = "2026-07-09T20:01:08.859Z" }, + { url = "https://files.pythonhosted.org/packages/1e/88/8cd62026802b16018ad06931d87997cf795ba2a6239ab659606c87d96bf0/ruff-0.15.21-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a10e74757dd65004d779b73e2f3c5210156d9980b41224d50d2ebcf1db51e67", size = 11572001, upload-time = "2026-07-09T20:01:11.092Z" }, + { url = "https://files.pythonhosted.org/packages/b2/97/f63084cf55444fc110e8cb985ebfcc592af47f597d44453d778cb81bc156/ruff-0.15.21-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bab0905d2f29e0d9fbc3c373ed23db0095edaa3f71f1f4f519ec15134d9e85c8", size = 11549239, upload-time = "2026-07-09T20:01:13.27Z" }, + { url = "https://files.pythonhosted.org/packages/9d/77/f107da4a2874b7715914b03f09ba9c54424de3ff8a1cc5d015d3ee2ce0ac/ruff-0.15.21-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:00eca240af5789fec6fe7df74c088cc1f9644ed83027113468efba7c92b94075", size = 11535340, upload-time = "2026-07-09T20:01:15.206Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e9/601deb322d3303a7bf212b0100ead6f2ee3f6a044d89c30f2f92bf83c731/ruff-0.15.21-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:262ab31557a75141325e32d3357f3597645a7f084e732b6b054dde428ecd9341", size = 10964048, upload-time = "2026-07-09T20:01:17.723Z" }, + { url = "https://files.pythonhosted.org/packages/ea/2e/0f2176d1e99c15192caea19c8c3a0a955246b4cb4de795042eeb616345cd/ruff-0.15.21-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:659c4e7a4212f83306045ec7c5e5a356d16d9a6ef4ae0c7a4d872914fc655d9d", size = 10667055, upload-time = "2026-07-09T20:01:19.73Z" }, + { url = "https://files.pythonhosted.org/packages/48/60/abd74a02e0c4214f12a68becfd30af7165cfdcb0e661ecdc60bbb949c09a/ruff-0.15.21-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9e866eab611a5f959d36df2d10e446973a3610bc42b0c15b31dc27977d59c233", size = 11242043, upload-time = "2026-07-09T20:01:21.947Z" }, + { url = "https://files.pythonhosted.org/packages/b2/c6/583075d8ccabb4b229345edcaf1545eb3d8d6be90f686a479d7e94088bbf/ruff-0.15.21-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e89bc93c0d3803ba870b55c29671bad9dc6d94bb1eb181b056b52eb05b52854f", size = 11648064, upload-time = "2026-07-09T20:01:24.023Z" }, + { url = "https://files.pythonhosted.org/packages/3a/3c/37d0ecb729a7cc2d393ea7dce316fc585680f35d93b8d62139d7d0a3700c/ruff-0.15.21-py3-none-win32.whl", hash = "sha256:01f8d5be84823c172b389e123174f781f9daf86d6c58719d603f941932195cdd", size = 10896555, upload-time = "2026-07-09T20:01:26.941Z" }, + { url = "https://files.pythonhosted.org/packages/c0/b8/e43466b2a6067ce91e669068f6e28d6c719a920f014b070d5c8731725de3/ruff-0.15.21-py3-none-win_amd64.whl", hash = "sha256:d4b8d9a2f0f12b816b50447f6eccb9f4bb01a6b82c86b50fb3b5354b458dc6d3", size = 12038772, upload-time = "2026-07-09T20:01:29.497Z" }, + { url = "https://files.pythonhosted.org/packages/dd/75/e90ab9aeece218a9fc5a5bc3ec97d0ee6bb3c4ff95869463c1de58e29a1c/ruff-0.15.21-py3-none-win_arm64.whl", hash = "sha256:6e83115d4b9377c1cbc13abf0e051f069fab0ef815ea0504a8a008cee24dd0a8", size = 11375265, upload-time = "2026-07-09T20:01:31.772Z" }, +] + [[package]] name = "send2trash" version = "2.1.0" @@ -2485,6 +2588,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, ] +[[package]] +name = "virtualenv" +version = "21.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "distlib" }, + { name = "filelock" }, + { name = "platformdirs" }, + { name = "python-discovery" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fe/25/e367a7229b0914772ca8d81b41fde012d9feda68523b52644a571bb21ce8/virtualenv-21.7.0.tar.gz", hash = "sha256:7f9519b9432ff11b6e1a3e94061664efc2ff99ea21780e3cf4f6bd0a5da8b37c", size = 5527510, upload-time = "2026-07-21T13:12:14.109Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/7a/ae29312b1e88a22e81f5d21fc11526d2a114089776c2550d2b205b6c2a47/virtualenv-21.7.0-py3-none-any.whl", hash = "sha256:a8370c1c5530fbabf955e40b8fbbc68a431648b10f9433faa587db30a06e51dd", size = 5507078, upload-time = "2026-07-21T13:12:12.136Z" }, +] + [[package]] name = "wcwidth" version = "0.8.2"