diff --git a/.github/ct.yaml b/.github/ct.yaml index 2222586db..4ee912ee4 100644 --- a/.github/ct.yaml +++ b/.github/ct.yaml @@ -8,7 +8,9 @@ chart-repos: - grafana=https://grafana.github.io/helm-charts - calypr=https://calypr.github.io/helm-charts helm-extra-args: --timeout 600s -check-version-increment: true +# Chart versions are frozen at a placeholder in git and derived from tags at +# release time, so there is no per-PR bump to enforce. +check-version-increment: false debug: false validate-maintainers: false -helm-dependency-extra-args: "--skip-refresh" \ No newline at end of file +helm-dependency-extra-args: "--skip-refresh" diff --git a/.github/scripts/regenerate_local_alloy_values.py b/.github/scripts/regenerate_local_alloy_values.py new file mode 100644 index 000000000..f2037e02d --- /dev/null +++ b/.github/scripts/regenerate_local_alloy_values.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +""" +Regenerate examples/local_alloy_values.yaml from the Alloy chart's own configuration. + +``helm/alloy/values.yaml`` carries Alloy's whole configuration in a single YAML string, +``alloy.alloyConfigmapData``. Helm can only replace such a value wholesale, never merge into it, +so the local-development overlay has to contain a full copy. This script produces that copy so +it stays byte-identical to the chart apart from a fixed set of substitutions: the three write +endpoints, which in the chart point at Mimir, Loki and Tempo hostnames that do not exist on a +laptop; the two external labels, which the chart writes as Go template syntax that +``templates/alloy-config.yaml`` never renders; and a ``loki.process`` stage inserted after the pod +log source. + +That last one is not an address rewrite but an added behaviour, and it is the only place it +exists. It promotes ``trace_id`` to Loki structured metadata and relabels ``service_name`` from the +log line, which is what makes Grafana's trace-to-logs query resolve. The chart has no equivalent, +so moving the stage into ``helm/alloy/values.yaml`` is what would extend correlation to deployed +clusters, and this substitution would then be dropped. + +Run it after any change to the chart's configuration: + + python3 .github/scripts/regenerate_local_alloy_values.py + +It exits non-zero, changing nothing, if the chart no longer contains a line it expects to +rewrite. That means the chart moved and the substitutions below need revisiting - it is the +signal that the overlay would otherwise have drifted silently. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +CHART_VALUES = REPO_ROOT / "helm" / "alloy" / "values.yaml" +OVERLAY = REPO_ROOT / "examples" / "local_alloy_values.yaml" + +# Applied in order to the copied configuration, each exactly once. +SUBSTITUTIONS = [ + ( + ' endpoint = "http://monitoring-tempo-distributor.monitoring:4317"\n', + ' endpoint = "lgtm.monitoring:4317"\n', + ), + ( + # X-Scope-OrgID is Mimir's tenant header; the local Prometheus has no notion of tenants. + ' url = "https://mimir.example.com/api/v1/push"\n' + "\n" + " headers = {\n" + ' "X-Scope-OrgID" = "anonymous",\n' + " }\n" + "\n", + ' url = "http://lgtm.monitoring:9090/api/v1/write"\n', + ), + ( + ' url = "https://loki.example.com/loki/api/v1/push"\n', + ' url = "http://lgtm.monitoring:3100/loki/api/v1/push"\n', + ), + ( + ' loki.source.kubernetes "pods" {\n' + " targets = discovery.relabel.all_pods.output\n" + " forward_to = [loki.write.endpoint.receiver]\n" + " }\n", + ' loki.source.kubernetes "pods" {\n' + " targets = discovery.relabel.all_pods.output\n" + " forward_to = [loki.process.pod_logs.receiver]\n" + " }\n" + "\n" + " // Gen3 services log JSON carrying the OpenTelemetry trace_id. Promoting it to\n" + " // structured metadata is what lets Grafana's trace-to-logs query filter on\n" + ' // `| trace_id = "..."` with no parser stage, which is how the Tempo datasource in\n' + " // the local LGTM image is provisioned. It deliberately does not become a stream\n" + " // label: a per-request value there would multiply Loki's stream cardinality.\n" + " //\n" + " // Lines that are not JSON, such as etcd and kube-proxy output, extract nothing and\n" + " // pass through unchanged.\n" + ' loki.process "pod_logs" {\n' + " forward_to = [loki.write.endpoint.receiver]\n" + "\n" + " stage.json {\n" + ' expressions = { trace_id = "trace_id", span_id = "span_id", otel_service = "service" }\n' + " }\n" + "\n" + " stage.structured_metadata {\n" + ' values = { trace_id = "", span_id = "" }\n' + " }\n" + "\n" + " // Each line reports the service.name its span was recorded under. Using that as\n" + " // the stream label is what keeps logs joinable to traces: Grafana builds its\n" + " // trace-to-logs query from service.name, while Loki would otherwise derive\n" + " // service_name from the pod's `app` label. Those two spellings differ whenever a\n" + " // service names itself with underscores, and the join then silently finds nothing.\n" + " //\n" + " // Lines logged outside a span extract nothing here and keep Loki's derived value.\n" + " stage.labels {\n" + ' values = { service_name = "otel_service" }\n' + " }\n" + " }\n", + ), + # Once per write endpoint, hence the repeated pairs. + (' cluster = "{{ .Values.cluster }}",\n', ' cluster = "local-kind",\n'), + (' project = "{{ .Values.project }}",\n', ' project = "local",\n'), + (' cluster = "{{ .Values.cluster }}",\n', ' cluster = "local-kind",\n'), + (' project = "{{ .Values.project }}",\n', ' project = "local",\n'), +] + +HEADER = """\ +# GENERATED by .github/scripts/regenerate_local_alloy_values.py - re-run that script rather than +# editing alloyConfigmapData below by hand. +# +# Grafana Alloy sized for a kind cluster, writing to the single-pod LGTM stack described in +# docs/local-observability.md. Install with: +# +# helm dependency update helm/alloy +# helm upgrade --install alloy ./helm/alloy -n monitoring -f examples/local_alloy_values.yaml +# +# The release must be named `alloy` in namespace `monitoring`: that is the collector address +# Gen3 service images have baked in as OTEL_EXPORTER_OTLP_ENDPOINT. +alloy: + controller: + type: deployment + replicas: 1 + # helm/alloy/values.yaml pins Alloy to topology.kubernetes.io/zone=us-east-1a. A kind node + # carries no zone label, so without this the pod never leaves Pending. It has to be null + # rather than {}: Helm merges an empty map, which leaves the chart's affinity in place, and + # only an explicit null deletes the key. + affinity: null + + alloy: + stabilityLevel: "public-preview" + uiPathPrefix: /alloy + # Lists replace rather than merge, so the OTLP ports have to be restated here. + extraPorts: + - name: "otel-grpc" + port: 4317 + targetPort: 4317 + protocol: "TCP" + - name: "otel-http" + port: 4318 + targetPort: 4318 + protocol: "TCP" + # A single replica has nobody to gossip with, and the peer lookups are noise in the logs. + clustering: + enabled: false + # The parent chart renders the alloy-gen3 ConfigMap; letting the subchart render one too + # would collide on the name. + configMap: + create: false + name: alloy-gen3 + key: config + resources: + requests: + cpu: 100m + memory: 256Mi + + # Copied from helm/alloy/values.yaml, changing the three write endpoints and the two external + # labels, and inserting the loki.process stage that follows the pod log source. That stage is + # local-only: the chart has no equivalent, so trace-to-logs correlation works here and not in a + # cluster deployed from helm/alloy. + # + # The labels are written out literally on purpose. templates/alloy-config.yaml renders this + # with toYaml rather than tpl, so any {{ }} left in here reaches the ConfigMap unrendered. + alloyConfigmapData: | +""" + +CONFIG_KEY = " alloyConfigmapData: |" + + +def main() -> int: + """ + Write the overlay from the current chart configuration. + + Returns: + int: 0 on success, 1 if the chart no longer matches what the substitutions expect. + """ + lines = CHART_VALUES.read_text().splitlines(keepends=True) + + try: + start = next(i for i, line in enumerate(lines) if line.startswith(CONFIG_KEY)) + except StopIteration: + print(f"{CHART_VALUES} has no '{CONFIG_KEY.strip()}' key", file=sys.stderr) + return 1 + + # alloyConfigmapData is the last key in the file, so the config block runs to the end. + config = "".join(lines[start + 1 :]).rstrip("\n") + "\n" + + for needle, replacement in SUBSTITUTIONS: + if needle not in config: + print(f"chart configuration no longer contains:\n{needle}", file=sys.stderr) + return 1 + config = config.replace(needle, replacement, 1) + + if "{{" in config: + print("template syntax left in the copied configuration", file=sys.stderr) + return 1 + + OVERLAY.write_text(HEADER + config) + print(f"wrote {OVERLAY} ({len(config.splitlines())} configuration lines)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/release.py b/.github/scripts/release.py new file mode 100755 index 000000000..58a675ea8 --- /dev/null +++ b/.github/scripts/release.py @@ -0,0 +1,539 @@ +#!/usr/bin/env python3 +"""Compute, stamp and publish Helm chart versions. + +Chart versions are frozen at a placeholder (0.0.0) in git. The real version is +derived from the existing ``-X.Y.Z`` git tags at release time and stamped +into a throwaway working tree just before packaging, so nothing is hardcoded in +the repo and nothing has to be bumped in a PR. + +Subcommands: + + plan --base SHA --head SHA compute the publish set; no side effects + stamp --plan plan.json rewrite Chart.yaml versions in the worktree + publish --plan plan.json package charts and upload releases + index + +``plan`` is pure so that PR CI can run the exact same code path as the release +job to preview what a merge would publish. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import subprocess +import sys +from collections import defaultdict, deque +from pathlib import Path + +import yaml + +HELM_DIR = Path("helm") +PLACEHOLDER = "0.0.0" +BASE_VERSION = "0.1.0" +FILE_PREFIX = "file://" + +# Anchored on exactly three numeric components so that the chart name is +# captured in full. Keying on the full name is what keeps `gen3-workflow-0.1.29` +# from being read as a `gen3` tag -- a `gen3-*` glob matches 7 different charts. +TAG_RE = re.compile(r"^(?P.+)-(?P\d+\.\d+\.\d+)$") + + +# -------------------------------------------------------------------------- +# git helpers +# -------------------------------------------------------------------------- + + +def git(*args: str, check: bool = True) -> str: + proc = subprocess.run( + ["git", *args], capture_output=True, text=True, check=False + ) + if check and proc.returncode != 0: + raise SystemExit(f"git {' '.join(args)} failed: {proc.stderr.strip()}") + return proc.stdout.strip() + + +def rev_exists(rev: str) -> bool: + return ( + subprocess.run( + ["git", "cat-file", "-e", f"{rev}^{{commit}}"], + capture_output=True, + ).returncode + == 0 + ) + + +def is_ancestor(a: str, b: str) -> bool: + return ( + subprocess.run( + ["git", "merge-base", "--is-ancestor", a, b], capture_output=True + ).returncode + == 0 + ) + + +def resolve_base(before: str | None, head: str) -> str | None: + """Pick the ref to diff against for a push event. + + Returns None when no trustworthy base exists. Callers must treat that as a + hard error rather than silently falling back to "publish everything" -- an + accidental ~50-chart release burst is far worse than a skipped run, and a + skipped run self-heals on the next merge. + """ + if not before or set(before) == {"0"}: + return None # branch creation / initial push + if not rev_exists(before): + return None # garbage collected after a force-push + if not is_ancestor(before, head): + # Force-push: `before..head` would silently under-report. + merge_base = git("merge-base", before, head, check=False) + if merge_base: + return merge_base + parent = git("rev-parse", f"{head}^1", check=False) + return parent or None + return before + + +def changed_paths(base: str, head: str) -> list[str]: + out = git("diff", "--name-only", f"{base}..{head}") + return [line for line in out.splitlines() if line.strip()] + + +# -------------------------------------------------------------------------- +# chart discovery and dependency graph +# -------------------------------------------------------------------------- + + +def load_charts() -> dict[str, dict]: + """Map chart *name* -> {dir, meta}. + + Keyed on the declared ``name:``, never the directory: helm/observability + declares ``name: lgtma-chart`` and owns the ``lgtma-chart-*`` tag line. + """ + charts: dict[str, dict] = {} + for chart_yaml in sorted(HELM_DIR.glob("*/Chart.yaml")): + meta = yaml.safe_load(chart_yaml.read_text()) or {} + name = meta.get("name") + if not name: + raise SystemExit(f"{chart_yaml} has no name:") + if name in charts: + # Turns a silent clobber (two charts publishing under one name, + # last writer wins) into a loud failure. + raise SystemExit( + f"duplicate chart name {name!r}: " + f"{charts[name]['dir']} and {chart_yaml.parent}" + ) + charts[name] = {"dir": chart_yaml.parent, "meta": meta} + return charts + + +def local_dep_target(dep: dict, chart_dir: Path, charts: dict[str, dict]) -> str | None: + """Resolve a file:// dependency to a chart name, by path. + + The path is authoritative; the dep's own ``name:`` field is only checked for + consistency. + """ + repo = str(dep.get("repository") or "") + if not repo.startswith(FILE_PREFIX): + return None + target_dir = (chart_dir / repo[len(FILE_PREFIX) :]).resolve() + for name, chart in charts.items(): + if chart["dir"].resolve() == target_dir: + declared = dep.get("name") + if declared and declared != name: + raise SystemExit( + f"{chart_dir}/Chart.yaml declares dependency name " + f"{declared!r} but {repo} resolves to chart {name!r}" + ) + return name + raise SystemExit(f"{chart_dir}/Chart.yaml: {repo} does not resolve to a chart") + + +def reverse_deps(charts: dict[str, dict]) -> dict[str, set[str]]: + """dep name -> set of charts that depend on it (file:// edges only).""" + rdeps: dict[str, set[str]] = defaultdict(set) + for name, chart in charts.items(): + for dep in chart["meta"].get("dependencies") or []: + target = local_dep_target(dep, chart["dir"], charts) + if target: + rdeps[target].add(name) + return rdeps + + +def transitive_closure(seeds: set[str], rdeps: dict[str, set[str]]) -> set[str]: + seen = set(seeds) + queue = deque(seeds) + while queue: + for parent in rdeps.get(queue.popleft(), ()): + if parent not in seen: + seen.add(parent) + queue.append(parent) + return seen + + +def charts_from_paths(paths: list[str], charts: dict[str, dict]) -> set[str]: + """Map changed file paths to chart names via their helm// component.""" + by_dir = {chart["dir"].name: name for name, chart in charts.items()} + touched: set[str] = set() + for path in paths: + parts = Path(path).parts + if len(parts) < 3 or parts[0] != HELM_DIR.name: + continue + # helm//charts/** is vendored build output, not a source change. + if len(parts) > 2 and parts[2] == "charts": + continue + name = by_dir.get(parts[1]) + if name: + touched.add(name) + return touched + + +# -------------------------------------------------------------------------- +# version derivation +# -------------------------------------------------------------------------- + + +def version_key(version: str) -> tuple[int, ...]: + return tuple(int(part) for part in version.split(".")) + + +def tags_by_chart() -> dict[str, list[str]]: + index: dict[str, list[str]] = defaultdict(list) + for tag in git("tag", "--list").splitlines(): + match = TAG_RE.match(tag.strip()) + if match: + index[match.group("name")].append(match.group("version")) + return index + + +def release_exists(tag: str) -> bool: + """Whether a GitHub release already exists for this tag. + + Only an extra guard against re-using a version after a partially failed + run; the tag list is the primary source. Treats an unavailable gh CLI or + missing token as "no release" rather than failing the run. + """ + if not os.environ.get("GITHUB_TOKEN") and not os.environ.get("CR_TOKEN"): + return False + if shutil.which("gh") is None: + return False + return ( + subprocess.run( + ["gh", "release", "view", tag], capture_output=True + ).returncode + == 0 + ) + + +def next_version(name: str, index: dict[str, list[str]], check_releases: bool = False) -> str: + versions = index.get(name) + if not versions: + return BASE_VERSION + # Semantic, not lexical: sorted() would pick fence-0.1.9 over fence-0.1.82. + highest = max(versions, key=version_key) + major, minor, patch = version_key(highest) + candidate = f"{major}.{minor}.{patch + 1}" + if check_releases: + # A prior run may have created the release but died before tagging. + while release_exists(f"{name}-{candidate}"): + patch += 1 + candidate = f"{major}.{minor}.{patch + 1}" + if candidate == PLACEHOLDER: + raise SystemExit(f"refusing to publish placeholder version for {name}") + return candidate + + +# -------------------------------------------------------------------------- +# stamping +# -------------------------------------------------------------------------- + + +def stamp_version(chart_yaml: Path, version: str) -> None: + """Rewrite the top-level version: line, preserving everything else. + + Dependency versions are indented, so anchoring at column 0 is unambiguous. + A line-oriented edit keeps the explanatory comment blocks intact. + """ + text = chart_yaml.read_text() + new_text, count = re.subn( + r"^version:.*$", f"version: {version}", text, count=1, flags=re.M + ) + if count != 1: + raise SystemExit(f"{chart_yaml}: expected exactly one top-level version:") + chart_yaml.write_text(new_text) + + +def stamp_dependencies(chart_yaml: Path, versions: dict[str, str], charts: dict[str, dict]) -> None: + """Point each file:// dependency at the version we are about to publish. + + Not required for resolution -- Chart.lock and the vendored subchart copies + already carry concrete versions -- but `helm show chart` is what consumers + read, and "*" tells them nothing. + """ + meta = yaml.safe_load(chart_yaml.read_text()) or {} + deps = meta.get("dependencies") or [] + if not deps: + return + chart_dir = chart_yaml.parent + lines = chart_yaml.read_text().splitlines(keepends=True) + + # Walk the dependency list textually so comments and key order survive. + current: str | None = None + for i, line in enumerate(lines): + name_match = re.match(r"^\s*-\s+name:\s*(\S+)", line) + if name_match: + current = name_match.group(1) + continue + version_match = re.match(r"^(\s+version:\s*)(\S+)(.*)$", line) + if version_match and current: + dep = next((d for d in deps if d.get("name") == current), None) + if dep is None: + continue + target = local_dep_target(dep, chart_dir, charts) + if target and target in versions: + lines[i] = f"{version_match.group(1)}{versions[target]}\n" + current = None + chart_yaml.write_text("".join(lines)) + + +# -------------------------------------------------------------------------- +# subcommands +# -------------------------------------------------------------------------- + + +def build_plan(base: str | None, head: str, all_charts: bool) -> dict: + charts = load_charts() + rdeps = reverse_deps(charts) + + if all_charts: + selected = set(charts) + directly = selected + else: + if base is None: + raise SystemExit( + "could not determine a trustworthy base commit (initial push, " + "force-push, or gc'd ref). Re-run via workflow_dispatch with an " + "explicit --base, or pass --all deliberately." + ) + directly = charts_from_paths(changed_paths(base, head), charts) + selected = transitive_closure(directly, rdeps) + + index = tags_by_chart() + entries = [] + for name in sorted(selected): + entries.append( + { + "name": name, + "dir": str(charts[name]["dir"]), + "version": next_version(name, index), + "previous": max(index.get(name, ["-"]), key=lambda v: version_key(v)) + if index.get(name) + else None, + "direct": name in directly, + } + ) + + # Charts that are not being released but get vendored into one that is. + # They still need a real version stamped in: an umbrella packaged with + # unstamped subcharts ships them at the 0.0.0 placeholder, and its own + # dependency block keeps the "*" constraint. Their current version is the + # newest existing tag -- they have not changed, so nothing is incremented. + vendored = [] + for name in sorted(set(charts) - selected): + versions = index.get(name) + vendored.append( + { + "name": name, + "dir": str(charts[name]["dir"]), + "version": max(versions, key=version_key) + if versions + else BASE_VERSION, + } + ) + return {"base": base, "head": head, "charts": entries, "vendored": vendored} + + +def cmd_plan(args: argparse.Namespace) -> int: + base = args.base + if base is None and not args.all: + base = resolve_base(os.environ.get("GITHUB_EVENT_BEFORE"), args.head) + plan = build_plan(base, args.head, args.all) + + if args.markdown: + entries = plan["charts"] + if not entries: + print("## Chart releases\n\nNo charts would be published by this change.") + return 0 + cascaded = sum(1 for e in entries if not e["direct"]) + print(f"## Chart releases ({len(entries)} charts)\n") + if cascaded: + print( + f"{len(entries) - cascaded} directly changed, " + f"{cascaded} cascaded via dependencies.\n" + ) + print("| Chart | Current | Would publish | Reason |") + print("| --- | --- | --- | --- |") + for e in entries: + reason = "changed" if e["direct"] else "depends on a changed chart" + print(f"| {e['name']} | {e['previous'] or '-'} | {e['version']} | {reason} |") + else: + print(json.dumps(plan, indent=2)) + + if args.output: + Path(args.output).write_text(json.dumps(plan, indent=2)) + return 0 + + +def cmd_stamp(args: argparse.Namespace) -> int: + plan = json.loads(Path(args.plan).read_text()) + charts = load_charts() + # Stamp released and merely-vendored charts alike. A released umbrella + # vendors its whole dependency tree, so any subchart left at the + # placeholder would ship inside it as 0.0.0. + everything = plan["charts"] + plan.get("vendored", []) + versions = {e["name"]: e["version"] for e in everything} + + for entry in everything: + stamp_version(Path(entry["dir"]) / "Chart.yaml", entry["version"]) + # Second pass, once every version is known: rewrite the "*" constraints so + # the published Chart.yaml records concrete versions. + for entry in everything: + stamp_dependencies(Path(entry["dir"]) / "Chart.yaml", versions, charts) + + for entry in plan["charts"]: + print(f"stamped {entry['name']} -> {entry['version']} (releasing)") + print(f"stamped {len(plan.get('vendored', []))} more charts at their current version") + return 0 + + +def count_index_versions( + pages_branch: str, index_path: str, fetch: bool = False +) -> int | None: + """Total chart versions listed in the published index.yaml. + + Used to assert the index never shrinks: it carries ~1700 versions, and + replacing rather than merging it would break every consumer pinned to an + older release. Returns None if the index can't be read, so a missing branch + doesn't fail the run on its own. + """ + if fetch: + subprocess.run( + ["git", "fetch", "origin", pages_branch], capture_output=True + ) + for ref in (f"origin/{pages_branch}", pages_branch): + raw = git("show", f"{ref}:{index_path}", check=False) + if raw: + entries = (yaml.safe_load(raw) or {}).get("entries") or {} + return sum(len(v or []) for v in entries.values()) + return None + + +def clean_vendored(chart_dir: Path) -> None: + """Remove build artifacts so each package is resolved from a clean slate. + + Both the chart's own charts/ dir and any nested ones left by a sibling's + earlier `dependency update`. Chart.lock goes too, since a stale lock makes + `dependency build` resolve the wrong versions. + """ + for path in [chart_dir / "charts", *chart_dir.glob("charts/*/charts")]: + if path.is_dir(): + shutil.rmtree(path) + lock = chart_dir / "Chart.lock" + if lock.exists(): + lock.unlink() + + +def cmd_publish(args: argparse.Namespace) -> int: + plan = json.loads(Path(args.plan).read_text()) + packages = Path(args.packages) + packages.mkdir(parents=True, exist_ok=True) + + for entry in plan["charts"]: + chart_dir = Path(entry["dir"]) + # Resolve dependencies immediately before packaging this chart, and + # clean first. Leftover charts/ dirs from a previous chart's resolution + # get vendored a second level deep (e.g. gen3/charts/fence/charts/common), + # and the nested copy shadows the parent's global values at render time. + clean_vendored(chart_dir) + # Always `update`: despite the !helm/funnel/charts gitignore exception, + # no helm/*/charts/* files are actually tracked, so there are no + # committed tarballs to preserve and the lock file may be stale. + subprocess.run(["helm", "dependency", "update", str(chart_dir)], check=True) + subprocess.run( + ["helm", "package", str(chart_dir), "-d", str(packages)], check=True + ) + # Don't leave this chart's vendored deps behind for the next one. + clean_vendored(chart_dir) + print(f"packaged {entry['name']}-{entry['version']}") + + if args.package_only: + return 0 + + owner, repo = args.repo.split("/", 1) + subprocess.run( + [ + "cr", "upload", + "-o", owner, + "-r", repo, + "--package-path", str(packages), + "--skip-existing", + "--make-release-latest=false", + ], + check=True, + ) + before = count_index_versions(args.pages_branch, args.index_path) + subprocess.run( + [ + "cr", "index", + "-o", owner, + "-r", repo, + "--package-path", str(packages), + "--index-path", args.index_path, + "--pages-branch", args.pages_branch, + "--push", + ], + check=True, + ) + after = count_index_versions(args.pages_branch, args.index_path, fetch=True) + if before is not None and after is not None and after < before: + raise SystemExit( + f"index shrank {before} -> {after} versions; the published repo has " + "lost entries and consumers pinning old versions will break" + ) + print(f"index versions: {before} -> {after}") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("plan", help="compute the publish set (no side effects)") + p.add_argument("--base") + p.add_argument("--head", default="HEAD") + p.add_argument("--all", action="store_true", help="select every chart") + p.add_argument("--markdown", action="store_true") + p.add_argument("--output") + p.set_defaults(func=cmd_plan) + + s = sub.add_parser("stamp", help="write real versions into the working tree") + s.add_argument("--plan", required=True) + s.set_defaults(func=cmd_stamp) + + u = sub.add_parser("publish", help="package and upload") + u.add_argument("--plan", required=True) + u.add_argument("--repo", default="uc-cdis/gen3-helm") + u.add_argument("--packages", default=".cr-release-packages") + u.add_argument("--index-path", default="index.yaml") + u.add_argument("--pages-branch", default="gh-pages") + u.add_argument("--package-only", action="store_true") + u.set_defaults(func=cmd_publish) + + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/integration_tests_on_kind.yaml b/.github/workflows/integration_tests_on_kind.yaml index 943299ca4..f81b9b03d 100644 --- a/.github/workflows/integration_tests_on_kind.yaml +++ b/.github/workflows/integration_tests_on_kind.yaml @@ -6,6 +6,7 @@ on: - .github/workflows/integration_tests_on_kind.yaml - helm/funnel/** - helm/gen3-workflow/** + - helm/revproxy/gen3.nginx.conf/gen3-workflow-service.conf permissions: id-token: write @@ -19,6 +20,7 @@ jobs: uses: uc-cdis/.github/.github/workflows/integration_tests.yaml@master with: EXTERNAL_TO_CTDS: "true" + HELM_BRANCH: ${{ github.event.pull_request.head.ref }} SERVICE_TO_TEST: gen3_workflow SETUP_SCRIPT_1: https://raw.githubusercontent.com/uc-cdis/gen3-workflow/refs/heads/master/.github/workflows/integration_tests_on_kind/ci_start_kind_cluster.sh SETUP_SCRIPT_2: https://raw.githubusercontent.com/uc-cdis/gen3-workflow/refs/heads/master/.github/workflows/integration_tests_on_kind/ci_override_config_and_start_minio.sh diff --git a/.github/workflows/lint_test.yaml b/.github/workflows/lint_test.yaml index b89d8c7ca..3df233a81 100644 --- a/.github/workflows/lint_test.yaml +++ b/.github/workflows/lint_test.yaml @@ -29,9 +29,19 @@ jobs: run: | changed=$(ct list-changed --config .github/ct.yaml) if [[ -n "$changed" ]]; then - echo "changed=true >> $GITHUB_OUTPUT" + echo "changed=true" >> "$GITHUB_OUTPUT" fi + # Chart versions are frozen in git, so show what merging this PR would + # actually publish -- including charts pulled in by the dependency + # cascade (a change to common fans out to everything that depends on it). + - name: Preview chart releases + run: | + python3 .github/scripts/release.py plan \ + --base "${{ github.event.pull_request.base.sha }}" \ + --head "${{ github.event.pull_request.head.sha }}" \ + --markdown >> "$GITHUB_STEP_SUMMARY" + - name: Run chart-testing (lint) run: ct lint --config .github/ct.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f4391ce5e..29da233ed 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -4,6 +4,23 @@ on: push: branches: - master + workflow_dispatch: + inputs: + base: + description: "Commit to diff against (recovery for a missed or force-pushed run)" + required: false + type: string + all: + description: "Release every chart, ignoring the diff" + required: false + type: boolean + default: false + +# Never run two releases at once: they would race on gh-pages, and cancelling +# mid-run can leave GitHub Releases that the index never learns about. +concurrency: + group: release-charts + cancel-in-progress: false jobs: release: @@ -12,7 +29,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -20,19 +37,70 @@ jobs: run: | git config user.name "$GITHUB_ACTOR" git config user.email "$GITHUB_ACTOR@users.noreply.github.com" - + - name: Install Helm uses: azure/setup-helm@v3 with: version: v3.10.0 - + - name: Add helm repositories run: | helm repo add bitnami https://charts.bitnami.com/bitnami + helm repo add elastic https://helm.elastic.co + helm repo add grafana https://grafana.github.io/helm-charts + helm repo add calypr https://calypr.github.io/helm-charts + helm repo update - - name: Run chart-releaser - uses: helm/chart-releaser-action@v1.4.1 - with: - charts_dir: helm + - name: Install chart-releaser + run: | + CR_VERSION=1.6.1 + curl -sSLo cr.tar.gz \ + "https://github.com/helm/chart-releaser/releases/download/v${CR_VERSION}/chart-releaser_${CR_VERSION}_linux_amd64.tar.gz" + tar -xzf cr.tar.gz cr + sudo mv cr /usr/local/bin/cr + rm cr.tar.gz + cr version + + # Versions live in git tags, not in Chart.yaml. Work out which charts this + # push touched (plus everything that depends on them) and what version + # each should be published at. + - name: Compute release plan env: - CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_EVENT_BEFORE: ${{ github.event.before }} + run: | + python3 .github/scripts/release.py plan \ + --head "${{ github.sha }}" \ + ${{ inputs.base && format('--base {0}', inputs.base) || '' }} \ + ${{ inputs.all && '--all' || '' }} \ + --output plan.json + python3 .github/scripts/release.py plan \ + --head "${{ github.sha }}" \ + ${{ inputs.base && format('--base {0}', inputs.base) || '' }} \ + ${{ inputs.all && '--all' || '' }} \ + --markdown >> "$GITHUB_STEP_SUMMARY" + + - name: Check whether anything needs releasing + id: check + run: | + count=$(jq '.charts | length' plan.json) + echo "count=$count" >> "$GITHUB_OUTPUT" + echo "Charts to release: $count" + + # Stamp the computed versions into the working tree. This is never + # committed -- Chart.yaml stays at the placeholder in git. + - name: Stamp versions + if: steps.check.outputs.count != '0' + run: python3 .github/scripts/release.py stamp --plan plan.json + + # Releases first, index last: a release with no index entry is invisible + # but harmless and self-heals, whereas an index entry with no release is a + # broken download link. + - name: Package and publish + if: steps.check.outputs.count != '0' + env: + CR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python3 .github/scripts/release.py publish \ + --plan plan.json \ + --repo "${{ github.repository }}" diff --git a/.gitignore b/.gitignore index 9c4dbf3ce..12c0efce0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ _sample-*/ !helm/funnel/charts helm/funnel/charts/common* *copy.yaml + +# Python bytecode +__pycache__/ +*.pyc diff --git a/.helm-docs/README.md.gotmpl b/.helm-docs/README.md.gotmpl new file mode 100644 index 000000000..5028432b2 --- /dev/null +++ b/.helm-docs/README.md.gotmpl @@ -0,0 +1,33 @@ +{{ template "chart.header" . }} + +{{/* + Chart versions are frozen at a placeholder in git and derived from git tags + when the chart is published, so the version badge and the dependency version + column would only ever show "0.0.0" and "*". They are omitted here rather + than rendered misleadingly -- see the releases page for published versions. +*/}} +![Type: {{ .Type }}](https://img.shields.io/badge/Type-{{ .Type }}-informational?style=flat-square) {{ if .AppVersion }}![AppVersion: {{ .AppVersion }}](https://img.shields.io/badge/AppVersion-{{ .AppVersion | replace "-" "--" }}-informational?style=flat-square){{ end }} + +{{ template "chart.description" . }} + +{{ template "chart.homepageLine" . }} + +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ if .Dependencies }} +## Requirements + +| Repository | Name | +|------------|------| +{{- range .Dependencies }} +| {{ .Repository }} | {{ .Name }} | +{{- end }} +{{ end }} + +{{ template "chart.valuesSection" . }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e6605717f..b3a251200 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,6 +16,7 @@ repos: # rev: v0.1.17 # Get the latest from: https://github.com/gruntwork-io/pre-commit/releases # hooks: # - id: helmlint + # - repo: https://github.com/norwoodj/helm-docs rev: "v1.14.2" @@ -27,21 +28,9 @@ repos: args: - "--chart-search-root=helm" - "--skip-version-footer" - - - repo: local - hooks: - - id: helm-chart-bump - args: [] - description: Updates the .Chart.yaml with updates version if there are changes since master branch. This is to ensure we bump our charts for updates. - entry: git-hook/helm-bump.sh - language: script - name: Helm Docs - require_serial: true - - - repo: local - hooks: - - id: funnel-chart-check - name: Funnel chart check - entry: git-hook/funnel-chart-check.sh - language: script - pass_filenames: false + # Shared template: omits the version badge and dependency version + # column, which are frozen placeholders in git rather than real values. + # Absolute path: pre-commit's docker_image language mounts the repo at + # /src, and helm-docs only resolves this flag as an absolute path or + # relative to each chart dir. + - "--template-files=/src/.helm-docs/README.md.gotmpl" \ No newline at end of file diff --git a/.secrets.baseline b/.secrets.baseline index 72eea1827..077d8ef79 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -137,14 +137,14 @@ "filename": "helm/portal/README.md", "hashed_secret": "eb9739c6625f06b4ab73035223366dda6262ae77", "is_verified": false, - "line_number": 37 + "line_number": 41 }, { "type": "Base64 High Entropy String", "filename": "helm/portal/README.md", "hashed_secret": "08eeb737b239bdb7362a875b90e22c10b8826b20", "is_verified": false, - "line_number": 42 + "line_number": 46 } ], "helm/portal/values.yaml": [ @@ -173,5 +173,5 @@ } ] }, - "generated_at": "2026-06-04T14:01:16Z" + "generated_at": "2026-08-10T16:27:55Z" } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c81ac0d39..09de74214 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,29 +58,50 @@ Each chart has its own README.md that is automatically built with [helm-docs](ht ## Helm chart release strategy -It is important to understand that when a branch is merged into the main branch, a GitHub action will generate a new helm chart release if the helm chart version in the chart.yaml file has been incremented. Consider the following example where a change to the Helm chart has been made and the contributor wants a new version to be released: - -The original Chart.yaml file: +**You do not need to bump any chart versions.** Every `Chart.yaml` in this repo +is frozen at the placeholder `version: 0.0.0`, and dependencies between charts +in this repo are declared as `version: "*"`: ```yaml apiVersion: v2 - name: Sheepdog + name: sheepdog description: A Helm chart for Kubernetes type: application - version: 0.1.0 + version: 0.0.0 # placeholder -- never edit this + dependencies: + - name: common + version: "*" # resolved locally, stamped at release time + repository: file://../common ``` -If a modification to the Helm chart is made (an update to the values.yaml file for instance) the version in Chart.yaml is incremented to `0.2.0`: +The real version is worked out when the chart is published. On a merge to +`master`, the release workflow: - ```yaml - apiVersion: v2 - name: Sheepdog - description: A Helm chart for Kubernetes - type: application - version: 0.2.0 # version updates to 0.2.0 - ``` +1. Diffs the merge to find which charts changed. +2. Adds every chart that depends on a changed chart -- so a change to `common` + republishes everything that uses it, and any subchart change republishes the + `gen3` umbrella. +3. Looks up the highest existing `-X.Y.Z` git tag for each of those + charts and increments the patch number. +4. Stamps those versions into `Chart.yaml` (in the CI working tree only -- this + is never committed), packages, and publishes. + +So if `sheepdog-0.1.47` is the latest tag and you change something under +`helm/sheepdog/`, merging produces `sheepdog-0.1.48` plus a new `gen3` release. +Nothing in the repo records that number. + +To see exactly what your PR would publish, check the **job summary** on the +"Lint and Test Charts" run -- it lists every chart that would be released, the +version it would get, and whether it was pulled in directly or by the +dependency cascade. + +Two consequences worth knowing: -Once the associated branch is merged into the main branch, the GitHub action packages and publishes an artifact, making it available for consumption. The release name is based off the 'name' field and the 'version' field in the Chart.yaml file. Given the example above, GitHub action will produce a release called `sheepdog-0.2.0`. +- A one-line change to `helm/common/` republishes ~43 charts. That is + intentional -- previously those dependents were silently left unpublished. +- Chart READMEs no longer show a version badge, since the in-repo version is + always the placeholder. Published versions are listed at + and on the [releases page](https://github.com/uc-cdis/gen3-helm/releases). ## Branch Naming Conventions @@ -114,7 +135,7 @@ Before submitting a PR for review, try to make sure you’ve accomplished these The PR: - contains a brief description of what it changes and/or adds - passes status checks -- If there are changes to the charts, it bumps the chart versions +- If there are changes to the charts, the release preview in the job summary looks right (chart versions are derived at release time -- do not bump them by hand) To merge the PR: diff --git a/docs/kubernetes-in-docker.md b/docs/kubernetes-in-docker.md index 91707a20c..8d6f35d6c 100644 --- a/docs/kubernetes-in-docker.md +++ b/docs/kubernetes-in-docker.md @@ -112,4 +112,8 @@ portal: ```bash helm repo add gen3 http://helm.gen3.org helm upgrade --install gen3 gen3/gen3 -f ./values.yaml -``` \ No newline at end of file +``` + +# Optional: metrics, traces, and profiles + +To see the metrics, traces, logs, and profiles your services emit, see [local-observability.md](local-observability.md). \ No newline at end of file diff --git a/docs/local-observability.md b/docs/local-observability.md new file mode 100644 index 000000000..74774b93d --- /dev/null +++ b/docs/local-observability.md @@ -0,0 +1,200 @@ +# Local observability on kind + +## Overview + +Gen3 services emit four kinds of telemetry: Prometheus metrics scraped from a `/metrics` +endpoint, logs written to stdout, OpenTelemetry traces pushed over OTLP, and continuous profiles +pushed to Pyroscope. In a deployed cluster the first three flow through +[Grafana Alloy](../helm/alloy/SETUP.md), which forwards metrics to Mimir, logs to Loki, and traces +to Tempo. Profiles take no such detour - the SDK inside each service pushes them straight to +Pyroscope, so nothing in Alloy's configuration is involved in carrying them. + +This guide stands the same Alloy pipeline up on a kind cluster, backed by a single-pod LGTM +stack instead of the [observability](../helm/observability/SETUP.md) chart. You get Grafana, +Prometheus, Tempo, Loki, and Pyroscope in one container, and Alloy configured as it is in a real +cluster apart from the three addresses it writes to and one added log-processing stage. + +That stage is the one deliberate behavioural difference, and it matters when comparing against a +deployed cluster: the overlay promotes each log line's `trace_id` to Loki structured metadata and +takes `service_name` from the line itself, which is what makes Grafana's trace-to-logs link +resolve. `helm/alloy/values.yaml` carries no such stage, so a deployed cluster using that chart +correlates traces to logs only once the change described in +[otel-logs-and-traces.md](otel-logs-and-traces.md) lands there. + +Use this when you are developing a service and want to see its own metrics, traces, logs, and +profiles. Do not use it as a model for a deployed cluster: + +- one replica of everything, no high availability +- `emptyDir` storage, so all data is lost when the pod restarts +- no ingress, no TLS, no authentication beyond Grafana's default `admin` / `admin` + +The `observability` chart is the deployed-cluster answer. It is sized for EKS - five Mimir +ingesters, S3 storage, ALB ingresses - and will not fit comfortably on a laptop. It also deploys +neither Tempo nor Pyroscope, so a deployed cluster gets traces and profiles only from backends +outside that chart; see [otel-logs-and-traces.md](otel-logs-and-traces.md). + +## Prerequisites + +A running kind cluster. See [kubernetes-in-docker.md](kubernetes-in-docker.md). + +# Step 1. Deploy the LGTM backend + +[examples/local_lgtm.yaml](../examples/local_lgtm.yaml) is adapted from the manifest published +by [grafana/docker-otel-lgtm](https://github.com/grafana/docker-otel-lgtm), with two changes: the +Loki port is exposed, so Alloy has somewhere to send logs, and the Pyroscope port is exposed, so +services can push profiles. Both listen inside the image already; only the Service was missing +them, and a Service without the port silently drops the traffic rather than refusing it. The +image already starts Prometheus with `--web.enable-remote-write-receiver`, which is what Alloy +needs in order to deliver metrics, so nothing has to be passed to enable it. + +```bash +kubectl apply -f examples/local_lgtm.yaml + +kubectl wait --namespace monitoring \ + --for=condition=ready pod \ + --selector=app=lgtm \ + --timeout=180s +``` + +Grafana comes with Prometheus, Tempo, Loki, and Pyroscope datasources already provisioned, so +there is nothing to wire up by hand. + +# Step 2. Deploy Alloy + +[examples/local_alloy_values.yaml](../examples/local_alloy_values.yaml) is the stock Alloy +configuration with its three write endpoints pointed at the pod from step 1. It also clears the +`us-east-1a` node affinity the chart applies by default, which no kind node satisfies. + +```bash +helm dependency update helm/alloy +helm upgrade --install alloy ./helm/alloy -n monitoring -f examples/local_alloy_values.yaml +``` + +The release has to be named `alloy` and live in `monitoring`. Gen3 service images ship with +`OTEL_EXPORTER_OTLP_ENDPOINT=http://alloy.monitoring:4318` baked in, and that address is +`.`. + +Confirm Alloy came up clean: + +```bash +kubectl -n monitoring logs deploy/alloy | grep '"level":"error"' +``` + +Expect `"level":"warn"` lines reading `tailer stopped; will retry` for any container that has not +started yet. `loki.source.kubernetes` gets a target per declared container as soon as the pod object +exists, so it retries on a backoff until the container runs. A pod stuck in +`Init:CreateContainerConfigError` or `ImagePullBackOff` produces these indefinitely without +affecting collection from healthy pods. + +# Step 3. Point a service at it + +**Metrics need nothing.** Every Gen3 chart stamps `prometheus.io/scrape: "true"` and +`prometheus.io/path: /metrics` onto its pods, controlled by `global.metricsEnabled` (default +`true`). Alloy discovers pods by those annotations and scrapes `:/metrics`. +The only requirement is that your service actually serves `/metrics`. + +**Traces need three values**, and only if you want to override what the image already does. For +`gen3-embeddings`: + +```yaml +gen3-embeddings: + otel: + enabled: true + endpoint: "http://alloy.monitoring:4318" + protocol: "http/protobuf" +``` + +`endpoint` and `protocol` have to change together: Alloy listens for `http/protobuf` on 4318 and +for `grpc` on 4317, and a mismatched pair fails when the first span is exported rather than at +startup. + +**Profiles need one address.** A service that ships a Pyroscope SDK likely pushes to +`PYROSCOPE_SERVER_ADDRESS`, which is something like: + +``` +PYROSCOPE_SERVER_ADDRESS=http://lgtm.monitoring:4040 +``` + +> NOTE: Check the individual service config for how to enable and configure observability. We are trying to consolidate Python observability into one of our Python packages that we import and use in the services, but there may be some differences across services. + +Services deployed in another namespace reach Alloy fine - `alloy.monitoring` resolves from +anywhere in the cluster, as does `lgtm.monitoring`. + +# Step 4. Look at the data + +```bash +kubectl port-forward -n monitoring svc/lgtm 3000:3000 # Grafana, admin / admin +kubectl port-forward -n monitoring svc/alloy 12345:12345 # Alloy UI, at /alloy +``` + +In Grafana, Explore against the Prometheus datasource for metrics, the Tempo datasource for +traces, the Loki datasource for logs, and the Pyroscope datasource for profiles. Traces are +searchable by `service.name`; logs are selected by `service_name`, for example +`{service_name="gen3_embeddings"}`. Profiles are selected by the application name the SDK +registers, which the service sets rather than the chart. + +To jump from a trace to its logs, open a span in Tempo and follow its logs link. The Tempo +datasource in the LGTM image builds that query as `{service_name="..."} | trace_id = "..."`, a +label filter with no parser stage, which resolves only because the Alloy overlay stores +`trace_id` as structured metadata. + +Every sample Alloy forwards carries `cluster="local-kind"` and `project="local"`, set as +external labels in the values file. If a metric has those labels it came through this pipeline. + +## When a log line or trace link does not show up + +Alloy tails every pod's containers by default, so absent logs are usually a write or a label +problem rather than a collection one. Work forwards along the path. + +1. **Is Alloy tailing the pod?** `kubectl -n monitoring logs deploy/alloy -c alloy | grep "opened log stream"` + names each container it reads. `tailer stopped; will retry` is normal against a container that + is not running, and clears once it starts. Against a pod wedged in `ImagePullBackOff` or + `Init:CreateContainerConfigError` it repeats on a backoff for as long as the pod exists - that is + a broken pod, not a broken collector. +1. **Is Loki accepting the writes?** `kubectl -n monitoring logs deploy/alloy -c alloy | grep "error sending batch"`. + A 500 reading `at least 1 live replicas required, could only find 0` means Loki's single + ingester missed its heartbeat, which on a laptop is resource pressure rather than + misconfiguration. Alloy retries, so short stalls only leave gaps in log history. +1. **Did the line land?** `{service_name=""}` in Explore. If the stream exists under a + different `service_name` than you expect, the line had no `service` field and Loki fell back to + deriving the label from the pod's `app` label. +1. **Is `trace_id` queryable?** `{service_name=""} | trace_id != ""` must return lines with + no `| json` stage. If it only works with `| json`, `stage.structured_metadata` is not taking + effect and the trace-to-logs link will stay empty. +1. **Does the span agree?** Compare `service.name` on a span in Tempo against `service_name` on + the log stream. They have to be spelled identically, underscores included. + +Lines logged outside a span carry `"trace_id": null`, which is expected for startup and for the +OTLP exporter's own HTTP calls. Only lines emitted while a span is active can correlate. + +## When a metric does not show up + +Work backwards along the path. + +1. **Is the endpoint serving?** `kubectl exec deploy/ -- curl -sf localhost:/metrics`. + An empty 200 is a real failure mode for Python services using `prometheus_client` in + multiprocess mode: the client picks its storage backend when it is first imported, so + `PROMETHEUS_MULTIPROC_DIR` has to be set in the environment before then, not at runtime. +1. **Did Alloy find the pod?** The Alloy UI lists the targets for `prometheus.scrape "metrics"`. + A missing pod means the annotations are absent; a target in state DOWN means Alloy reached + the pod and the endpoint failed. +1. **Did the sample land?** Query Prometheus directly through the port-forward at + `http://localhost:9090`, which rules out a Grafana datasource problem. + +For traces, check the Alloy logs for OTLP export failures, then confirm the service is exporting +at all - some Gen3 services log the collector address they were configured with at startup. + +## Keeping the values file current + +`examples/local_alloy_values.yaml` contains a copy of `alloyConfigmapData` from +[helm/alloy/values.yaml](../helm/alloy/values.yaml). Helm treats that setting as one string, so +it can only be replaced wholesale, never merged into. Regenerate the copy rather than editing +it, after any change to the chart's configuration: + +```bash +python3 .github/scripts/regenerate_local_alloy_values.py +``` + +The script exits non-zero and writes nothing if the chart no longer contains a line it expects +to rewrite, which means the substitutions need revisiting. Running it in CI and checking for a +dirty tree would catch the overlay drifting from the chart. \ No newline at end of file diff --git a/docs/otel-logs-and-traces.md b/docs/otel-logs-and-traces.md new file mode 100644 index 000000000..eac53ef6a --- /dev/null +++ b/docs/otel-logs-and-traces.md @@ -0,0 +1,346 @@ +# Structured logs and tracing in a deployed cluster + +> **Working notes** This is one reading of what the platform would need in order to +> support JSON logging and tracing for Gen3 services in general. This file is meant to be deleted once the work is scoped. + +## What the services (will eventually) emit + +The Gen3 AI services (`gen3-embeddings` and its siblings) v1.0.0 will emit logs in JSON, tracing info with Open Telemetry, metrics with prometheus, and continuous profiles with Pyroscope. Metrics need no +chart work from what I can tell, and profiles need only an address plus a backend to put at the +other end of it. + +`gen3logging`'s JSON formatter writes one object per line: + +```json +{ + "timestamp": "2026-08-17T17:43:05.149Z", + "logger": "gen3_embeddings", + "level": "INFO", + "message": "...", + "trace_id": "4bf92f3577b34da6a3ce929d0e0e4736", + "span_id": "00f067aa0ba902b7", + "service": "gen3_embeddings" +} +``` + +* Traces go to Alloy over OTLP on port 4318 (`http/protobuf`). +* Profiles go to Pyroscope on port 4040, pushed by the SDK in the service rather than collected by + Alloy. The service reads `PYROSCOPE_SERVER_ADDRESS` for the destination, and `PROFILE_CPU` + (default true) and `PROFILE_MEMORY` (default false) for what to push. All three are process + environment variables, so in a chart they belong in the container `env` rather than in a config + file. +* `trace_id` and `span_id` come from `opentelemetry-instrumentation-logging`(python library), which puts them on + every log record. They are `null` for anything logged outside a request span, such as startup or + the OTLP exporter's own HTTP calls. Only lines emitted while a span is active can correlate. +* `service` is the OpenTelemetry `service.name`, and it is **underscored** (`gen3_embeddings`), + matching the Python package. Kubernetes labels for the same service appear **hyphenated** + (`gen3-embeddings`). + - I opted to keep the service name aligned to the Python package and repo in the traces +* `GEN3_JSON_LOGS=false` switches to existing text formatter that appends `[trace_id=... span_id=...]` as + a suffix if we need to. But everything below assumes JSON. + +For the laptop equivalent, see [local-observability.md](local-observability.md). + +## What production seemingly runs today + +Alloy reaches clusters by two different paths, but they are not equivalent: + +| Path | Config lives in | Traces go to | +| ----------------------- | ------------------------------------------ | --------------------------------------------------------------------- | +| ArgoCD (the one in use) | `helm/cluster-level-resources/values.yaml` | `https://tempo.planx-pla.net:443`, external and CTDS-managed | +| The `helm/alloy` chart | `helm/alloy/values.yaml` | `monitoring-tempo-distributor.monitoring:4317`, which nothing creates | + +Logs and metrics in both cases go to the Loki and Mimir deployed by `helm/observability` (the +`lgtm-distributed` chart, with `lgtm.tempo.enabled: false`). + +Everything below refers to the ArgoCD path unless it says otherwise. + +# Getting correlation working + +These three sections are ordered by dependency. Nothing in the second is observable until the +first is done, and the third has no effect without the second. + +## 1. Loki has to accept the streams + +**Symptom:** no logs in Loki for a service, and `final error sending batch ... status 400` in +Alloy's own logs, reading `has N label names; limit 15`. + +`helm/observability/values.yaml` sets `loki.structuredConfig.limits_config` with +`max_query_series`, `max_streams_per_user`, and `max_entries_limit_per_query`, but not +`max_label_names_per_series`, which therefore sits at Loki's default of **15**. + +A Gen3 pod stream carries about fifteen. `discovery.relabel "all_pods"` in +`cluster-level-resources/values.yaml` sets five labels, `labelmap`s every pod label on top, then +drops nine high-cardinality ones; `loki.write` adds `cluster` and `project`, and +`loki.source.kubernetes` adds `instance` and `job`. What pushes it over is the network-policy +labels Gen3 charts attach: `netnolimit`, `public`, `userhelper`, and for some services +`authprovider`, `internet`, `linklocal`, `netvpc`. `fence` is the clearest case. + +Two fixes, not exclusive: + +```yaml +# helm/observability/values.yaml, under lgtm.loki.structuredConfig.limits_config +limits_config: + max_label_names_per_series: 32 +``` + +or extend the existing `labeldrop` regex in `cluster-level-resources/values.yaml` to drop the +network-policy labels, which are probably not useful for querying logs: + +``` +regex = "pod_template_hash|...|netnolimit|public|userhelper|authprovider|internet|linklocal|netvpc" +``` + +Raising the limit is the smaller change and keeps the labels available. Dropping them is better +hygiene, since each one multiplies Loki's stream count. Prefer dropping, and raise the limit as +well so that a chart adding one more label does not silently start dropping logs again. + +**Verify:** `kubectl -n monitoring logs deploy/alloy | grep "status 400"` is quiet, and +`{service_name="gen3_embeddings"}` in Explore returns lines. + +## 2. `trace_id` and `service_name` have to be queryable + +`cluster-level-resources/values.yaml` wires pod logs straight through: + +```alloy +loki.source.kubernetes "pods" { + targets = discovery.relabel.all_pods.output + forward_to = [loki.write.endpoint.receiver] +} +``` + +Two problems follow from that. + +`trace_id` is only text inside the log message, so a LogQL label filter cannot see it and a query +built from a trace id matches nothing. + +The service name also disagrees with itself. Grafana builds its trace-to-logs stream selector from +the span's `service.name`, while Loki derives `service_name` from the pod's `app` label when the +label is absent. A span says `service.name = gen3_embeddings` and its logs land on a stream +labelled `service_name = gen3-embeddings`, so the join finds nothing. Underscore against hyphen. + +Insert a processing stage between the source and the write: + +```alloy +loki.source.kubernetes "pods" { + targets = discovery.relabel.all_pods.output + forward_to = [loki.process.pod_logs.receiver] +} + +loki.process "pod_logs" { + forward_to = [loki.write.endpoint.receiver] + + stage.json { + expressions = { trace_id = "trace_id", span_id = "span_id", otel_service = "service" } + } + + stage.structured_metadata { + values = { trace_id = "", span_id = "" } + } + + stage.labels { + values = { service_name = "otel_service" } + } +} +``` + +`trace_id` becomes **structured metadata**, not a stream label. That distinction is the point: +structured metadata is filterable with `| trace_id = "..."` and no parser stage, while a +per-request value used as a stream label would multiply Loki's stream cardinality without bound. + +`stage.labels` takes `service_name` from the line's own `service` field, so the label matches the +span by construction. Lines logged outside a span have no `service` value and keep Loki's derived +label, which is why both spellings appear in a healthy cluster. + +Requires Loki on `tsdb` with schema `v13` or later. `helm/observability/values.yaml` already +configures `store: tsdb` with `schema: v13`. + +Lines that are not JSON, such as etcd, kube-proxy, and nginx, extract nothing and pass through +unchanged. + +This stage currently ships in one place only: the kind overlay, where +`.github/scripts/regenerate_local_alloy_values.py` inserts it while generating +`examples/local_alloy_values.yaml`. Neither `cluster-level-resources/values.yaml` nor +`helm/alloy/values.yaml` contains it, so correlation is a local-development capability until this +lands in one of them. Anything verified against a kind cluster is exercising the overlay rather +than the configuration a cluster runs. + +**Verify:** `{service_name="gen3_embeddings"} | trace_id != ""` returns lines **with no `| json` +stage**. That absence is the proof the value is structured metadata rather than text. If the +stream exists but the filter returns nothing, `stage.json` is not matching, so check that +`GEN3_JSON_LOGS` is not false. Then compare `service.name` on a span in Tempo against +`service_name` on the log stream; they have to be spelled identically. + +## 3. Grafana has to be wired for it + +`helm/observability/values.yaml` does not override `grafana.datasources`, so the +`lgtm-distributed` defaults apply. Those give the Tempo datasource a `tracesToLogsV2` block with +nothing but `datasourceUid: loki` in it. + +Without a `customQuery`, trace-to-logs does a plain stream lookup over a time window rather than +filtering to the request, so "Logs for this span" returns everything the service logged around +that moment. And the Loki datasource has no `derivedFields`, so there is no link in the other +direction, from a log line to its trace. Both need stating explicitly: + +```yaml +lgtm: + grafana: + datasources: + datasources.yaml: + apiVersion: 1 + datasources: + - name: Tempo + uid: tempo + type: tempo + url: http://{{ .Release.Name }}-tempo-query-frontend:3200 + jsonData: + tracesToLogsV2: + customQuery: true + datasourceUid: loki + query: '{$${__tags}} | trace_id = "$${__trace.traceId}"' + tags: + - key: service.name + value: service_name + - name: Loki + uid: loki + type: loki + url: http://{{ .Release.Name }}-loki-gateway + jsonData: + derivedFields: + - name: trace_id + matcherType: label + matcherRegex: trace_id + url: "$${__value.raw}" + datasourceUid: tempo +``` + +Overriding `datasources.yaml` replaces the whole list, so the Mimir datasource has to be restated +alongside these. The `$$` escaping is required because these strings pass through Helm templating. + +`matcherType: label` is what makes the derived field read structured metadata rather than +re-parsing the line, which is why section 2 has to land first. + +While editing datasources, `exemplarTraceIdDestinations` on the Mimir datasource is worth adding. +It turns latency panels into click-throughs to a trace, but only once Mimir is started with +exemplar storage enabled, which is a separate change. + +**Verify:** open a span in Tempo and use its logs link. It should return only that request's +lines, not everything in the window. + +# Separate decisions + +## There is no Tempo in the observability chart + +`helm/observability/values.yaml:12` sets `lgtm.tempo.enabled: false`, and the `helm/alloy` chart +points at a `monitoring-tempo-distributor` that nothing creates. Clusters using the ArgoCD path +send traces to the external `tempo.planx-pla.net` instead, so tracing works there and only there. + +Enabling Tempo needs three values, not one: + +```yaml +lgtm: + tempo: + enabled: true + traces: + otlp: + grpc: + enabled: true # tempo-distributed defaults this to false + storage: + trace: + backend: s3 # `local` cannot work: ingesters and queriers share no filesystem + s3: {} # bucket, region, and an IRSA policy alongside Mimir's and Loki's +``` + +The `traces.otlp.grpc.enabled` default is the one that catches people: enabling Tempo alone leaves +port 4317 closed, and Alloy's exporter fails with nothing obviously wrong in the chart. + +Whether a cluster should run its own Tempo at all, given the central one, is a decision rather +than a defect. + +## There is no Pyroscope in the observability chart either + +The same gap as Tempo, one step further along. `lgtm-distributed` has no Pyroscope subchart at +all: its dependencies are Grafana, `loki-distributed`, `mimir-distributed`, `tempo-distributed`, +and OnCall. Nor is there a CTDS-managed Pyroscope playing the role `tempo.planx-pla.net` plays for +traces. No chart in this repo sets `PYROSCOPE_SERVER_ADDRESS`, so no deployed service pushes +profiles today. + +Local Grafana showing a Pyroscope datasource is not evidence against this. That comes from the +`grafana/otel-lgtm` container in [local-observability.md](local-observability.md), which bundles +Pyroscope and provisions the datasource itself. The two stacks share the letters and nothing +else: the kind overlay runs one process with everything in it, `helm/observability` runs the +distributed charts, and only the first has a profiling backend. + +Supporting profiles in a cluster is a separate release rather than a values change: + +- the [`pyroscope`](https://github.com/grafana/pyroscope/tree/main/operations/pyroscope/helm/pyroscope) + chart from `grafana/helm-charts`, whose distributed mode needs S3 and so an IRSA policy + alongside Mimir's and Loki's +- a Pyroscope datasource in Grafana, which section 3 has to restate anyway once + `datasources.yaml` is overridden +- `PYROSCOPE_SERVER_ADDRESS` on each service that should profile, and `PROFILE_MEMORY` where heap + profiles are wanted + +Nothing in Alloy changes for any of it, which is the one simplifying difference from traces. + +## Alert rules assume a different log shape + +`helm/observability/values.yaml` provisions Grafana alert rules that parse logs. Two filter on a +JSON field the new format does not emit: + +```logql +sum by (cluster) (count_over_time({cluster=~".+"} | json | http_status_code="500" [1h])) > 0 +sum(count_over_time({cluster=~".+"} | json | http_status_code="431" [5m])) >= 2 +``` + +The Gen3 AI services emit `timestamp`, `logger`, `level`, `message`, `trace_id`, `span_id`, and +`service`. There is no `http_status_code`, so these alerts will never fire for them. They were +written against another service's log shape. + +Either have the services add an `http_status_code` field when logging a response, or narrow the +alerts to the services that do emit it. Adding the field is the smaller change and makes the alert +mean what it says. + +> Do we need/use this rule? Should I change the Gen3 AI services to include `http_status_code` when it's available? The problem is that we'd have to ensure a log at the end of every request when the status code is locked in. + +# Seemingly we have defects in the `helm/alloy` chart + +Not required for the ArgoCD path, but it should probably not stay broken. Grouped by what each one costs. + +**Prevents startup.** + +- The template opens a block scalar with `config: |` and then runs the value through `toYaml`, + which emits a second block scalar inside the first. A literal `|` becomes the first line of the + ConfigMap and Alloy fails with `missing second | in ||`. + `cluster-level-resources/templates/alloy-configmap.yaml:8` avoids this by not opening a scalar + and letting `toYaml` produce it: + `config: {{ tpl (index .Values "alloy-configmap-data") . | toYaml | indent 2}}`. +- `values.yaml:12` pins the pod to `topology.kubernetes.io/zone=us-east-1a`, so it stays `Pending` + anywhere else. Clearing this from a values file needs `affinity: null`; `affinity: {}` merges an + empty map and changes nothing. + +**Degrades the data.** + +- The config is never passed through `tpl`, so `cluster` and `project` reach the ConfigMap as the + literal strings `{{ .Values.cluster }}` and `{{ .Values.project }}`, and every metric and log + stream Alloy forwards carries those as external labels. Dropping `toYaml` alone does not fix + this; only `tpl` evaluates the template. Note that `tpl` evaluates the whole config, so any + future stage using Go-style braces, `stage.template` being the common one, would break. +- No `labeldrop`, unlike the ArgoCD config, which makes the label limit in section 1 worse. + +**Blocks correlation.** + +- No `loki.process` stage, so section 2 does not apply and pod logs reach Loki with `trace_id` as + message text only. + +**Collides with another chart.** + +- `helm/faro-collector/templates/alloy-config.yaml` uses the same construction and also renders a + ConfigMap named `alloy-gen3`, so the two charts overwrite each other in one namespace. + +# What needs no work + +Metrics. `common.grafanaAnnotations` already puts `prometheus.io/scrape` and `prometheus.io/path` +on every Gen3 pod, `global.metricsEnabled` defaults to true, and Alloy's +`annotation_autodiscovery_pods` relabel already resolves those to `podIP:/metrics`. +A service only has to serve the endpoint. \ No newline at end of file diff --git a/examples/local_alloy_values.yaml b/examples/local_alloy_values.yaml new file mode 100644 index 000000000..767a0c3f8 --- /dev/null +++ b/examples/local_alloy_values.yaml @@ -0,0 +1,490 @@ +# GENERATED by .github/scripts/regenerate_local_alloy_values.py - re-run that script rather than +# editing alloyConfigmapData below by hand. +# +# Grafana Alloy sized for a kind cluster, writing to the single-pod LGTM stack described in +# docs/local-observability.md. Install with: +# +# helm dependency update helm/alloy +# helm upgrade --install alloy ./helm/alloy -n monitoring -f examples/local_alloy_values.yaml +# +# The release must be named `alloy` in namespace `monitoring`: that is the collector address +# Gen3 service images have baked in as OTEL_EXPORTER_OTLP_ENDPOINT. +alloy: + controller: + type: deployment + replicas: 1 + # helm/alloy/values.yaml pins Alloy to topology.kubernetes.io/zone=us-east-1a. A kind node + # carries no zone label, so without this the pod never leaves Pending. It has to be null + # rather than {}: Helm merges an empty map, which leaves the chart's affinity in place, and + # only an explicit null deletes the key. + affinity: null + + alloy: + stabilityLevel: "public-preview" + uiPathPrefix: /alloy + # Lists replace rather than merge, so the OTLP ports have to be restated here. + extraPorts: + - name: "otel-grpc" + port: 4317 + targetPort: 4317 + protocol: "TCP" + - name: "otel-http" + port: 4318 + targetPort: 4318 + protocol: "TCP" + # A single replica has nobody to gossip with, and the peer lookups are noise in the logs. + clustering: + enabled: false + # The parent chart renders the alloy-gen3 ConfigMap; letting the subchart render one too + # would collide on the name. + configMap: + create: false + name: alloy-gen3 + key: config + resources: + requests: + cpu: 100m + memory: 256Mi + + # Copied from helm/alloy/values.yaml, changing the three write endpoints and the two external + # labels, and inserting the loki.process stage that follows the pod log source. That stage is + # local-only: the chart has no equivalent, so trace-to-logs correlation works here and not in a + # cluster deployed from helm/alloy. + # + # The labels are written out literally on purpose. templates/alloy-config.yaml renders this + # with toYaml rather than tpl, so any {{ }} left in here reaches the ConfigMap unrendered. + alloyConfigmapData: | + logging { + level = "info" + format = "json" + write_to = [loki.write.endpoint.receiver] + } + + /////////////////////// OTLP START /////////////////////// + + otelcol.receiver.otlp "default" { + grpc {} + http {} + + output { + metrics = [otelcol.processor.batch.default.input] + traces = [otelcol.processor.batch.default.input] + } + } + + otelcol.processor.batch "default" { + output { + metrics = [otelcol.exporter.prometheus.default.input] + traces = [otelcol.exporter.otlp.tempo.input] + } + } + + otelcol.exporter.prometheus "default" { + forward_to = [prometheus.remote_write.default.receiver] + } + + otelcol.exporter.otlp "tempo" { + client { + endpoint = "lgtm.monitoring:4317" + // Configure TLS settings for communicating with the endpoint. + tls { + // The connection is insecure. + insecure = true + // Do not verify TLS certificates when connecting. + insecure_skip_verify = true + } + } + } + + + /////////////////////// OTLP END /////////////////////// + + // discover all pods, to be used later in this config + discovery.kubernetes "pods" { + role = "pod" + } + + // discover all services, to be used later in this config + discovery.kubernetes "services" { + role = "service" + } + + // discover all nodes, to be used later in this config + discovery.kubernetes "nodes" { + role = "node" + } + + // Generic scrape of any pod with Annotation "prometheus.io/scrape: true" + discovery.relabel "annotation_autodiscovery_pods" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"] + regex = "true" + action = "keep" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_job"] + action = "replace" + target_label = "job" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_instance"] + action = "replace" + target_label = "instance" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_path"] + action = "replace" + target_label = "__metrics_path__" + } + + // Choose the pod port + // The discovery generates a target for each declared container port of the pod. + // If the metricsPortName annotation has value, keep only the target where the port name matches the one of the annotation. + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_portName"] + regex = "(.+)" + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_port_name"] + action = "keepequal" + target_label = "__tmp_port" + } + + // If the metrics port number annotation has a value, override the target address to use it, regardless whether it is + // one of the declared ports on that Pod. + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"] + regex = "(\\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4})" + replacement = "[$2]:$1" // IPv6 + target_label = "__address__" + } + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_port", "__meta_kubernetes_pod_ip"] + regex = "(\\d+);((([0-9]+?)(\\.|$)){4})" // IPv4, takes priority over IPv6 when both exists + replacement = "$2:$1" + target_label = "__address__" + } + + rule { + source_labels = ["__meta_kubernetes_pod_annotation_prometheus_io_scheme"] + action = "replace" + target_label = "__scheme__" + } + + + // add labels + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_pod_controller_name"] + target_label = "controller" + } + + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + + + rule { + source_labels = ["__meta_kubernetes_pod_label_app"] + target_label = "app" + } + + // map all labels + rule { + action = "labelmap" + regex = "__meta_kubernetes_pod_label_(.+)" + } + } + + // Generic scrape of any service with + // Annotation Autodiscovery + discovery.relabel "annotation_autodiscovery_services" { + targets = discovery.kubernetes.services.targets + rule { + source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scrape"] + regex = "true" + action = "keep" + } + rule { + source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_job"] + action = "replace" + target_label = "job" + } + rule { + source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_instance"] + action = "replace" + target_label = "instance" + } + rule { + source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_path"] + action = "replace" + target_label = "__metrics_path__" + } + + // Choose the service port + rule { + source_labels = ["__meta_kubernetes_service_port_name"] + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_portName"] + regex = "(.+)" + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_service_port_name"] + action = "keepequal" + target_label = "__tmp_port" + } + + rule { + source_labels = ["__meta_kubernetes_service_port_number"] + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_port"] + regex = "(.+)" + target_label = "__tmp_port" + } + rule { + source_labels = ["__meta_kubernetes_service_port_number"] + action = "keepequal" + target_label = "__tmp_port" + } + + rule { + source_labels = ["__meta_kubernetes_service_annotation_prometheus_io_scheme"] + action = "replace" + target_label = "__scheme__" + } + } + + prometheus.scrape "metrics" { + job_name = "integrations/autodiscovery_metrics" + targets = concat(discovery.relabel.annotation_autodiscovery_pods.output, discovery.relabel.annotation_autodiscovery_services.output) + honor_labels = true + clustering { + enabled = true + } + forward_to = [prometheus.relabel.metrics_service.receiver] + } + + + // Node Exporter + // TODO: replace with https://grafana.com/docs/alloy/latest/reference/components/prometheus.exporter.unix/ + discovery.relabel "node_exporter" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_instance"] + regex = "monitoring-extras" + action = "keep" + } + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] + regex = "node-exporter" + action = "keep" + } + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + action = "replace" + target_label = "instance" + } + } + + prometheus.scrape "node_exporter" { + job_name = "integrations/node_exporter" + targets = discovery.relabel.node_exporter.output + scrape_interval = "60s" + clustering { + enabled = true + } + forward_to = [prometheus.relabel.node_exporter.receiver] + } + + prometheus.relabel "node_exporter" { + rule { + source_labels = ["__name__"] + regex = "up|node_cpu.*|node_network.*|node_exporter_build_info|node_filesystem.*|node_memory.*|process_cpu_seconds_total|process_resident_memory_bytes" + action = "keep" + } + forward_to = [prometheus.relabel.metrics_service.receiver] + } + + // Logs from all pods + discovery.relabel "all_pods" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_pod_controller_name"] + target_label = "controller" + } + + rule { + source_labels = ["__meta_kubernetes_pod_label_app"] + target_label = "app" + } + + // map all labels + rule { + action = "labelmap" + regex = "__meta_kubernetes_pod_label_(.+)" + } + + } + + loki.source.kubernetes "pods" { + targets = discovery.relabel.all_pods.output + forward_to = [loki.process.pod_logs.receiver] + } + + // Gen3 services log JSON carrying the OpenTelemetry trace_id. Promoting it to + // structured metadata is what lets Grafana's trace-to-logs query filter on + // `| trace_id = "..."` with no parser stage, which is how the Tempo datasource in + // the local LGTM image is provisioned. It deliberately does not become a stream + // label: a per-request value there would multiply Loki's stream cardinality. + // + // Lines that are not JSON, such as etcd and kube-proxy output, extract nothing and + // pass through unchanged. + loki.process "pod_logs" { + forward_to = [loki.write.endpoint.receiver] + + stage.json { + expressions = { trace_id = "trace_id", span_id = "span_id", otel_service = "service" } + } + + stage.structured_metadata { + values = { trace_id = "", span_id = "" } + } + + // Each line reports the service.name its span was recorded under. Using that as + // the stream label is what keeps logs joinable to traces: Grafana builds its + // trace-to-logs query from service.name, while Loki would otherwise derive + // service_name from the pod's `app` label. Those two spellings differ whenever a + // service names itself with underscores, and the join then silently finds nothing. + // + // Lines logged outside a span extract nothing here and keep Loki's derived value. + stage.labels { + values = { service_name = "otel_service" } + } + } + + // kube-state-metrics + discovery.relabel "relabel_kube_state_metrics" { + targets = discovery.kubernetes.services.targets + rule { + source_labels = ["__meta_kubernetes_namespace"] + regex = "monitoring" + action = "keep" + } + rule { + source_labels = ["__meta_kubernetes_service_name"] + regex = "monitoring-extras-kube-state-metrics" + action = "keep" + } + } + + prometheus.scrape "kube_state_metrics" { + targets = discovery.relabel.relabel_kube_state_metrics.output + job_name = "kube-state-metrics" + metrics_path = "/metrics" + forward_to = [prometheus.remote_write.default.receiver] + } + + // Kubelet + discovery.relabel "kubelet" { + targets = discovery.kubernetes.nodes.targets + rule { + target_label = "__address__" + replacement = "kubernetes.default.svc.cluster.local:443" + } + rule { + source_labels = ["__meta_kubernetes_node_name"] + regex = "(.+)" + replacement = "/api/v1/nodes/${1}/proxy/metrics" + target_label = "__metrics_path__" + } + } + + prometheus.scrape "kubelet" { + job_name = "integrations/kubernetes/kubelet" + targets = discovery.relabel.kubelet.output + scheme = "https" + scrape_interval = "60s" + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + tls_config { + insecure_skip_verify = true + } + clustering { + enabled = true + } + forward_to = [prometheus.relabel.kubelet.receiver] + } + + prometheus.relabel "kubelet" { + rule { + source_labels = ["__name__"] + regex = "up|container_cpu_usage_seconds_total|kubelet_certificate_manager_client_expiration_renew_errors|kubelet_certificate_manager_client_ttl_seconds|kubelet_certificate_manager_server_ttl_seconds|kubelet_cgroup_manager_duration_seconds_bucket|kubelet_cgroup_manager_duration_seconds_count|kubelet_node_config_error|kubelet_node_name|kubelet_pleg_relist_duration_seconds_bucket|kubelet_pleg_relist_duration_seconds_count|kubelet_pleg_relist_interval_seconds_bucket|kubelet_pod_start_duration_seconds_bucket|kubelet_pod_start_duration_seconds_count|kubelet_pod_worker_duration_seconds_bucket|kubelet_pod_worker_duration_seconds_count|kubelet_running_container_count|kubelet_running_containers|kubelet_running_pod_count|kubelet_running_pods|kubelet_runtime_operations_errors_total|kubelet_runtime_operations_total|kubelet_server_expiration_renew_errors|kubelet_volume_stats_available_bytes|kubelet_volume_stats_capacity_bytes|kubelet_volume_stats_inodes|kubelet_volume_stats_inodes_used|kubernetes_build_info|namespace_workload_pod|rest_client_requests_total|storage_operation_duration_seconds_count|storage_operation_errors_total|volume_manager_total_volumes" + action = "keep" + } + forward_to = [prometheus.relabel.metrics_service.receiver] + } + + // Cluster Events + loki.source.kubernetes_events "cluster_events" { + job_name = "integrations/kubernetes/eventhandler" + log_format = "logfmt" + forward_to = [loki.write.endpoint.receiver] + } + + prometheus.relabel "metrics_service" { + forward_to = [prometheus.remote_write.default.receiver] + } + + + // Write Endpoints + // prometheus write endpoint + prometheus.remote_write "default" { + external_labels = { + cluster = "local-kind", + project = "local", + } + endpoint { + url = "http://lgtm.monitoring:9090/api/v1/write" + } + } + + // loki write endpoint + loki.write "endpoint" { + external_labels = { + cluster = "local-kind", + project = "local", + } + endpoint { + url = "http://lgtm.monitoring:3100/loki/api/v1/push" + } + } diff --git a/examples/local_dev_values.yaml b/examples/local_dev_values.yaml index 1250b2c51..f0f96d86f 100644 --- a/examples/local_dev_values.yaml +++ b/examples/local_dev_values.yaml @@ -1,6 +1,6 @@ global: dev: true - hostname: localhost + hostname: localhost # configuration for fence helm chart. You can add it for all our services. fence: @@ -33,4 +33,4 @@ portal: resources: requests: cpu: 0.2 - memory: 500Mi \ No newline at end of file + memory: 500Mi diff --git a/examples/local_lgtm.yaml b/examples/local_lgtm.yaml new file mode 100644 index 000000000..a8df42ec6 --- /dev/null +++ b/examples/local_lgtm.yaml @@ -0,0 +1,95 @@ +# A single-pod Grafana + Prometheus + Tempo + Loki + Pyroscope stack for local development, backing the +# Alloy install described in docs/local-observability.md. Apply with: +# +# kubectl apply -f examples/local_lgtm.yaml +# +# Adapted from the manifest published by grafana/docker-otel-lgtm, with two changes: the Loki +# port is exposed, so Alloy has somewhere to send logs, and the Pyroscope port is exposed, so +# services can push profiles. Both already listen inside the image; a Service that omits the port +# drops the traffic rather than refusing it. +# +# For local development only. One replica, emptyDir storage so everything is lost on restart, +# no ingress, no TLS, and Grafana's default admin/admin. The observability chart is the answer +# for a deployed cluster. +--- +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: lgtm + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: + app: lgtm + template: + metadata: + labels: + app: lgtm + spec: + containers: + - name: lgtm + image: grafana/otel-lgtm:0.30.2 + # Do not add PROMETHEUS_EXTRA_ARGS=--web.enable-remote-write-receiver. The image's + # run-prometheus.sh already passes that flag, and appending a second copy stops + # Prometheus from starting at all, which shows up only as the pod never going Ready. + env: + # Gen3 pods carry enough labels that Alloy's labelmap of them exceeds Loki's + # default of 15 per stream, and every log push is then rejected with a 400. + # Old entries are accepted too, because the pod tailers start from the beginning + # of each pod's log rather than from now. + - name: LOKI_EXTRA_ARGS + value: "-validation.max-label-names-per-series=32 -validation.reject-old-samples=false" + ports: + - { name: grafana, containerPort: 3000 } + - { name: loki, containerPort: 3100 } + - { name: tempo, containerPort: 3200 } + - { name: otel-grpc, containerPort: 4317 } + - { name: otel-http, containerPort: 4318 } + - { name: prometheus, containerPort: 9090 } + - { name: pyroscope, containerPort: 4040 } + readinessProbe: + exec: + command: ["cat", "/tmp/ready"] + initialDelaySeconds: 10 + periodSeconds: 5 + resources: + requests: + cpu: 200m + memory: 1Gi + volumeMounts: + - { name: tempo-data, mountPath: /data/tempo } + - { name: grafana-data, mountPath: /data/grafana } + - { name: loki-data, mountPath: /data/loki } + - { name: loki-storage, mountPath: /loki } + - { name: p8s-storage, mountPath: /data/prometheus } + - { name: pyroscope-storage, mountPath: /data/pyroscope } + volumes: + - { name: tempo-data, emptyDir: {} } + - { name: grafana-data, emptyDir: {} } + - { name: loki-data, emptyDir: {} } + - { name: loki-storage, emptyDir: {} } + - { name: p8s-storage, emptyDir: {} } + - { name: pyroscope-storage, emptyDir: {} } +--- +apiVersion: v1 +kind: Service +metadata: + name: lgtm + namespace: monitoring +spec: + selector: + app: lgtm + ports: + - { name: grafana, port: 3000, targetPort: 3000 } + - { name: loki, port: 3100, targetPort: 3100 } + - { name: tempo, port: 3200, targetPort: 3200 } + - { name: otel-grpc, port: 4317, targetPort: 4317 } + - { name: otel-http, port: 4318, targetPort: 4318 } + - { name: prometheus, port: 9090, targetPort: 9090 } + - { name: pyroscope, port: 4040, targetPort: 4040 } diff --git a/git-hook/funnel-chart-check.sh b/git-hook/funnel-chart-check.sh deleted file mode 100755 index 9f59fa783..000000000 --- a/git-hook/funnel-chart-check.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -funnelVer=$(yq '.dependencies[] | select(.name == "funnel") .version' helm/funnel/Chart.yaml) -chartPath=helm/funnel/charts/funnel-$funnelVer.tgz -if [ ! -f "$chartPath" ]; then - echo "$chartPath does not exist. Please run 'cd helm/funnel && helm dependency update'. Existing files:" - ls helm/funnel/charts - exit 1 -fi diff --git a/git-hook/helm-bump.sh b/git-hook/helm-bump.sh deleted file mode 100755 index 54180601a..000000000 --- a/git-hook/helm-bump.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash - -function bump_semver() { - # Get the current version - version=$1 - - # Split the version into its components - IFS='.' read -r -a version_components <<< "$version" - - # Increment the patch version - version_components[2]=$((version_components[2] + 1)) - - # Reassemble the version string - new_version="${version_components[0]}.${version_components[1]}.${version_components[2]}" - - # Print the new version - echo $new_version -} - -# Array to track which charts were updated -declare -a updated_charts=() - -diff=$(git diff --name-only master helm/* | awk -F '/' '{print $2}' | sort -u) - -for i in $diff; do - # Skip processing the gen3 umbrella chart in the first loop - if [[ "$i" == "gen3" ]]; then - continue - fi - - if git show master:helm/$i/Chart.yaml > /dev/null 2>&1; then - current_version=$(grep -E '^version:' helm/$i/Chart.yaml | awk '{print $2}') - master_version=$(git show master:helm/$i/Chart.yaml | grep -E '^version:' | awk '{print $2}') - if [[ "${current_version}" == "${master_version}" ]]; then - new_version=$(bump_semver $current_version) - echo "Bumping chart version in $i to $new_version" - # Creating a backup file so sed command works on both linux and mac - sed -i.bak "s#^version:.*#version: ${new_version/v/}#g" "helm/$i/Chart.yaml" && rm "helm/$i/Chart.yaml.bak" - - # Track this chart as updated - updated_charts+=("$i") - else - echo "Chart $i already has version bumped from $master_version to $current_version" - # Still track it as updated since it has changes - updated_charts+=("$i") - fi - else - echo "No Chart.yaml for $i in master branch. Maybe it's a new chart?" - # Track new charts too - updated_charts+=("$i") - fi -done - -# Now handle the gen3 umbrella chart -umbrella_chart_path="helm/gen3/Chart.yaml" -umbrella_needs_update=false - -# Check if gen3 umbrella chart itself was modified -if echo "$diff" | grep -q "^gen3$"; then - umbrella_needs_update=true -fi - -# Check if umbrella chart version needs bumping -if git show master:$umbrella_chart_path > /dev/null 2>&1; then - current_umbrella_version=$(grep -E '^version:' $umbrella_chart_path | awk '{print $2}') - master_umbrella_version=$(git show master:$umbrella_chart_path | grep -E '^version:' | awk '{print $2}') - - # Update dependency versions for changed charts - for chart in "${updated_charts[@]}"; do - # Get the new version of the dependency chart - if [[ -f "helm/$chart/Chart.yaml" ]]; then - new_dep_version=$(grep -E '^version:' helm/$chart/Chart.yaml | awk '{print $2}') - - # Update the dependency version in umbrella chart - # Look for the dependency entry and update its version - if grep -q "name: $chart" $umbrella_chart_path; then - echo "Updating $chart dependency version to $new_dep_version in umbrella chart" - # Use awk to update the version line that comes after the matching name - awk -v chart="$chart" -v new_version="$new_dep_version" ' - /^ - name: / { in_dep = ($3 == chart) } - /^ version: / && in_dep { - print " version: " new_version - in_dep = 0 - next - } - { print } - ' $umbrella_chart_path > ${umbrella_chart_path}.tmp && mv ${umbrella_chart_path}.tmp $umbrella_chart_path - - umbrella_needs_update=true - fi - fi - done - - # Bump umbrella chart version if needed - if [[ "$umbrella_needs_update" == "true" ]]; then - if [[ "${current_umbrella_version}" == "${master_umbrella_version}" ]]; then - new_umbrella_version=$(bump_semver $current_umbrella_version) - echo "Bumping gen3 umbrella chart version to $new_umbrella_version" - sed -i.bak "s#^version:.*#version: ${new_umbrella_version/v/}#g" "$umbrella_chart_path" && rm "${umbrella_chart_path}.bak" - else - echo "Gen3 umbrella chart version already bumped from $master_umbrella_version to $current_umbrella_version" - fi - fi -else - echo "No umbrella Chart.yaml found in master branch" -fi - -# Handle common chart special case -if printf '%s\n' "${updated_charts[@]}" | grep -q "^common$"; then - echo "Common chart was updated - this affects all charts that depend on it" - # You might want to add logic here to bump versions of charts that depend on common - # For now, just print a warning - echo "WARNING: Common chart updated. Consider if other chart versions need manual bumping." -fi - -echo "Charts updated: ${updated_charts[*]}" \ No newline at end of file diff --git a/helm/access-backend/Chart.yaml b/helm/access-backend/Chart.yaml index 210286dfb..76a3ef811 100644 --- a/helm/access-backend/Chart.yaml +++ b/helm/access-backend/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.20 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "1.6.1" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/access-backend/README.md b/helm/access-backend/README.md index f8b4ed90d..c3c24ba50 100644 --- a/helm/access-backend/README.md +++ b/helm/access-backend/README.md @@ -1,14 +1,18 @@ # access-backend -![Version: 0.1.20](https://img.shields.io/badge/Version-0.1.20-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.6.1](https://img.shields.io/badge/AppVersion-1.6.1-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.6.1](https://img.shields.io/badge/AppVersion-1.6.1-informational?style=flat-square) A Helm chart for Kubernetes +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values diff --git a/helm/alloy/Chart.yaml b/helm/alloy/Chart.yaml index ac429f844..8377ec932 100644 --- a/helm/alloy/Chart.yaml +++ b/helm/alloy/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.2 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/alloy/README.md b/helm/alloy/README.md index 97a37142a..8caf56baf 100644 --- a/helm/alloy/README.md +++ b/helm/alloy/README.md @@ -1,14 +1,18 @@ # alloy -![Version: 0.1.2](https://img.shields.io/badge/Version-0.1.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for deploying Grafana Alloy +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| https://grafana.github.io/helm-charts | alloy | 0.9.1 | +| Repository | Name | +|------------|------| +| https://grafana.github.io/helm-charts | alloy | ## Values diff --git a/helm/alloy/SETUP.md b/helm/alloy/SETUP.md index 6a0b28f23..0fb74cc4e 100644 --- a/helm/alloy/SETUP.md +++ b/helm/alloy/SETUP.md @@ -8,6 +8,10 @@ In this deployment, the Alloy ConfigMap plays a crucial role in configuring whic Before deploying Alloy, it is important to first deploy the "observability" Helm chart, as it provides the necessary components and configuration for Alloy to function properly. Please refer to the [SETUP.md](https://github.com/uc-cdis/gen3-helm/blob/master/helm/observability/SETUP.md) observability chart documentation for instructions on how to set it up before proceeding with the Alloy deployment. +For a kind cluster, follow [docs/local-observability.md](../../docs/local-observability.md) instead. The observability chart is sized for EKS and does not deploy Tempo, so traces have nowhere to go. + +Profiles are the one kind of telemetry Alloy does not carry. A service that ships a Pyroscope SDK pushes to the address in its own `PYROSCOPE_SERVER_ADDRESS`, so supporting profiling needs a Pyroscope for that address to point at and no change to `alloyConfigmapData`. + ## Configuring Alloy ### Helm Chart Configuration diff --git a/helm/alloy/templates/alloy-config.yaml b/helm/alloy/templates/alloy-config.yaml index 0bf028758..c49e453c4 100644 --- a/helm/alloy/templates/alloy-config.yaml +++ b/helm/alloy/templates/alloy-config.yaml @@ -5,5 +5,5 @@ metadata: data: config: | {{- with .Values.alloy.alloyConfigmapData }} - {{- toYaml . | nindent 4 }} + {{- . | nindent 4 }} {{ end }} \ No newline at end of file diff --git a/helm/ambassador/Chart.yaml b/helm/ambassador/Chart.yaml index 34024a11e..4de067b5c 100644 --- a/helm/ambassador/Chart.yaml +++ b/helm/ambassador/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.37 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "1.4.2" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/ambassador/README.md b/helm/ambassador/README.md index 63054f7c4..312614f7e 100644 --- a/helm/ambassador/README.md +++ b/helm/ambassador/README.md @@ -1,14 +1,18 @@ # ambassador -![Version: 0.1.37](https://img.shields.io/badge/Version-0.1.37-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.4.2](https://img.shields.io/badge/AppVersion-1.4.2-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.4.2](https://img.shields.io/badge/AppVersion-1.4.2-informational?style=flat-square) A Helm chart for deploying ambassador for gen3 +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values diff --git a/helm/arborist/Chart.yaml b/helm/arborist/Chart.yaml index 261659d79..56793bfdc 100644 --- a/helm/arborist/Chart.yaml +++ b/helm/arborist/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.34 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,7 +25,7 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common - name: postgresql version: 11.9.13 diff --git a/helm/arborist/README.md b/helm/arborist/README.md index 1409c32cc..cc91a6821 100644 --- a/helm/arborist/README.md +++ b/helm/arborist/README.md @@ -1,15 +1,19 @@ # arborist -![Version: 0.1.34](https://img.shields.io/badge/Version-0.1.34-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 arborist +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | -| https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | +| Repository | Name | +|------------|------| +| file://../common | common | +| https://charts.bitnami.com/bitnami | postgresql | ## Values diff --git a/helm/argo-wrapper/Chart.yaml b/helm/argo-wrapper/Chart.yaml index 182ad309f..35b567a7c 100644 --- a/helm/argo-wrapper/Chart.yaml +++ b/helm/argo-wrapper/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.30 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/argo-wrapper/README.md b/helm/argo-wrapper/README.md index a87b9c334..4a238d390 100644 --- a/helm/argo-wrapper/README.md +++ b/helm/argo-wrapper/README.md @@ -1,14 +1,18 @@ # argo-wrapper -![Version: 0.1.30](https://img.shields.io/badge/Version-0.1.30-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 Argo Wrapper Service +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values @@ -61,6 +65,7 @@ A Helm chart for gen3 Argo Wrapper Service | revisionHistoryLimit | int | `2` | Number of old revisions to retain | | s3Bucket | string | `"argo-artifact-downloadable"` | S3 bucket name for Argo artifacts (allows pre-signed URLs). | | scalingGroups | list | `[{"user1":"workflow1"},{"user2":"workflow2"},{"user3":"workflow3"}]` | The workflow scaling groups to be used by Argo. | +| securityContext | map | `{}` | Security context for the containers in the pod | | selectorLabels | map | `nil` | Will completely override the selectorLabels defined in the common chart's _label_setup.tpl | | service | map | `{"port":8000,"targetPort":8000,"type":"ClusterIP"}` | Kubernetes service information. | | service.port | int | `8000` | The port number that the service exposes. | diff --git a/helm/argo-wrapper/templates/deployment.yaml b/helm/argo-wrapper/templates/deployment.yaml index 44ffdc0be..ab8c70233 100644 --- a/helm/argo-wrapper/templates/deployment.yaml +++ b/helm/argo-wrapper/templates/deployment.yaml @@ -55,6 +55,8 @@ spec: {{- end }} containers: - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" livenessProbe: httpGet: diff --git a/helm/argo-wrapper/values.yaml b/helm/argo-wrapper/values.yaml index 95e7aae18..8837da919 100644 --- a/helm/argo-wrapper/values.yaml +++ b/helm/argo-wrapper/values.yaml @@ -111,6 +111,16 @@ resources: # -- (string) The maximum amount of memory the container can use memory: 128Mi +# -- (map) Security context for the containers in the pod +securityContext: + {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + # -- (map) Kubernetes service information. service: # -- (string) Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". diff --git a/helm/audit/Chart.yaml b/helm/audit/Chart.yaml index ad2c2de58..d51cd32d4 100644 --- a/helm/audit/Chart.yaml +++ b/helm/audit/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.42 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -24,7 +24,7 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common - name: postgresql version: 11.9.13 diff --git a/helm/audit/README.md b/helm/audit/README.md index 1ca245644..c38aff7a9 100644 --- a/helm/audit/README.md +++ b/helm/audit/README.md @@ -1,15 +1,19 @@ # audit -![Version: 0.1.42](https://img.shields.io/badge/Version-0.1.42-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for Kubernetes +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | -| https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | +| Repository | Name | +|------------|------| +| file://../common | common | +| https://charts.bitnami.com/bitnami | postgresql | ## Values diff --git a/helm/audit/templates/deployment.yaml b/helm/audit/templates/deployment.yaml index dd740acf6..cd85eaa6f 100644 --- a/helm/audit/templates/deployment.yaml +++ b/helm/audit/templates/deployment.yaml @@ -33,6 +33,7 @@ spec: {{- include "audit.selectorLabels" . | nindent 8 }} {{- include "common.extraLabels" . | nindent 8 }} internet: "yes" + linklocal: "yes" netvpc: "yes" spec: {{- if .Values.global.topologySpread.enabled }} @@ -48,6 +49,8 @@ spec: {{- end }} containers: - name: audit + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} ports: @@ -114,6 +117,8 @@ spec: {{- end }} initContainers: - name: audit-init + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} env: diff --git a/helm/aws-es-proxy/Chart.yaml b/helm/aws-es-proxy/Chart.yaml index bf9421817..cdae086cd 100644 --- a/helm/aws-es-proxy/Chart.yaml +++ b/helm/aws-es-proxy/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.41 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/aws-es-proxy/README.md b/helm/aws-es-proxy/README.md index 56c05879f..61c80bc2d 100644 --- a/helm/aws-es-proxy/README.md +++ b/helm/aws-es-proxy/README.md @@ -1,14 +1,18 @@ # aws-es-proxy -![Version: 0.1.41](https://img.shields.io/badge/Version-0.1.41-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for AWS ES Proxy Service for gen3 +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values @@ -69,6 +73,7 @@ A Helm chart for AWS ES Proxy Service for gen3 | secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null}` | Secret information to access AWS ES cluster. | | secrets.awsAccessKeyId | str | `nil` | AWS access key ID. Overrides global key. | | secrets.awsSecretAccessKey | str | `nil` | AWS secret access key ID. Overrides global key. | +| securityContext | map | `{}` | Security context for the containers in the pod | | selectorLabels | map | `nil` | Will completely override the selectorLabels defined in the common chart's _label_setup.tpl | | service | map | `{"port":9200,"type":"ClusterIP"}` | Kubernetes service information. | | service.port | int | `9200` | The port number that the service exposes. | diff --git a/helm/aws-es-proxy/templates/deployment.yaml b/helm/aws-es-proxy/templates/deployment.yaml index 22af7d97b..a21dd8c34 100644 --- a/helm/aws-es-proxy/templates/deployment.yaml +++ b/helm/aws-es-proxy/templates/deployment.yaml @@ -43,6 +43,8 @@ spec: optional: true containers: - name: "esproxy" + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} {{- with .Values.ports}} diff --git a/helm/aws-es-proxy/values.yaml b/helm/aws-es-proxy/values.yaml index 7a96c025c..fb8bf58d2 100644 --- a/helm/aws-es-proxy/values.yaml +++ b/helm/aws-es-proxy/values.yaml @@ -131,6 +131,16 @@ resources: # -- (string) The maximum amount of memory the container can use memory: 2Gi +# -- (map) Security context for the containers in the pod +securityContext: + {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + # -- (map) Kubernetes service information. service: # -- (string) Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". diff --git a/helm/aws-sigv4-proxy/Chart.yaml b/helm/aws-sigv4-proxy/Chart.yaml index 09e4eb4c2..a7115ec8a 100644 --- a/helm/aws-sigv4-proxy/Chart.yaml +++ b/helm/aws-sigv4-proxy/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.2 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/aws-sigv4-proxy/README.md b/helm/aws-sigv4-proxy/README.md index 8b48b327b..c4422188f 100644 --- a/helm/aws-sigv4-proxy/README.md +++ b/helm/aws-sigv4-proxy/README.md @@ -1,14 +1,18 @@ # aws-sigv4-proxy -![Version: 0.1.2](https://img.shields.io/badge/Version-0.1.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for AWS ES Proxy Service for gen3 +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values diff --git a/helm/cedar/Chart.yaml b/helm/cedar/Chart.yaml index 1658e0ec1..5088eeb9c 100644 --- a/helm/cedar/Chart.yaml +++ b/helm/cedar/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.26 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/cedar/README.md b/helm/cedar/README.md index 170b9741e..4f20d69f6 100644 --- a/helm/cedar/README.md +++ b/helm/cedar/README.md @@ -1,14 +1,18 @@ # cedar -![Version: 0.1.26](https://img.shields.io/badge/Version-0.1.26-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 cedar wrapper +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values diff --git a/helm/cluster-level-resources/Chart.yaml b/helm/cluster-level-resources/Chart.yaml index 938343293..e8ed0007d 100644 --- a/helm/cluster-level-resources/Chart.yaml +++ b/helm/cluster-level-resources/Chart.yaml @@ -4,6 +4,6 @@ description: An app-of-apps Helm chart that allows for flexible deployment of re type: application -version: 0.6.45 +version: 0.0.0 appVersion: "1.17.0" diff --git a/helm/cluster-level-resources/README.md b/helm/cluster-level-resources/README.md index 42d8af39a..c194c0699 100644 --- a/helm/cluster-level-resources/README.md +++ b/helm/cluster-level-resources/README.md @@ -1,9 +1,13 @@ # cluster-level-resources -![Version: 0.6.45](https://img.shields.io/badge/Version-0.6.45-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.17.0](https://img.shields.io/badge/AppVersion-1.17.0-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.17.0](https://img.shields.io/badge/AppVersion-1.17.0-informational?style=flat-square) An app-of-apps Helm chart that allows for flexible deployment of resources that support Gen3 +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Values | Key | Type | Default | Description | @@ -57,6 +61,9 @@ An app-of-apps Helm chart that allows for flexible deployment of resources that | ebs-csi-driver.configuration.enabled | bool | `false` | | | ebs-csi-driver.enabled | bool | `false` | | | ebs-csi-driver.targetRevision | string | `"2.48.0"` | | +| efs-csi-driver.configuration.enabled | bool | `false` | | +| efs-csi-driver.enabled | bool | `false` | | +| efs-csi-driver.targetRevision | string | `"4.2.0"` | | | eksClusterEndpoint | string | `""` | | | external-secrets.configuration.enabled | bool | `false` | | | external-secrets.enabled | bool | `false` | | @@ -173,6 +180,7 @@ An app-of-apps Helm chart that allows for flexible deployment of resources that | karpenter.controller.image.digest | string | `"sha256:0c142050d872cb0ac7b30a188ec36aa765b449718cde0c7e49f7495b28f47c29"` | | | karpenter.controller.image.tag | string | `"1.0.8"` | | | karpenter.enabled | bool | `false` | | +| karpenter.interruptionQueue | string | `""` | | | karpenter.resources.limits.cpu | string | `"1"` | | | karpenter.resources.limits.memory | string | `"1Gi"` | | | karpenter.resources.requests.cpu | string | `"1"` | | diff --git a/helm/cluster-level-resources/templates/certmanager.yaml b/helm/cluster-level-resources/templates/certmanager.yaml index 81ebca55d..785892602 100644 --- a/helm/cluster-level-resources/templates/certmanager.yaml +++ b/helm/cluster-level-resources/templates/certmanager.yaml @@ -15,6 +15,9 @@ spec: values: | crds: enabled: true + {{- if (index .Values "cert-manager" "configuration" "enabled") }} + {{ toYaml (index .Values "cert-manager" "configuration") | nindent 10 }} + {{- end }} destination: server: "https://kubernetes.default.svc" namespace: cert-manager diff --git a/helm/cluster-level-resources/templates/efs-csi-driver.yaml b/helm/cluster-level-resources/templates/efs-csi-driver.yaml new file mode 100644 index 000000000..d7b1142e9 --- /dev/null +++ b/helm/cluster-level-resources/templates/efs-csi-driver.yaml @@ -0,0 +1,44 @@ +{{ if index .Values "efs-csi-driver" "enabled" }} +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: efs-csi-driver + namespace: argocd +spec: + project: default + sources: + - chart: aws-efs-csi-driver + repoURL: https://kubernetes-sigs.github.io/aws-efs-csi-driver + targetRevision: {{ index .Values "efs-csi-driver" "targetRevision" }} + helm: + releaseName: efs-csi-driver + {{- if index .Values "efs-csi-driver" "configuration" "enabled" }} + valueFiles: + - $values/{{ .Values.cluster }}/cluster-values/efs-csi-driver.yaml + - repoURL: {{ .Values.configuration.configurationRepo }} + targetRevision: {{ .Values.configuration.configurationRevision }} + ref: values + {{- else }} + values: | + image: + tag: v3.3.0 + useFIPS: true + + controller: + serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::{{ .Values.accountNumber }}:role/{{ .Values.eksClusterName | default .Values.cluster }}--kube-system--efs-csi-controller + node: + serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::{{ .Values.accountNumber }}:role/{{ .Values.eksClusterName | default .Values.cluster }}--kube-system--efs-csi-node + {{- end }} + destination: + server: "https://kubernetes.default.svc" + namespace: kube-system + syncPolicy: + syncOptions: + - CreateNamespace=false + automated: + selfHeal: true +{{ end }} diff --git a/helm/cluster-level-resources/templates/karpenter.yaml b/helm/cluster-level-resources/templates/karpenter.yaml index ed28085ed..882c078b7 100644 --- a/helm/cluster-level-resources/templates/karpenter.yaml +++ b/helm/cluster-level-resources/templates/karpenter.yaml @@ -38,6 +38,9 @@ spec: settings: clusterName: {{ .Values.eksClusterName | default .Values.cluster }} clusterEndpoint: {{ .Values.eksClusterEndpoint }} + {{- if .Values.karpenter.interruptionQueue }} + interruptionQueue: {{ .Values.karpenter.interruptionQueue }} + {{- end }} controller: env: - name: AWS_REGION diff --git a/helm/cluster-level-resources/values.yaml b/helm/cluster-level-resources/values.yaml index 7386c66f7..7cda90eab 100644 --- a/helm/cluster-level-resources/values.yaml +++ b/helm/cluster-level-resources/values.yaml @@ -93,6 +93,12 @@ ebs-csi-driver: configuration: enabled: false +efs-csi-driver: + enabled: false + targetRevision: "4.2.0" + configuration: + enabled: false + external-secrets: enabled: false targetRevision: "0.9.13" @@ -115,6 +121,7 @@ karpenter: enabled: false awsRegion: "us-east-1" targetRevision: 1.0.8 + interruptionQueue: "" configuration: enabled: false resources: diff --git a/helm/cohort-middleware/Chart.yaml b/helm/cohort-middleware/Chart.yaml index df20e8914..29ae1ac88 100644 --- a/helm/cohort-middleware/Chart.yaml +++ b/helm/cohort-middleware/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.22 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/cohort-middleware/README.md b/helm/cohort-middleware/README.md index 8b5e69dcf..3abb23738 100644 --- a/helm/cohort-middleware/README.md +++ b/helm/cohort-middleware/README.md @@ -1,14 +1,18 @@ # cohort-middleware -![Version: 0.1.22](https://img.shields.io/badge/Version-0.1.22-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 cohort-middleware +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values diff --git a/helm/cohort-middleware/templates/deployment.yaml b/helm/cohort-middleware/templates/deployment.yaml index 9c76d647d..511f4f8a7 100644 --- a/helm/cohort-middleware/templates/deployment.yaml +++ b/helm/cohort-middleware/templates/deployment.yaml @@ -15,11 +15,11 @@ spec: {{- include "cohort-middleware.selectorLabels" . | nindent 6 }} template: metadata: - {{- with .Values.podAnnotations }} annotations: checksum/config: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} - {{- end }} + {{- end }} labels: dbohdsi: "yes" dbomop-data: "yes" @@ -85,3 +85,4 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + diff --git a/helm/common/Chart.yaml b/helm/common/Chart.yaml index 797834d62..f3f2d8e86 100644 --- a/helm/common/Chart.yaml +++ b/helm/common/Chart.yaml @@ -15,7 +15,7 @@ type: library # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.36 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/common/README.md b/helm/common/README.md index 1438d8d0c..1e9ad2ca1 100644 --- a/helm/common/README.md +++ b/helm/common/README.md @@ -1,9 +1,13 @@ # common -![Version: 0.1.36](https://img.shields.io/badge/Version-0.1.36-informational?style=flat-square) ![Type: library](https://img.shields.io/badge/Type-library-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: library](https://img.shields.io/badge/Type-library-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for provisioning databases in gen3 +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Values | Key | Type | Default | Description | diff --git a/helm/common/templates/_db_setup_job.tpl b/helm/common/templates/_db_setup_job.tpl index ee610ef60..395721688 100644 --- a/helm/common/templates/_db_setup_job.tpl +++ b/helm/common/templates/_db_setup_job.tpl @@ -165,11 +165,15 @@ spec: psql -d $SERVICE_PGDB -c "ALTER SCHEMA public OWNER TO \"$SERVICE_PGUSER\";" psql -d $SERVICE_PGDB -c "GRANT ALL ON SCHEMA public TO \"$SERVICE_PGUSER\";" psql -d $SERVICE_PGDB -c "GRANT ALL ON ALL TABLES IN SCHEMA public TO \"$SERVICE_PGUSER\";" + psql -d $SERVICE_PGDB -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO \"$SERVICE_PGUSER\";" psql -d $SERVICE_PGDB -c "ALTER ROLE \"$SERVICE_PGUSER\" WITH LOGIN;" echo "Creating ltree extension..." psql -d $SERVICE_PGDB -c "CREATE EXTENSION IF NOT EXISTS ltree;" + echo "Creating pgvector extension..." + psql -d $SERVICE_PGDB -c "CREATE EXTENSION IF NOT EXISTS vector;" + PGPASSWORD=$SERVICE_PGPASS psql -d $SERVICE_PGDB -h $PGHOST -p $PGPORT -U $SERVICE_PGUSER -c "\conninfo" kubectl patch secret/{{ .Chart.Name }}-dbcreds -p '{"data":{"dbcreated":"dHJ1ZQo="}}' fi diff --git a/helm/dashboard/Chart.yaml b/helm/dashboard/Chart.yaml index 6a39369ae..f3027a8b3 100644 --- a/helm/dashboard/Chart.yaml +++ b/helm/dashboard/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.20 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "1.16.0" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/dashboard/README.md b/helm/dashboard/README.md index 3d87f0458..dcdf09cf2 100644 --- a/helm/dashboard/README.md +++ b/helm/dashboard/README.md @@ -1,14 +1,18 @@ # dashboard -![Version: 0.1.20](https://img.shields.io/badge/Version-0.1.20-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) A Helm chart for Kubernetes +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values diff --git a/helm/dashboard/templates/deployment.yaml b/helm/dashboard/templates/deployment.yaml index eb445cbed..a00b72977 100644 --- a/helm/dashboard/templates/deployment.yaml +++ b/helm/dashboard/templates/deployment.yaml @@ -25,6 +25,7 @@ spec: s3: "yes" netvpc: "yes" internet: "yes" + linklocal: "yes" app: "dashboard" {{- include "dashboard.labels" . | nindent 8 }} {{- with .Values.podLabels }} diff --git a/helm/data-upload-cron/Chart.yaml b/helm/data-upload-cron/Chart.yaml index 79e789658..9e63c612e 100644 --- a/helm/data-upload-cron/Chart.yaml +++ b/helm/data-upload-cron/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.6 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/data-upload-cron/README.md b/helm/data-upload-cron/README.md index a012ceac3..9901fc07a 100644 --- a/helm/data-upload-cron/README.md +++ b/helm/data-upload-cron/README.md @@ -1,14 +1,18 @@ # data-upload-cron -![Version: 0.1.6](https://img.shields.io/badge/Version-0.1.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for the data upload cronjob +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values diff --git a/helm/datareplicate/Chart.yaml b/helm/datareplicate/Chart.yaml index bfe8a0d0e..93aea2dee 100644 --- a/helm/datareplicate/Chart.yaml +++ b/helm/datareplicate/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.21 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/datareplicate/README.md b/helm/datareplicate/README.md index 7f4e63118..d933a45b5 100644 --- a/helm/datareplicate/README.md +++ b/helm/datareplicate/README.md @@ -1,14 +1,18 @@ # datareplicate -![Version: 0.1.21](https://img.shields.io/badge/Version-0.1.21-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 datareplicate +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values diff --git a/helm/dicom-server/Chart.yaml b/helm/dicom-server/Chart.yaml index acaa6c980..5b6d415c1 100644 --- a/helm/dicom-server/Chart.yaml +++ b/helm/dicom-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.31 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/dicom-server/README.md b/helm/dicom-server/README.md index 0f9a08d23..c29c9c16e 100644 --- a/helm/dicom-server/README.md +++ b/helm/dicom-server/README.md @@ -1,14 +1,18 @@ # dicom-server -![Version: 0.1.31](https://img.shields.io/badge/Version-0.1.31-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 Dicom Server +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values diff --git a/helm/embedding-management-service/Chart.yaml b/helm/embedding-management-service/Chart.yaml index 85e740977..4fbc8e68b 100644 --- a/helm/embedding-management-service/Chart.yaml +++ b/helm/embedding-management-service/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.7 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "1.16.0" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/embedding-management-service/README.md b/helm/embedding-management-service/README.md index 0bc401fa0..ca4542204 100644 --- a/helm/embedding-management-service/README.md +++ b/helm/embedding-management-service/README.md @@ -1,14 +1,18 @@ # embedding-management-service -![Version: 0.1.7](https://img.shields.io/badge/Version-0.1.7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) A Helm chart for Kubernetes +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values diff --git a/helm/embedding-management-service/templates/deployment.yaml b/helm/embedding-management-service/templates/deployment.yaml index 9fdb690d1..2225b97d4 100644 --- a/helm/embedding-management-service/templates/deployment.yaml +++ b/helm/embedding-management-service/templates/deployment.yaml @@ -38,7 +38,7 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} ports: - name: http - containerPort: {{ .Values.service.targetPort }} + containerPort: {{ .Values.service.port.targetPort }} protocol: TCP env: - name: DB_HOST diff --git a/helm/etl/Chart.yaml b/helm/etl/Chart.yaml index d35774eaa..c32a33b0f 100644 --- a/helm/etl/Chart.yaml +++ b/helm/etl/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.23 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/etl/README.md b/helm/etl/README.md index 832f1a91e..e431de400 100644 --- a/helm/etl/README.md +++ b/helm/etl/README.md @@ -1,9 +1,13 @@ # etl -![Version: 0.1.23](https://img.shields.io/badge/Version-0.1.23-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 etl +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Values | Key | Type | Default | Description | diff --git a/helm/etl/templates/etl-job.yaml b/helm/etl/templates/etl-job.yaml index 685ee8361..f5b4a2e26 100644 --- a/helm/etl/templates/etl-job.yaml +++ b/helm/etl/templates/etl-job.yaml @@ -10,11 +10,11 @@ spec: backoffLimit: 0 template: metadata: - {{- with .Values.podAnnotations }} annotations: - {{- toYaml . | nindent 12 }} checksum/config: {{ include (print $.Template.BasePath "/etl-mapping.yaml") . | sha256sum }} - {{- end }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 12 }} + {{- end }} labels: app: gen3job spec: diff --git a/helm/faro-collector/Chart.yaml b/helm/faro-collector/Chart.yaml index ac429f844..65344a84a 100644 --- a/helm/faro-collector/Chart.yaml +++ b/helm/faro-collector/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -name: alloy -description: A Helm chart for deploying Grafana Alloy +name: faro-collector +description: A Helm chart for deploying Grafana Alloy as a Faro collector # A chart can be either an 'application' or a 'library' chart. # @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.2 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/faro-collector/README.md b/helm/faro-collector/README.md index fd1f86dbd..a8a35c59c 100644 --- a/helm/faro-collector/README.md +++ b/helm/faro-collector/README.md @@ -1,14 +1,18 @@ -# alloy +# faro-collector -![Version: 0.1.2](https://img.shields.io/badge/Version-0.1.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) -A Helm chart for deploying Grafana Alloy +A Helm chart for deploying Grafana Alloy as a Faro collector + +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| https://grafana.github.io/helm-charts | alloy | 0.9.1 | +| Repository | Name | +|------------|------| +| https://grafana.github.io/helm-charts | alloy | ## Values diff --git a/helm/fence/Chart.yaml b/helm/fence/Chart.yaml index 51d761e48..bf13dd279 100644 --- a/helm/fence/Chart.yaml +++ b/helm/fence/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.78 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -24,7 +24,7 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common - name: postgresql version: 11.9.13 diff --git a/helm/fence/README.md b/helm/fence/README.md index 5867fba2a..833fc0603 100644 --- a/helm/fence/README.md +++ b/helm/fence/README.md @@ -1,21 +1,25 @@ # fence -![Version: 0.1.78](https://img.shields.io/badge/Version-0.1.78-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 Fence +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | -| https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | +| Repository | Name | +|------------|------| +| file://../common | common | +| https://charts.bitnami.com/bitnami | postgresql | ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| -| FENCE_CONFIG | map | `{"ACCESS_TOKEN_COOKIE_NAME":"access_token","ACCESS_TOKEN_EXPIRES_IN":1200,"ALLOWED_USER_SERVICE_ACCOUNT_DOMAINS":["developer.gserviceaccount.com","appspot.gserviceaccount.com","iam.gserviceaccount.com"],"ALLOW_GOOGLE_LINKING":true,"ALLOW_NEW_USER_ON_LOGIN":true,"APPLICATION_ROOT":"/user","APP_NAME":"Gen3 Data Commons","ARBORIST":"http://arborist-service","ASSUME_ROLE_CACHE_SECONDS":1800,"AUDIT_SERVICE":"http://audit-service","AUTHLIB_INSECURE_TRANSPORT":true,"AWS_CREDENTIALS":{},"AZ_BLOB_CONTAINER_URL":"https://myfakeblob.blob.core.windows.net/my-fake-container/","AZ_BLOB_CREDENTIALS":null,"BILLING_PROJECT_FOR_SA_CREDS":null,"BILLING_PROJECT_FOR_SIGNED_URLS":null,"CIRRUS_CFG":{"GOOGLE_ADMIN_EMAIL":"","GOOGLE_API_KEY":"","GOOGLE_APPLICATION_CREDENTIALS":"","GOOGLE_CLOUD_IDENTITY_ADMIN_EMAIL":"","GOOGLE_IDENTITY_DOMAIN":"","GOOGLE_PROJECT_ID":"","GOOGLE_STORAGE_CREDS":""},"CLIENT_ALLOWED_SCOPES":["openid","user","data","google_credentials","google_service_account","google_link","ga4gh_passport_v1"],"DATA_UPLOAD_BUCKET":"bucket1","DBGAP_ACCESSION_WITH_CONSENT_REGEX":"(?Pphs[0-9]+)(.(?Pv[0-9]+)){0,1}(.(?Pp[0-9]+)){0,1}.(?Pc[0-9]+)","DEBUG":false,"DEFAULT_LOGIN_IDP":"google","DEFAULT_LOGIN_URL":"{{BASE_URL}}/login/google","DEV_LOGIN_COOKIE_NAME":"dev_login","DREAM_CHALLENGE_GROUP":"DREAM","DREAM_CHALLENGE_TEAM":"DREAM","EMAIL_SERVER":"localhost","ENABLED_IDENTITY_PROVIDERS":{},"ENABLE_AUDIT_LOGS":{"login":false,"presigned_url":false},"ENABLE_AUTOMATIC_BILLING_PERMISSION_SA_CREDS":false,"ENABLE_AUTOMATIC_BILLING_PERMISSION_SIGNED_URLS":false,"ENABLE_CSRF_PROTECTION":true,"ENABLE_DB_MIGRATION":true,"ENABLE_PROMETHEUS_METRICS":false,"ENABLE_VISA_UPDATE_CRON":false,"ENCRYPTION_KEY":"REPLACEME","GA4GH_VISA_ISSUER_ALLOWLIST":["{{BASE_URL}}","https://sts.nih.gov","https://stsstg.nih.gov"],"GEN3_PASSPORT_EXPIRES_IN":43200,"GLOBAL_PARSE_VISAS_ON_LOGIN":false,"GOOGLE_ACCOUNT_ACCESS_EXPIRES_IN":86400,"GOOGLE_BULK_UPDATES":false,"GOOGLE_GROUP_PREFIX":"","GOOGLE_MANAGED_SERVICE_ACCOUNT_DOMAINS":["dataflow-service-producer-prod.iam.gserviceaccount.com","cloudbuild.gserviceaccount.com","cloud-ml.google.com.iam.gserviceaccount.com","container-engine-robot.iam.gserviceaccount.com","dataflow-service-producer-prod.iam.gserviceaccount.com","sourcerepo-service-accounts.iam.gserviceaccount.com","dataproc-accounts.iam.gserviceaccount.com","gae-api-prod.google.com.iam.gserviceaccount.com","genomics-api.google.com.iam.gserviceaccount.com","containerregistry.iam.gserviceaccount.com","container-analysis.iam.gserviceaccount.com","cloudservices.gserviceaccount.com","stackdriver-service.iam.gserviceaccount.com","appspot.gserviceaccount.com","partnercontent.gserviceaccount.com","trifacta-gcloud-prod.iam.gserviceaccount.com","gcf-admin-robot.iam.gserviceaccount.com","compute-system.iam.gserviceaccount.com","gcp-sa-websecurityscanner.iam.gserviceaccount.com","storage-transfer-service.iam.gserviceaccount.com","firebase-sa-management.iam.gserviceaccount.com","firebase-rules.iam.gserviceaccount.com","gcp-sa-cloudbuild.iam.gserviceaccount.com","gcp-sa-automl.iam.gserviceaccount.com","gcp-sa-datalabeling.iam.gserviceaccount.com","gcp-sa-cloudscheduler.iam.gserviceaccount.com"],"GOOGLE_SERVICE_ACCOUNT_KEY_FOR_URL_SIGNING_EXPIRES_IN":2592000,"GOOGLE_SERVICE_ACCOUNT_PREFIX":"","GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN":604800,"GUN_MAIL":{"datacommons.io":{"api_key":"","api_url":"https://api.mailgun.net/v3/mailgun.example.com","default_login":"postmaster@mailgun.example.com","smtp_hostname":"smtp.mailgun.org","smtp_password":""}},"HTTP_PROXY":{"host":null,"port":3128},"INDEXD":"http://indexd-service","INDEXD_PASSWORD":"","INDEXD_USERNAME":"fence","ITRUST_GLOBAL_LOGOUT":"https://auth.nih.gov/siteminderagent/smlogout.asp?mode=nih&AppReturnUrl=","LOGIN_OPTIONS":[{"desc":"description","idp":"google","name":"Login from Google"}],"LOGIN_REDIRECT_WHITELIST":[],"MAX_ACCESS_TOKEN_TTL":3600,"MAX_API_KEY_TTL":2592000,"MAX_PRESIGNED_URL_TTL":3600,"MAX_ROLE_SESSION_INCREASE":false,"MOCK_AUTH":false,"MOCK_GOOGLE_AUTH":false,"MOCK_STORAGE":false,"OAUTH2_JWT_ALG":"RS256","OAUTH2_JWT_ENABLED":true,"OAUTH2_JWT_ISS":"{{BASE_URL}}","OAUTH2_PROVIDER_ERROR_URI":"/api/oauth2/errors","OAUTH2_TOKEN_EXPIRES_IN":{"authorization_code":1200,"implicit":1200},"OPENID_CONNECT":{"cilogon":{"client_id":"","client_secret":"","discovery_url":"https://cilogon.org/.well-known/openid-configuration","mock":false,"mock_default_user":"http://cilogon.org/serverT/users/64703","redirect_url":"{{BASE_URL}}/login/cilogon/login/","scope":"openid email profile"},"cognito":{"client_id":"","client_secret":"","discovery_url":"https://cognito-idp.{REGION}.amazonaws.com/{USER-POOL-ID}/.well-known/openid-configuration","redirect_url":"{{BASE_URL}}/login/cognito/login/","scope":"openid email"},"fence":{"access_token_url":"{{api_base_url}}/oauth2/token","api_base_url":"","authorize_url":"{{api_base_url}}/oauth2/authorize","client_id":"","client_kwargs":{"redirect_uri":"{{BASE_URL}}/login/fence/login","scope":"openid"},"client_secret":"","mock":false,"mock_default_user":"test@example.com","name":"","refresh_token_url":"{{api_base_url}}/oauth2/token","shibboleth_discovery_url":"https://login.bionimbus.org/Shibboleth.sso/DiscoFeed"},"generic_oidc_idp":{"client_id":"","client_secret":"","discovery":{"authorization_endpoint":"","jwks_uri":"","token_endpoint":""},"discovery_url":"https://server.com/.well-known/openid-configuration","email_field":"","name":"some_idp","redirect_url":"{{BASE_URL}}/login/some_idp/login","scope":"","user_id_field":""},"google":{"client_id":"","client_secret":"","discovery_url":"https://accounts.google.com/.well-known/openid-configuration","mock":"","mock_default_user":"test@example.com","redirect_url":"{{BASE_URL}}/login/google/login/","scope":"openid email"},"microsoft":{"client_id":"","client_secret":"","discovery_url":"https://login.microsoftonline.com/organizations/v2.0/.well-known/openid-configuration","mock":false,"mock_default_user":"test@example.com","redirect_url":"{{BASE_URL}}/login/microsoft/login/","scope":"openid email"},"okta":{"client_id":"","client_secret":"","discovery_url":"","redirect_url":"{{BASE_URL}}/login/okta/login/","scope":"openid email"},"orcid":{"client_id":"","client_secret":"","discovery_url":"https://orcid.org/.well-known/openid-configuration","mock":false,"mock_default_user":"0000-0002-2601-8132","redirect_url":"{{BASE_URL}}/login/orcid/login/","scope":"openid"},"ras":{"client_id":"","client_secret":"","discovery_url":"https://sts.nih.gov/.well-known/openid-configuration","mock":false,"mock_default_user":"test@example.com","redirect_url":"{{BASE_URL}}/login/ras/callback","scope":"openid email profile ga4gh_passport_v1"},"shibboleth":{"client_id":"","client_secret":"","redirect_url":"{{BASE_URL}}/login/shib/login"},"synapse":{"client_id":"","client_secret":"","discovery_url":"","redirect_url":"","scope":"openid"}},"OVERRIDE_NGINX_RATE_LIMIT":18,"PRIVACY_POLICY_URL":null,"PROBLEM_USER_EMAIL_NOTIFICATION":{"admin":["admin@example.edu"],"content":"The Data Commons Framework utilizes dbGaP for data access authorization. Another member of a Google project you belong to ({}) is attempting to register a service account to the following additional datasets ({}). Please contact dbGaP to request access.\n","domain":"example.com","from":"do-not-reply@example.com","subject":"Account access error notification"},"PUSH_AUDIT_LOGS_CONFIG":{"aws_sqs_config":{"aws_cred":null,"region":null,"sqs_url":null},"type":"aws_sqs"},"RAS_REFRESH_EXPIRATION":1296000,"RAS_USERINFO_ENDPOINT":"/openid/connect/v1.1/userinfo","REFRESH_TOKEN_EXPIRES_IN":2592000,"REGISTERED_USERS_GROUP":"","REGISTER_USERS_ON":false,"REMOVE_SERVICE_ACCOUNT_EMAIL_NOTIFICATION":{"admin":["admin@example.edu"],"content":"Service accounts were removed from access control data because some users or service accounts of GCP Project {} are not authorized to access the data sets associated to the service accounts, or do not adhere to the security policies.\n","domain":"example.com","enable":false,"from":"do-not-reply@example.com","subject":"User service account removal notification"},"RENEW_ACCESS_TOKEN_BEFORE_EXPIRATION":false,"S3_BUCKETS":{},"SEND_FROM":"example@gmail.com","SEND_TO":"example@gmail.com","SERVICE_ACCOUNT_LIMIT":6,"SESSION_ALLOWED_SCOPES":["openid","user","credentials","data","admin","google_credentials","google_service_account","google_link","ga4gh_passport_v1"],"SESSION_COOKIE_DOMAIN":null,"SESSION_COOKIE_NAME":"fence","SESSION_COOKIE_SECURE":true,"SESSION_LIFETIME":28800,"SESSION_TIMEOUT":1800,"SHIBBOLETH_HEADER":"persistent_id","SSO_URL":"https://auth.nih.gov/affwebservices/public/saml2sso?SPID={{BASE_URL}}/shibboleth&RelayState=","STORAGE_CREDENTIALS":{},"SUPPORT_EMAIL_FOR_ERRORS":null,"SYNAPSE_AUTHZ_TTL":86400,"SYNAPSE_DISCOVERY_URL":null,"SYNAPSE_JWKS_URI":null,"SYNAPSE_URI":"https://repo-prod.prod.sagebase.org/auth/v1","TOKEN_PROJECTS_CUTOFF":10,"USERSYNC":{"fallback_to_dbgap_sftp":false,"sync_from_visas":false,"visa_types":{"ras":["https://ras.nih.gov/visas/v1","https://ras.nih.gov/visas/v1.1"]}},"USER_ALLOWED_SCOPES":["fence","openid","user","data","admin","google_credentials","google_service_account","google_link","ga4gh_passport_v1"],"WHITE_LISTED_GOOGLE_PARENT_ORGS":[],"WHITE_LISTED_SERVICE_ACCOUNT_EMAILS":[],"WTF_CSRF_SECRET_KEY":"{{ENCRYPTION_KEY}}","dbGaP":[{"decrypt_key":"","enable_common_exchange_area_access":false,"info":{"host":"","password":"","port":22,"proxy":"","username":""},"parse_consent_code":true,"protocol":"sftp","study_common_exchange_areas":{"example":"test_common_exchange_area"},"study_to_resource_namespaces":{"_default":["/"],"test_common_exchange_area":["/dbgap/"]}}]}` | Private configuration settings for Fence app | +| FENCE_CONFIG | map | `{"ACCESS_TOKEN_COOKIE_NAME":"access_token","ACCESS_TOKEN_EXPIRES_IN":1200,"ALLOWED_GEN3_EMBEDDINGS_BULK_URL_PREFIXES":null,"ALLOWED_USER_SERVICE_ACCOUNT_DOMAINS":["developer.gserviceaccount.com","appspot.gserviceaccount.com","iam.gserviceaccount.com"],"ALLOW_GOOGLE_LINKING":true,"ALLOW_NEW_USER_ON_LOGIN":true,"APPLICATION_ROOT":"/user","APP_NAME":"Gen3 Data Commons","ARBORIST":"http://arborist-service","ASSUME_ROLE_CACHE_SECONDS":1800,"AUDIT_SERVICE":"http://audit-service","AUTHLIB_INSECURE_TRANSPORT":true,"AWS_CREDENTIALS":{},"AZ_BLOB_CONTAINER_URL":"https://myfakeblob.blob.core.windows.net/my-fake-container/","AZ_BLOB_CREDENTIALS":null,"BILLING_PROJECT_FOR_SA_CREDS":null,"BILLING_PROJECT_FOR_SIGNED_URLS":null,"CIRRUS_CFG":{"GOOGLE_ADMIN_EMAIL":"","GOOGLE_API_KEY":"","GOOGLE_APPLICATION_CREDENTIALS":"","GOOGLE_CLOUD_IDENTITY_ADMIN_EMAIL":"","GOOGLE_IDENTITY_DOMAIN":"","GOOGLE_PROJECT_ID":"","GOOGLE_STORAGE_CREDS":""},"CLIENT_ALLOWED_SCOPES":["openid","user","data","google_credentials","google_service_account","google_link","ga4gh_passport_v1"],"DATA_UPLOAD_BUCKET":"bucket1","DBGAP_ACCESSION_WITH_CONSENT_REGEX":"(?Pphs[0-9]+)(.(?Pv[0-9]+)){0,1}(.(?Pp[0-9]+)){0,1}.(?Pc[0-9]+)","DEBUG":false,"DEFAULT_LOGIN_IDP":"google","DEFAULT_LOGIN_URL":"{{BASE_URL}}/login/google","DEV_LOGIN_COOKIE_NAME":"dev_login","DREAM_CHALLENGE_GROUP":"DREAM","DREAM_CHALLENGE_TEAM":"DREAM","EMAIL_SERVER":"localhost","ENABLED_IDENTITY_PROVIDERS":{},"ENABLE_AUDIT_LOGS":{"login":false,"presigned_url":false},"ENABLE_AUTOMATIC_BILLING_PERMISSION_SA_CREDS":false,"ENABLE_AUTOMATIC_BILLING_PERMISSION_SIGNED_URLS":false,"ENABLE_CSRF_PROTECTION":true,"ENABLE_DB_MIGRATION":true,"ENABLE_PROMETHEUS_METRICS":false,"ENABLE_VISA_UPDATE_CRON":false,"ENCRYPTION_KEY":"REPLACEME","GA4GH_VISA_ISSUER_ALLOWLIST":["{{BASE_URL}}","https://sts.nih.gov","https://stsstg.nih.gov"],"GEN3_EMBEDDINGS_API_REGEX":"/vectorstore/collections/(?P[^/]+)/embeddings/(?P[^/]+)","GEN3_PASSPORT_EXPIRES_IN":43200,"GLOBAL_PARSE_VISAS_ON_LOGIN":false,"GOOGLE_ACCOUNT_ACCESS_EXPIRES_IN":86400,"GOOGLE_BULK_UPDATES":false,"GOOGLE_GROUP_PREFIX":"","GOOGLE_MANAGED_SERVICE_ACCOUNT_DOMAINS":["dataflow-service-producer-prod.iam.gserviceaccount.com","cloudbuild.gserviceaccount.com","cloud-ml.google.com.iam.gserviceaccount.com","container-engine-robot.iam.gserviceaccount.com","dataflow-service-producer-prod.iam.gserviceaccount.com","sourcerepo-service-accounts.iam.gserviceaccount.com","dataproc-accounts.iam.gserviceaccount.com","gae-api-prod.google.com.iam.gserviceaccount.com","genomics-api.google.com.iam.gserviceaccount.com","containerregistry.iam.gserviceaccount.com","container-analysis.iam.gserviceaccount.com","cloudservices.gserviceaccount.com","stackdriver-service.iam.gserviceaccount.com","appspot.gserviceaccount.com","partnercontent.gserviceaccount.com","trifacta-gcloud-prod.iam.gserviceaccount.com","gcf-admin-robot.iam.gserviceaccount.com","compute-system.iam.gserviceaccount.com","gcp-sa-websecurityscanner.iam.gserviceaccount.com","storage-transfer-service.iam.gserviceaccount.com","firebase-sa-management.iam.gserviceaccount.com","firebase-rules.iam.gserviceaccount.com","gcp-sa-cloudbuild.iam.gserviceaccount.com","gcp-sa-automl.iam.gserviceaccount.com","gcp-sa-datalabeling.iam.gserviceaccount.com","gcp-sa-cloudscheduler.iam.gserviceaccount.com"],"GOOGLE_SERVICE_ACCOUNT_KEY_FOR_URL_SIGNING_EXPIRES_IN":2592000,"GOOGLE_SERVICE_ACCOUNT_PREFIX":"","GOOGLE_USER_SERVICE_ACCOUNT_ACCESS_EXPIRES_IN":604800,"GUN_MAIL":{"datacommons.io":{"api_key":"","api_url":"https://api.mailgun.net/v3/mailgun.example.com","default_login":"postmaster@mailgun.example.com","smtp_hostname":"smtp.mailgun.org","smtp_password":""}},"HTTP_PROXY":{"host":null,"port":3128},"INDEXD":"http://indexd-service","INDEXD_PASSWORD":"","INDEXD_USERNAME":"fence","ITRUST_GLOBAL_LOGOUT":"https://auth.nih.gov/siteminderagent/smlogout.asp?mode=nih&AppReturnUrl=","LOGIN_OPTIONS":[{"desc":"description","idp":"google","name":"Login from Google"}],"LOGIN_REDIRECT_WHITELIST":[],"MAX_ACCESS_TOKEN_TTL":3600,"MAX_API_KEY_TTL":2592000,"MAX_BULK_CONTENT_GUIDS_COUNT":500,"MAX_PRESIGNED_URL_TTL":3600,"MAX_ROLE_SESSION_INCREASE":false,"MOCK_AUTH":false,"MOCK_GOOGLE_AUTH":false,"MOCK_STORAGE":false,"OAUTH2_JWT_ALG":"RS256","OAUTH2_JWT_ENABLED":true,"OAUTH2_JWT_ISS":"{{BASE_URL}}","OAUTH2_PROVIDER_ERROR_URI":"/api/oauth2/errors","OAUTH2_TOKEN_EXPIRES_IN":{"authorization_code":1200,"implicit":1200},"OPENID_CONNECT":{"cilogon":{"client_id":"","client_secret":"","discovery_url":"https://cilogon.org/.well-known/openid-configuration","mock":false,"mock_default_user":"http://cilogon.org/serverT/users/64703","redirect_url":"{{BASE_URL}}/login/cilogon/login/","scope":"openid email profile"},"cognito":{"client_id":"","client_secret":"","discovery_url":"https://cognito-idp.{REGION}.amazonaws.com/{USER-POOL-ID}/.well-known/openid-configuration","redirect_url":"{{BASE_URL}}/login/cognito/login/","scope":"openid email"},"fence":{"access_token_url":"{{api_base_url}}/oauth2/token","api_base_url":"","authorize_url":"{{api_base_url}}/oauth2/authorize","client_id":"","client_kwargs":{"redirect_uri":"{{BASE_URL}}/login/fence/login","scope":"openid"},"client_secret":"","mock":false,"mock_default_user":"test@example.com","name":"","refresh_token_url":"{{api_base_url}}/oauth2/token","shibboleth_discovery_url":"https://login.bionimbus.org/Shibboleth.sso/DiscoFeed"},"generic_oidc_idp":{"client_id":"","client_secret":"","discovery":{"authorization_endpoint":"","jwks_uri":"","token_endpoint":""},"discovery_url":"https://server.com/.well-known/openid-configuration","email_field":"","name":"some_idp","redirect_url":"{{BASE_URL}}/login/some_idp/login","scope":"","user_id_field":""},"google":{"client_id":"","client_secret":"","discovery_url":"https://accounts.google.com/.well-known/openid-configuration","mock":"","mock_default_user":"test@example.com","redirect_url":"{{BASE_URL}}/login/google/login/","scope":"openid email"},"microsoft":{"client_id":"","client_secret":"","discovery_url":"https://login.microsoftonline.com/organizations/v2.0/.well-known/openid-configuration","mock":false,"mock_default_user":"test@example.com","redirect_url":"{{BASE_URL}}/login/microsoft/login/","scope":"openid email"},"okta":{"client_id":"","client_secret":"","discovery_url":"","redirect_url":"{{BASE_URL}}/login/okta/login/","scope":"openid email"},"orcid":{"client_id":"","client_secret":"","discovery_url":"https://orcid.org/.well-known/openid-configuration","mock":false,"mock_default_user":"0000-0002-2601-8132","redirect_url":"{{BASE_URL}}/login/orcid/login/","scope":"openid"},"ras":{"client_id":"","client_secret":"","discovery_url":"https://sts.nih.gov/.well-known/openid-configuration","mock":false,"mock_default_user":"test@example.com","redirect_url":"{{BASE_URL}}/login/ras/callback","scope":"openid email profile ga4gh_passport_v1"},"shibboleth":{"client_id":"","client_secret":"","redirect_url":"{{BASE_URL}}/login/shib/login"},"synapse":{"client_id":"","client_secret":"","discovery_url":"","redirect_url":"","scope":"openid"}},"OVERRIDE_NGINX_RATE_LIMIT":18,"PRIVACY_POLICY_URL":null,"PROBLEM_USER_EMAIL_NOTIFICATION":{"admin":["admin@example.edu"],"content":"The Data Commons Framework utilizes dbGaP for data access authorization. Another member of a Google project you belong to ({}) is attempting to register a service account to the following additional datasets ({}). Please contact dbGaP to request access.\n","domain":"example.com","from":"do-not-reply@example.com","subject":"Account access error notification"},"PUSH_AUDIT_LOGS_CONFIG":{"aws_sqs_config":{"aws_cred":null,"region":null,"sqs_url":null},"type":"aws_sqs"},"RAS_REFRESH_EXPIRATION":1296000,"RAS_USERINFO_ENDPOINT":"/openid/connect/v1.1/userinfo","REFRESH_TOKEN_EXPIRES_IN":2592000,"REGISTERED_USERS_GROUP":"","REGISTER_USERS_ON":false,"REMOVE_SERVICE_ACCOUNT_EMAIL_NOTIFICATION":{"admin":["admin@example.edu"],"content":"Service accounts were removed from access control data because some users or service accounts of GCP Project {} are not authorized to access the data sets associated to the service accounts, or do not adhere to the security policies.\n","domain":"example.com","enable":false,"from":"do-not-reply@example.com","subject":"User service account removal notification"},"RENEW_ACCESS_TOKEN_BEFORE_EXPIRATION":false,"S3_BUCKETS":{},"SEND_FROM":"example@gmail.com","SEND_TO":"example@gmail.com","SERVICE_ACCOUNT_LIMIT":6,"SESSION_ALLOWED_SCOPES":["openid","user","credentials","data","admin","google_credentials","google_service_account","google_link","ga4gh_passport_v1"],"SESSION_COOKIE_DOMAIN":null,"SESSION_COOKIE_NAME":"fence","SESSION_COOKIE_SECURE":true,"SESSION_LIFETIME":28800,"SESSION_TIMEOUT":1800,"SHIBBOLETH_HEADER":"persistent_id","SSO_URL":"https://auth.nih.gov/affwebservices/public/saml2sso?SPID={{BASE_URL}}/shibboleth&RelayState=","STORAGE_CREDENTIALS":{},"SUPPORT_EMAIL_FOR_ERRORS":null,"SYNAPSE_AUTHZ_TTL":86400,"SYNAPSE_DISCOVERY_URL":null,"SYNAPSE_JWKS_URI":null,"SYNAPSE_URI":"https://repo-prod.prod.sagebase.org/auth/v1","TOKEN_PROJECTS_CUTOFF":10,"USERSYNC":{"fallback_to_dbgap_sftp":false,"sync_from_visas":false,"visa_types":{"ras":["https://ras.nih.gov/visas/v1","https://ras.nih.gov/visas/v1.1"]}},"USER_ALLOWED_SCOPES":["fence","openid","user","data","admin","google_credentials","google_service_account","google_link","ga4gh_passport_v1"],"WHITE_LISTED_GOOGLE_PARENT_ORGS":[],"WHITE_LISTED_SERVICE_ACCOUNT_EMAILS":[],"WTF_CSRF_SECRET_KEY":"{{ENCRYPTION_KEY}}","dbGaP":[{"decrypt_key":"","enable_common_exchange_area_access":false,"info":{"host":"","password":"","port":22,"proxy":"","username":""},"parse_consent_code":true,"protocol":"sftp","study_common_exchange_areas":{"example":"test_common_exchange_area"},"study_to_resource_namespaces":{"_default":["/"],"test_common_exchange_area":["/dbgap/"]}}]}` | Private configuration settings for Fence app | | FENCE_CONFIG.APP_NAME | string | `"Gen3 Data Commons"` | Name of the Fence app | | FENCE_CONFIG.AUTHLIB_INSECURE_TRANSPORT | bool | `true` | allow OIDC traffic on http for development. By default it requires https. WARNING: ONLY set to true when fence will be deployed in such a way that it will ONLY receive traffic from internal clients and can safely use HTTP. | | FENCE_CONFIG.CLIENT_ALLOWED_SCOPES | list | `["openid","user","data","google_credentials","google_service_account","google_link","ga4gh_passport_v1"]` | These are the *possible* scopes a client can be given, NOT scopes that are given to all clients. You can be more restrictive during client creation | diff --git a/helm/fence/templates/fence-deployment.yaml b/helm/fence/templates/fence-deployment.yaml index ab7a59dc7..4af4c0428 100644 --- a/helm/fence/templates/fence-deployment.yaml +++ b/helm/fence/templates/fence-deployment.yaml @@ -33,6 +33,8 @@ spec: labels: authprovider: "yes" netnolimit: "yes" + internet: "yes" + linklocal: "yes" userhelper: "yes" {{- include "fence.selectorLabels" . | nindent 8 }} {{- include "common.extraLabels" . | nindent 8 }} @@ -46,6 +48,8 @@ spec: {{- toYaml .Values.volumes | nindent 8 }} containers: - name: fence + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} ports: @@ -88,6 +92,8 @@ spec: {{- toYaml .Values.volumeMounts | nindent 12 }} initContainers: - name: fence-init + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} ports: @@ -133,4 +139,4 @@ spec: {{- with .Values.tolerations }} tolerations: {{- toYaml . | nindent 8 }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/helm/fence/templates/presigned-url-fence.yaml b/helm/fence/templates/presigned-url-fence.yaml index a230188b2..6d356f8da 100644 --- a/helm/fence/templates/presigned-url-fence.yaml +++ b/helm/fence/templates/presigned-url-fence.yaml @@ -35,6 +35,8 @@ spec: authprovder: "yes" netnolimit: "yes" public: "yes" + internet: "yes" + linklocal: "yes" userhelper: "yes" spec: serviceAccountName: {{ include "fence.serviceAccountName" . }} @@ -132,4 +134,4 @@ spec: - gen3job policyTypes: - Egress -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm/fence/values.yaml b/helm/fence/values.yaml index ed773feb0..cb628a52c 100644 --- a/helm/fence/values.yaml +++ b/helm/fence/values.yaml @@ -1226,6 +1226,12 @@ FENCE_CONFIG: # The maximum lifetime of a Gen3 passport in seconds GEN3_PASSPORT_EXPIRES_IN: 43200 + # Max number of GUIDs to request content retrieval for + # NOTE: This is limited by the MAX allowed for any underlying bulk content requests + # e.g. if the Gen3 Embeddings service is configured to allow max 1000, then this + # can't be more than that for embeddings content resolutions + MAX_BULK_CONTENT_GUIDS_COUNT: 500 + ######################################################################################## # OPTIONAL CONFIGURATIONS # ######################################################################################## @@ -1436,6 +1442,19 @@ FENCE_CONFIG: # this is the password which fence uses to make authenticated requests to indexd INDEXD_PASSWORD: "" + # allowlist for Gen3 Embeddings URL resolutions. + # use this to provide the prefix for allowed URLs for Gen3 Embeddings + # for example, if you want to be able to resolve Gen3 Embeddings from + # https://example.com/ai/vectorstore/collections/... + # you need to add "https://example.com/ai" to the allowlist here + # basically, everything BEFORE "/vectorstore/collections/..." + ALLOWED_GEN3_EMBEDDINGS_BULK_URL_PREFIXES: + # - 'https://example.com/ai' + + # this is used to identify indexed record URLs which are pointing to embeddings, and thus support bulk content retrieval + # unless the Gen3 Embeddings API changes, you should NOT change this default + GEN3_EMBEDDINGS_API_REGEX: '/vectorstore/collections/(?P[^/]+)/embeddings/(?P[^/]+)' + # ////////////////////////////////////////////////////////////////////////////////////// # AZURE STORAGE BLOB CONFIGURATION # - Support Azure Blob Data Access Methods diff --git a/helm/frontend-framework/Chart.yaml b/helm/frontend-framework/Chart.yaml index 5d2e3ca71..15029725d 100644 --- a/helm/frontend-framework/Chart.yaml +++ b/helm/frontend-framework/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.29 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "develop" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/frontend-framework/README.md b/helm/frontend-framework/README.md index a4c4a3184..c2c5d86b2 100644 --- a/helm/frontend-framework/README.md +++ b/helm/frontend-framework/README.md @@ -1,14 +1,18 @@ # frontend-framework -![Version: 0.1.29](https://img.shields.io/badge/Version-0.1.29-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: develop](https://img.shields.io/badge/AppVersion-develop-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: develop](https://img.shields.io/badge/AppVersion-develop-informational?style=flat-square) A Helm chart for the gen3 frontend framework +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values diff --git a/helm/funnel/Chart.yaml b/helm/funnel/Chart.yaml index 8061230b7..c8657cc4b 100644 --- a/helm/funnel/Chart.yaml +++ b/helm/funnel/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.26 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -24,15 +24,5 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common - - name: funnel - # NOTE: - # When updating this version: - # 1) Run `helm dependency update` in this directory to generate a new .tgz file - # 2) Commit the updated .tgz file into gen3-helm in the same PR - # - # ArgoCD relies on this checked-in .tgz reference — if it's missing, - # Funnel will not be deployed as a dependency. - version: 0.1.99-rc.36 - repository: "https://calypr.github.io/helm-charts" diff --git a/helm/funnel/README.md b/helm/funnel/README.md index 1f168fe79..8b25fa75c 100644 --- a/helm/funnel/README.md +++ b/helm/funnel/README.md @@ -1,82 +1,158 @@ # funnel -![Version: 0.1.26](https://img.shields.io/badge/Version-0.1.26-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for Kubernetes +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | -| https://calypr.github.io/helm-charts | funnel | 0.1.99-rc.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| +| AWSBatch.DisableReconciler | bool | `true` | | +| AWSBatch.JobDefinition | string | `"funnel-job-def"` | | +| AWSBatch.JobQueue | string | `"funnel-job-queue"` | | +| AWSBatch.Key | string | `""` | | +| AWSBatch.ReconcileRate | string | `"10s"` | | +| AWSBatch.Region | string | `""` | | +| AWSBatch.Secret | string | `""` | | +| AmazonS3.AWSConfig.Key | string | `""` | | +| AmazonS3.AWSConfig.MaxRetries | int | `10` | | +| AmazonS3.AWSConfig.Secret | string | `""` | | +| AmazonS3.Disabled | bool | `false` | | +| AmazonS3.SSE.CustomerKeyFile | string | `""` | | +| AmazonS3.SSE.KMSKey | string | `""` | | +| BoltDB | object | `{"Path":"./funnel-work-dir/funnel.db"}` | Local file database configuration. | +| Compute | string | `"kubernetes"` | | +| Database | string | `"postgres"` | | +| Datastore.CredentialsFile | string | `""` | | +| Datastore.Project | string | `""` | | +| Datastore.Timeout.duration | string | `"300s"` | | +| DynamoDB.AWSConfig.Key | string | `""` | | +| DynamoDB.AWSConfig.Region | string | `""` | | +| DynamoDB.AWSConfig.Secret | string | `""` | | +| DynamoDB.TableBasename | string | `"funnel"` | | +| Elastic.IndexPrefix | string | `"funnel"` | | +| Elastic.URL | string | `"http://localhost:9200"` | | +| EventWriters[0] | string | `"postgres"` | | +| EventWriters[1] | string | `"log"` | | +| FTPStorage.Disabled | bool | `false` | | +| FTPStorage.Password | string | `"anonymous"` | | +| FTPStorage.Timeout | string | `"10s"` | | +| FTPStorage.User | string | `"anonymous"` | | +| GoogleStorage.CredentialsFile | string | `""` | | +| GoogleStorage.Disabled | bool | `false` | | +| GridEngine.Template | string | `"#!bin/bash\n#$ -N {{.TaskId}}\n#$ -o {{.WorkDir}}/funnel-stdout\n#$ -e {{.WorkDir}}/funnel-stderr\n#$ -l nodes=1\n{{if ne .Cpus 0 -}}\n{{printf \"#$ -pe mpi %d\" .Cpus}}\n{{- end}}\n{{if ne .RamGb 0.0 -}}\n{{printf \"#$ -l h_vmem=%.0fG\" .RamGb}}\n{{- end}}\n{{if ne .DiskGb 0.0 -}}\n{{printf \"#$ -l h_fsize=%.0fG\" .DiskGb}}\n{{- end}}\n\n{{.Executable}} worker run --config {{.Config}} --taskID {{.TaskId}}\n"` | | +| GridEngine.TemplateFile | string | `""` | | +| HTCondor | object | `{"DisableReconciler":true,"ReconcileRate":"10s","Template":"universe = vanilla\ngetenv = True\nexecutable = {{.Executable}}\narguments = worker run --config {{.Config}} --task-id {{.TaskId}}\nlog = {{.WorkDir}}/condor-event-log\nerror = {{.WorkDir}}/funnel-stderr\noutput = {{.WorkDir}}/funnel-stdout\nshould_transfer_files = YES\nwhen_to_transfer_output = ON_EXIT_OR_EVICT\n{{if ne .Cpus 0 -}}\n{{printf \"request_cpus = %d\" .Cpus}}\n{{- end}}\n{{if ne .RamGb 0.0 -}}\n{{printf \"request_memory = %.0f GB\" .RamGb}}\n{{- end}}\n{{if ne .DiskGb 0.0 -}}\n{{printf \"request_disk = %.0f GB\" .DiskGb}}\n{{- end}}\n\nqueue\n","TemplateFile":""}` | HTCondor compute backend configuration. | +| HTTPStorage.Timeout | string | `"30s"` | | +| Kafka.Topic | string | `"funnel"` | | +| Kubernetes.DisableJobCleanup | bool | `false` | | +| Kubernetes.DisableReconciler | bool | `false` | | +| Kubernetes.Executor.Annotations."karpenter.sh/do-not-disrupt" | string | `"true"` | | +| Kubernetes.Executor.PriorityClassName | string | `""` | | +| Kubernetes.Executor.backoffLimit | int | `0` | | +| Kubernetes.Executor.completions | int | `1` | | +| Kubernetes.Executor.restartPolicy | string | `"Never"` | | +| Kubernetes.ExecutorTemplate | string | `""` | | +| Kubernetes.ForbiddenPathPrefixes | list | `[]` | Path prefixes that Funnel's Kubernetes backend must reject when creating worker pods. | +| Kubernetes.JobsNamespace | string | `""` | | +| Kubernetes.Namespace | string | `""` | | +| Kubernetes.NodeSelector | object | `{}` | | +| Kubernetes.PVCTemplate | string | `""` | | +| Kubernetes.PVTemplate | string | `""` | | +| Kubernetes.ReconcileRate | string | `"120s"` | | +| Kubernetes.Resources.Defaults.Cpus | string | `"1000m"` | | +| Kubernetes.Resources.Defaults.DiskGb | string | `"512Mi"` | | +| Kubernetes.Resources.Defaults.RamGb | string | `"512Mi"` | | +| Kubernetes.Resources.Limits.Cpus | string | `"8000m"` | | +| Kubernetes.Resources.Limits.DiskGb | string | `"4096Mi"` | | +| Kubernetes.Resources.Limits.RamGb | string | `"4096Mi"` | | +| Kubernetes.ServiceAccount | string | `""` | | +| Kubernetes.Storage.type | string | `"mountpoint_s3"` | | +| Kubernetes.Timeout.duration | string | `"300s"` | | +| Kubernetes.Tolerations | list | `[]` | | +| Kubernetes.Worker.Annotations."karpenter.sh/do-not-disrupt" | string | `"true"` | | +| Kubernetes.Worker.PriorityClassName | string | `"system-cluster-critical"` | | +| Kubernetes.Worker.backoffLimit | int | `0` | | +| Kubernetes.Worker.completions | int | `1` | | +| Kubernetes.Worker.restartPolicy | string | `"Never"` | | +| Kubernetes.WorkerTemplate | string | `""` | | +| LocalStorage | object | `{"AllowedDirs":["./"]}` | Local file system storage configuration. | +| Logger.Formatter | string | `"json"` | | +| Logger.Level | string | `"debug"` | | +| Logger.OutputFile | string | `""` | | +| Logger.TextFormat.ForceColors | bool | `true` | | +| Logger.TextFormat.FullTimestamp | bool | `true` | | +| Logger.TextFormat.TimestampFormat | string | `"2006-01-02T15:04:05Z07:00"` | | +| Node.ID | string | `""` | | +| Node.Resources.Cpus | int | `0` | | +| Node.Resources.DiskGb | float | `0` | | +| Node.Resources.RamGb | float | `0` | | +| Node.Timeout.disabled | bool | `true` | | +| Node.UpdateRate | string | `"5s"` | | +| PBS.DisableReconciler | bool | `true` | | +| PBS.ReconcileRate | string | `"10s"` | | +| PBS.Template | string | `"#!bin/bash\n#PBS -N {{.TaskId}}\n#PBS -o {{.WorkDir}}/funnel-stdout\n#PBS -e {{.WorkDir}}/funnel-stderr\n{{if ne .Cpus 0 -}}\n{{printf \"#PBS -l nodes=1:ppn=%d\" .Cpus}}\n{{- end}}\n{{if ne .RamGb 0.0 -}}\n{{printf \"#PBS -l mem=%.0fgb\" .RamGb}}\n{{- end}}\n{{if ne .DiskGb 0.0 -}}\n{{printf \"#PBS -l file=%.0fgb\" .DiskGb}}\n{{- end}}\n\n{{.Executable}} worker run --config {{.Config}} --taskID {{.TaskId}}\n"` | | +| PBS.TemplateFile | string | `""` | | +| Plugins.Params.OidcClientId | string | `"FUNNEL_PLUGIN_OIDC_CLIENT_ID_PLACEHOLDER"` | | +| Plugins.Params.OidcClientSecret | string | `"FUNNEL_PLUGIN_OIDC_CLIENT_SECRET_PLACEHOLDER"` | | +| Plugins.Params.S3Url | string | `"FUNNEL_PLUGIN_S3URL_PLACEHOLDER"` | | +| Plugins.Path | string | `"plugin-binaries/auth-plugin"` | | +| Postgres.AdminPassword | string | `"example"` | | +| Postgres.AdminUser | string | `"postgres"` | | +| Postgres.Database | string | `"FUNNEL_POSTGRES_DATABASE_PLACEHOLDER"` | | +| Postgres.Host | string | `"FUNNEL_POSTGRES_HOST_PLACEHOLDER"` | | +| Postgres.Password | string | `"FUNNEL_POSTGRES_PASSWORD_PLACEHOLDER"` | | +| Postgres.Timeout.duration | string | `"300s"` | | +| Postgres.User | string | `"FUNNEL_POSTGRES_USER_PLACEHOLDER"` | | +| RPCClient.MaxRetries | int | `10` | | +| RPCClient.ServerAddress | string | `"localhost:9090"` | | +| RPCClient.Timeout.duration | string | `"60s"` | | +| Scheduler.NodeInitTimeout.duration | string | `"300s"` | | +| Scheduler.NodePingTimeout.duration | string | `"60s"` | | +| Scheduler.ScheduleChunk | int | `10` | | +| Scheduler.ScheduleRate | string | `"1s"` | | +| Server.DisableHTTPCache | bool | `true` | | +| Server.HTTPPort | string | `"8000"` | | +| Server.HostName | string | `"funnel"` | | +| Server.RPCPort | string | `"9090"` | | +| Slurm.DisableReconciler | bool | `true` | | +| Slurm.ReconcileRate | string | `"10s"` | | +| Slurm.Template | string | `"#!/bin/bash\n#SBATCH --job-name {{.TaskId}}\n#SBATCH --ntasks 1\n#SBATCH --error {{.WorkDir}}/funnel-stderr\n#SBATCH --output {{.WorkDir}}/funnel-stdout\n{{if ne .Cpus 0 -}}\n{{printf \"#SBATCH --cpus-per-task %d\" .Cpus}}\n{{- end}}\n{{if ne .RamGb 0.0 -}}\n{{printf \"#SBATCH --mem %.0fGB\" .RamGb}}\n{{- end}}\n{{if ne .DiskGb 0.0 -}}\n{{printf \"#SBATCH --tmp %.0fGB\" .DiskGb}}\n{{- end}}\n\n{{.Executable}} worker run --config {{.Config}} --taskID {{.TaskId}}\n"` | | +| Slurm.TemplateFile | string | `""` | | +| Swift.AuthURL | string | `""` | | +| Swift.ChunkSizeBytes | int | `500000000` | | +| Swift.Disabled | bool | `false` | | +| Swift.Password | string | `""` | | +| Swift.RegionName | string | `""` | | +| Swift.TenantID | string | `""` | | +| Swift.TenantName | string | `""` | | +| Swift.UserName | string | `""` | | +| Worker.LeaveWorkDir | bool | `true` | | +| Worker.LogTailSize | int | `10000` | | +| Worker.LogUpdateRate | string | `"5s"` | | +| Worker.MaxParallelTransfers | int | `10` | | +| Worker.PollingRate | string | `"5s"` | | +| Worker.WorkDir | string | `"./funnel-work-dir"` | | +| authenticationSource | string | `"pod"` | | +| cleanup.enabled | bool | `false` | | +| cleanup.schedule | string | `""` | | +| cleanup.scheduleOffsetMinutes | int | `0` | | | criticalService | string | `"false"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | | externalSecrets | map | `{"createFunnelOidcClientSecret":true,"dbcreds":"","funnelOidcClient":null}` | External Secrets settings. | | externalSecrets.createFunnelOidcClientSecret | bool | `true` | Whether to create the Funnel OIDC client secret using the oidc job. | | externalSecrets.dbcreds | string | `""` | Name of the secret that will be created in secrets manager | | externalSecrets.funnelOidcClient | string | `nil` | Will override the name of the aws secrets manager secret. Default is "funnel-oidc-client". | -| funnel.Database | string | `"postgres"` | | -| funnel.EventWriters[0] | string | `"postgres"` | | -| funnel.EventWriters[1] | string | `"log"` | | -| funnel.Kubernetes.Executor.Annotations."karpenter.sh/do-not-disrupt" | string | `"true"` | | -| funnel.Kubernetes.Executor.backoffLimit | int | `0` | | -| funnel.Kubernetes.Executor.restartPolicy | string | `"Never"` | | -| funnel.Kubernetes.ReconcileRate | string | `"120s"` | | -| funnel.Kubernetes.Timeout.duration | string | `"300s"` | | -| funnel.Kubernetes.Worker.Annotations."karpenter.sh/do-not-disrupt" | string | `"true"` | | -| funnel.Kubernetes.Worker.PriorityClassName | string | `"system-cluster-critical"` | | -| funnel.Kubernetes.Worker.backoffLimit | int | `1` | | -| funnel.Kubernetes.Worker.restartPolicy | string | `"Never"` | | -| funnel.Logger.Level | string | `"info"` | | -| funnel.Plugins.Params.OidcClientId | string | `"FUNNEL_PLUGIN_OIDC_CLIENT_ID_PLACEHOLDER"` | | -| funnel.Plugins.Params.OidcClientSecret | string | `"FUNNEL_PLUGIN_OIDC_CLIENT_SECRET_PLACEHOLDER"` | | -| funnel.Plugins.Params.S3Url | string | `"FUNNEL_PLUGIN_S3URL_PLACEHOLDER"` | | -| funnel.Plugins.Path | string | `"plugin-binaries/auth-plugin"` | | -| funnel.Postgres.Database | string | `"FUNNEL_POSTGRES_DATABASE_PLACEHOLDER"` | | -| funnel.Postgres.Host | string | `"FUNNEL_POSTGRES_HOST_PLACEHOLDER"` | | -| funnel.Postgres.Password | string | `"FUNNEL_POSTGRES_PASSWORD_PLACEHOLDER"` | | -| funnel.Postgres.User | string | `"FUNNEL_POSTGRES_USER_PLACEHOLDER"` | | -| funnel.Worker.LeaveWorkDir | bool | `true` | | -| funnel.image | map | `{"initContainers":[{"command":["cp","/app/build/plugins/authorizer","/opt/funnel/plugin-binaries/auth-plugin"],"image":"quay.io/cdis/funnel-gen3-plugin","name":"plugin","pullPolicy":"Always","tag":"main-gen3","volumeMounts":[{"mountPath":"/opt/funnel/plugin-binaries","name":"plugin-volume"}]},{"args":["-c","# Create a funnel-patched.conf since /etc/config/funnel.conf is readonly\nCONFIG=/tmp/funnel-patched.conf\ncp /etc/config/funnel.conf $CONFIG\n\nnamespace=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)\nJOBS_NAMESPACE=workflow-pods-$namespace\nS3_URL=gen3-workflow-service.$namespace.svc.cluster.local\nDB_HOST=$DB_HOST:5432\n\n# `Kubernetes.JobsNamespace` has to be configured manually because of templating\n# limitations. This ensures it is configured to the value that is hardcoded elsewhere.\nconfigured=$(yq -r '.Kubernetes.JobsNamespace' \"$CONFIG\")\nif [[ \"$configured\" != \"$JOBS_NAMESPACE\" ]]; then\n echo \"ERROR: funnel.Kubernetes.JobsNamespace is set to '$configured' instead of '$JOBS_NAMESPACE'. Please fix the configuration\" >&2\n exit 1\nfi\n\necho \"======= Funnel configuration =======\"\necho \" Kubernetes.JobsNamespace : $JOBS_NAMESPACE\"\necho \" Plugins.Params.OidcClientId: $FUNNEL_OIDC_CLIENT_ID\"\necho \" Plugins.Params.S3Url : $S3_URL\"\necho \" Postgres.Host : $DB_HOST\"\necho \" Postgres.Database : $DB_DATABASE\"\necho \" Postgres.User : $DB_USER\"\necho \"====================================\"\n\n# Replace placeholders with actual values (in-place)\nsed -i \"s|FUNNEL_PLUGIN_OIDC_CLIENT_ID_PLACEHOLDER|${FUNNEL_OIDC_CLIENT_ID}|g\" $CONFIG\nsed -i \"s|FUNNEL_PLUGIN_OIDC_CLIENT_SECRET_PLACEHOLDER|${FUNNEL_OIDC_CLIENT_SECRET}|g\" $CONFIG\nsed -i \"s|FUNNEL_PLUGIN_S3URL_PLACEHOLDER|${S3_URL}|g\" $CONFIG\nsed -i \"s/FUNNEL_POSTGRES_HOST_PLACEHOLDER/${DB_HOST}/g\" $CONFIG\nsed -i \"s/FUNNEL_POSTGRES_DATABASE_PLACEHOLDER/${DB_DATABASE}/g\" $CONFIG\nsed -i \"s/FUNNEL_POSTGRES_USER_PLACEHOLDER/${DB_USER}/g\" $CONFIG\nsed -i \"s/FUNNEL_POSTGRES_PASSWORD_PLACEHOLDER/${DB_PASSWORD}/g\" $CONFIG\n"],"command":["/bin/bash"],"env":[{"name":"FUNNEL_OIDC_CLIENT_ID","valueFrom":{"secretKeyRef":{"key":"client_id","name":"funnel-oidc-client","optional":false}}},{"name":"FUNNEL_OIDC_CLIENT_SECRET","valueFrom":{"secretKeyRef":{"key":"client_secret","name":"funnel-oidc-client","optional":false}}},{"name":"DB_HOST","valueFrom":{"secretKeyRef":{"key":"host","name":"funnel-dbcreds","optional":false}}},{"name":"DB_USER","valueFrom":{"secretKeyRef":{"key":"username","name":"funnel-dbcreds","optional":false}}},{"name":"DB_PASSWORD","valueFrom":{"secretKeyRef":{"key":"password","name":"funnel-dbcreds","optional":false}}},{"name":"DB_DATABASE","valueFrom":{"secretKeyRef":{"key":"database","name":"funnel-dbcreds","optional":false}}}],"image":"quay.io/cdis/awshelper","name":"config-updater","tag":"master","volumeMounts":[{"mountPath":"/tmp","name":"funnel-patched-config-volume"},{"mountPath":"/etc/config/funnel.conf","name":"funnel-config-volume","subPath":"funnel-server.yaml"}]}],"pullPolicy":"Always","repository":"quay.io/ohsu-comp-bio/funnel"}` | Configuration for the Funnel container image. | -| funnel.image.initContainers | map | `[{"command":["cp","/app/build/plugins/authorizer","/opt/funnel/plugin-binaries/auth-plugin"],"image":"quay.io/cdis/funnel-gen3-plugin","name":"plugin","pullPolicy":"Always","tag":"main-gen3","volumeMounts":[{"mountPath":"/opt/funnel/plugin-binaries","name":"plugin-volume"}]},{"args":["-c","# Create a funnel-patched.conf since /etc/config/funnel.conf is readonly\nCONFIG=/tmp/funnel-patched.conf\ncp /etc/config/funnel.conf $CONFIG\n\nnamespace=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)\nJOBS_NAMESPACE=workflow-pods-$namespace\nS3_URL=gen3-workflow-service.$namespace.svc.cluster.local\nDB_HOST=$DB_HOST:5432\n\n# `Kubernetes.JobsNamespace` has to be configured manually because of templating\n# limitations. This ensures it is configured to the value that is hardcoded elsewhere.\nconfigured=$(yq -r '.Kubernetes.JobsNamespace' \"$CONFIG\")\nif [[ \"$configured\" != \"$JOBS_NAMESPACE\" ]]; then\n echo \"ERROR: funnel.Kubernetes.JobsNamespace is set to '$configured' instead of '$JOBS_NAMESPACE'. Please fix the configuration\" >&2\n exit 1\nfi\n\necho \"======= Funnel configuration =======\"\necho \" Kubernetes.JobsNamespace : $JOBS_NAMESPACE\"\necho \" Plugins.Params.OidcClientId: $FUNNEL_OIDC_CLIENT_ID\"\necho \" Plugins.Params.S3Url : $S3_URL\"\necho \" Postgres.Host : $DB_HOST\"\necho \" Postgres.Database : $DB_DATABASE\"\necho \" Postgres.User : $DB_USER\"\necho \"====================================\"\n\n# Replace placeholders with actual values (in-place)\nsed -i \"s|FUNNEL_PLUGIN_OIDC_CLIENT_ID_PLACEHOLDER|${FUNNEL_OIDC_CLIENT_ID}|g\" $CONFIG\nsed -i \"s|FUNNEL_PLUGIN_OIDC_CLIENT_SECRET_PLACEHOLDER|${FUNNEL_OIDC_CLIENT_SECRET}|g\" $CONFIG\nsed -i \"s|FUNNEL_PLUGIN_S3URL_PLACEHOLDER|${S3_URL}|g\" $CONFIG\nsed -i \"s/FUNNEL_POSTGRES_HOST_PLACEHOLDER/${DB_HOST}/g\" $CONFIG\nsed -i \"s/FUNNEL_POSTGRES_DATABASE_PLACEHOLDER/${DB_DATABASE}/g\" $CONFIG\nsed -i \"s/FUNNEL_POSTGRES_USER_PLACEHOLDER/${DB_USER}/g\" $CONFIG\nsed -i \"s/FUNNEL_POSTGRES_PASSWORD_PLACEHOLDER/${DB_PASSWORD}/g\" $CONFIG\n"],"command":["/bin/bash"],"env":[{"name":"FUNNEL_OIDC_CLIENT_ID","valueFrom":{"secretKeyRef":{"key":"client_id","name":"funnel-oidc-client","optional":false}}},{"name":"FUNNEL_OIDC_CLIENT_SECRET","valueFrom":{"secretKeyRef":{"key":"client_secret","name":"funnel-oidc-client","optional":false}}},{"name":"DB_HOST","valueFrom":{"secretKeyRef":{"key":"host","name":"funnel-dbcreds","optional":false}}},{"name":"DB_USER","valueFrom":{"secretKeyRef":{"key":"username","name":"funnel-dbcreds","optional":false}}},{"name":"DB_PASSWORD","valueFrom":{"secretKeyRef":{"key":"password","name":"funnel-dbcreds","optional":false}}},{"name":"DB_DATABASE","valueFrom":{"secretKeyRef":{"key":"database","name":"funnel-dbcreds","optional":false}}}],"image":"quay.io/cdis/awshelper","name":"config-updater","tag":"master","volumeMounts":[{"mountPath":"/tmp","name":"funnel-patched-config-volume"},{"mountPath":"/etc/config/funnel.conf","name":"funnel-config-volume","subPath":"funnel-server.yaml"}]}]` | Configuration for the Funnel init container. | -| funnel.image.initContainers[0].command | list | `["cp","/app/build/plugins/authorizer","/opt/funnel/plugin-binaries/auth-plugin"]` | Arguments to pass to the init container. | -| funnel.image.initContainers[0].image | string | `"quay.io/cdis/funnel-gen3-plugin"` | The Docker image repository for the Funnel init/plugin container. | -| funnel.image.initContainers[0].pullPolicy | string | `"Always"` | When to pull the image. This value should be "Always" to ensure the latest image is used. | -| funnel.image.initContainers[0].tag | string | `"main-gen3"` | The Docker image tag for the Funnel init/plugin container. | -| funnel.image.pullPolicy | string | `"Always"` | When to pull the image. This value should be "Always" to ensure the latest image is used. | -| funnel.image.repository | string | `"quay.io/ohsu-comp-bio/funnel"` | The Docker image repository for the Funnel service. | -| funnel.postgresql.enabled | bool | `false` | | -| funnel.resources.requests.ephemeral_storage | string | `"2Gi"` | | -| funnel.resources.requests.memory | string | `"2Gi"` | | -| funnel.volumeMounts[0].mountPath | string | `"/etc/config/funnel-server.yaml"` | | -| funnel.volumeMounts[0].name | string | `"funnel-patched-config-volume"` | | -| funnel.volumeMounts[0].subPath | string | `"funnel-patched.conf"` | | -| funnel.volumeMounts[1].mountPath | string | `"/etc/config/oidc"` | | -| funnel.volumeMounts[1].name | string | `"funnel-oidc-volume"` | | -| funnel.volumeMounts[1].readOnly | bool | `true` | | -| funnel.volumeMounts[2].mountPath | string | `"/etc/funnel/templates"` | | -| funnel.volumeMounts[2].name | string | `"worker-templates-volume"` | | -| funnel.volumeMounts[3].mountPath | string | `"/opt/funnel/plugin-binaries"` | | -| funnel.volumeMounts[3].name | string | `"plugin-volume"` | | -| funnel.volumes[0].configMap.name | string | `"funnel-server-config"` | | -| funnel.volumes[0].name | string | `"funnel-config-volume"` | | -| funnel.volumes[1].name | string | `"funnel-oidc-volume"` | | -| funnel.volumes[1].secret.items[0].key | string | `"client_id"` | | -| funnel.volumes[1].secret.items[0].path | string | `"client_id"` | | -| funnel.volumes[1].secret.items[1].key | string | `"client_secret"` | | -| funnel.volumes[1].secret.items[1].path | string | `"client_secret"` | | -| funnel.volumes[1].secret.secretName | string | `"funnel-oidc-client"` | | -| funnel.volumes[2].configMap.name | string | `"funnel-worker-templates"` | | -| funnel.volumes[2].name | string | `"worker-templates-volume"` | | -| funnel.volumes[3].emptyDir | object | `{}` | | -| funnel.volumes[3].name | string | `"plugin-volume"` | | -| funnel.volumes[4].emptyDir | object | `{}` | | -| funnel.volumes[4].name | string | `"funnel-patched-config-volume"` | | | global.aws.awsAccessKeyId | string | `nil` | Credentials for AWS stuff. | | global.aws.awsSecretAccessKey | string | `nil` | Credentials for AWS stuff. | | global.aws.enabled | bool | `false` | Set to true if deploying to AWS. Controls ingress annotations. | @@ -105,6 +181,15 @@ A Helm chart for Kubernetes | global.topologySpread.enabled | bool | `false` | Whether to enable topology spread constraints for all subcharts that support it. | | global.topologySpread.maxSkew | int | `1` | The maxSkew to use for topology spread constraints. Defaults to 1. | | global.topologySpread.topologyKey | string | `"topology.kubernetes.io/zone"` | The topology key to use for spreading. Defaults to "topology.kubernetes.io/zone". | +| image.initContainers | map | `[{"command":["cp","/app/build/plugins/authorizer","/opt/funnel/plugin-binaries/auth-plugin"],"image":"quay.io/cdis/funnel-gen3-plugin","name":"plugin","pullPolicy":"Always","tag":"main-gen3","volumeMounts":[{"mountPath":"/opt/funnel/plugin-binaries","name":"plugin-volume"}]},{"args":["-c","# Create a funnel-patched.conf since /etc/config/funnel.conf is readonly\nCONFIG=/tmp/funnel-patched.conf\ncp /etc/config/funnel.conf $CONFIG\n\nnamespace=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)\nJOBS_NAMESPACE=workflow-pods-$namespace\nS3_URL=gen3-workflow-service.$namespace.svc.cluster.local\nDB_HOST=$DB_HOST:5432\n\n# `Kubernetes.JobsNamespace` has to be configured manually because of templating\n# limitations. This ensures it is configured to the value that is hardcoded elsewhere.\nconfigured=$(yq -r '.Kubernetes.JobsNamespace' \"$CONFIG\")\nif [[ \"$configured\" != \"$JOBS_NAMESPACE\" ]]; then\n echo \"ERROR: funnel.Kubernetes.JobsNamespace is set to '$configured' instead of '$JOBS_NAMESPACE'. Please fix the configuration\" >&2\n exit 1\nfi\n\necho \"======= Funnel configuration =======\"\necho \" Kubernetes.JobsNamespace : $JOBS_NAMESPACE\"\necho \" Plugins.Params.OidcClientId: $FUNNEL_OIDC_CLIENT_ID\"\necho \" Plugins.Params.S3Url : $S3_URL\"\necho \" Postgres.Host : $DB_HOST\"\necho \" Postgres.Database : $DB_DATABASE\"\necho \" Postgres.User : $DB_USER\"\necho \"====================================\"\n\n# Replace placeholders with actual values (in-place)\nsed -i \"s|FUNNEL_PLUGIN_OIDC_CLIENT_ID_PLACEHOLDER|${FUNNEL_OIDC_CLIENT_ID}|g\" $CONFIG\nsed -i \"s|FUNNEL_PLUGIN_OIDC_CLIENT_SECRET_PLACEHOLDER|${FUNNEL_OIDC_CLIENT_SECRET}|g\" $CONFIG\nsed -i \"s|FUNNEL_PLUGIN_S3URL_PLACEHOLDER|${S3_URL}|g\" $CONFIG\nsed -i \"s/FUNNEL_POSTGRES_HOST_PLACEHOLDER/${DB_HOST}/g\" $CONFIG\nsed -i \"s/FUNNEL_POSTGRES_DATABASE_PLACEHOLDER/${DB_DATABASE}/g\" $CONFIG\nsed -i \"s/FUNNEL_POSTGRES_USER_PLACEHOLDER/${DB_USER}/g\" $CONFIG\nsed -i \"s/FUNNEL_POSTGRES_PASSWORD_PLACEHOLDER/${DB_PASSWORD}/g\" $CONFIG\n"],"command":["/bin/bash"],"env":[{"name":"FUNNEL_OIDC_CLIENT_ID","valueFrom":{"secretKeyRef":{"key":"client_id","name":"funnel-oidc-client","optional":false}}},{"name":"FUNNEL_OIDC_CLIENT_SECRET","valueFrom":{"secretKeyRef":{"key":"client_secret","name":"funnel-oidc-client","optional":false}}},{"name":"DB_HOST","valueFrom":{"secretKeyRef":{"key":"host","name":"funnel-dbcreds","optional":false}}},{"name":"DB_USER","valueFrom":{"secretKeyRef":{"key":"username","name":"funnel-dbcreds","optional":false}}},{"name":"DB_PASSWORD","valueFrom":{"secretKeyRef":{"key":"password","name":"funnel-dbcreds","optional":false}}},{"name":"DB_DATABASE","valueFrom":{"secretKeyRef":{"key":"database","name":"funnel-dbcreds","optional":false}}}],"image":"quay.io/cdis/awshelper","name":"config-updater","tag":"master","volumeMounts":[{"mountPath":"/tmp","name":"funnel-patched-config-volume"},{"mountPath":"/etc/config/funnel.conf","name":"funnel-config-volume","subPath":"funnel-server.yaml"}]}]` | Configuration for the Funnel init container. | +| image.initContainers[0].command | list | `["cp","/app/build/plugins/authorizer","/opt/funnel/plugin-binaries/auth-plugin"]` | Arguments to pass to the init container. | +| image.initContainers[0].image | string | `"quay.io/cdis/funnel-gen3-plugin"` | The Docker image repository for the Funnel init/plugin container. | +| image.initContainers[0].pullPolicy | string | `"Always"` | When to pull the image. This value should be "Always" to ensure the latest image is used. | +| image.initContainers[0].tag | string | `"main-gen3"` | The Docker image tag for the Funnel init/plugin container. | +| image.pullPolicy | string | `"Always"` | When to pull the image. This value should be "Always" to ensure the latest image is used. | +| image.repository | string | `"quay.io/ohsu-comp-bio/funnel"` | The Docker image repository for the Funnel service. | +| image.tag | string | `"develop-2026-07-09-19-49-55Z-97d1df55"` | | +| labels.app | string | `"funnel"` | | | metricsEnabled | bool | `false` | | | netPolicy | map | `{"egressApps":["gen3-workflow"],"ingressApps":["gen3-workflow"]}` | Configuration for network policies created by this chart. Only relevant if "global.netPolicy.enabled" is set to true | | netPolicy.egressApps | array | `["gen3-workflow"]` | List of apps that this app requires egress to | @@ -121,7 +206,43 @@ A Helm chart for Kubernetes | postgres.username | string | `nil` | Username for postgres. This is a service override, defaults to - | | postgresql | map | `{"primary":{"persistence":{"enabled":false}}}` | Postgresql subchart settings if deployed separately option is set to "true". Disable persistence by default so we can spin up and down ephemeral environments | | postgresql.primary.persistence.enabled | bool | `false` | Option to persist the dbs data. | +| rbac.create | bool | `true` | | | release | string | `"production"` | Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". | +| replicaCount | int | `1` | | +| resources.limits.cpu | string | `"1000m"` | | +| resources.limits.ephemeral_storage | string | `"2Gi"` | | +| resources.limits.memory | string | `"2Gi"` | | +| resources.requests.cpu | string | `"100m"` | | +| resources.requests.ephemeral_storage | string | `"2Gi"` | | +| resources.requests.memory | string | `"2Gi"` | | | secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null}` | Secret information for External Secrets. | | secrets.awsAccessKeyId | str | `nil` | AWS access key ID. Overrides global key. | | secrets.awsSecretAccessKey | str | `nil` | AWS secret access key ID. Overrides global key. | +| service.httpPort | int | `8000` | | +| service.rpcPort | int | `9090` | | +| service.type | string | `"ClusterIP"` | | +| stsRegion | string | `"us-east-1"` | | +| volumeMounts[0].mountPath | string | `"/etc/config/funnel-server.yaml"` | | +| volumeMounts[0].name | string | `"funnel-patched-config-volume"` | | +| volumeMounts[0].subPath | string | `"funnel-patched.conf"` | | +| volumeMounts[1].mountPath | string | `"/etc/config/oidc"` | | +| volumeMounts[1].name | string | `"funnel-oidc-volume"` | | +| volumeMounts[1].readOnly | bool | `true` | | +| volumeMounts[2].mountPath | string | `"/etc/funnel/templates"` | | +| volumeMounts[2].name | string | `"worker-templates-volume"` | | +| volumeMounts[3].mountPath | string | `"/opt/funnel/plugin-binaries"` | | +| volumeMounts[3].name | string | `"plugin-volume"` | | +| volumes[0].configMap.name | string | `"funnel-server-config"` | | +| volumes[0].name | string | `"funnel-config-volume"` | | +| volumes[1].name | string | `"funnel-oidc-volume"` | | +| volumes[1].secret.items[0].key | string | `"client_id"` | | +| volumes[1].secret.items[0].path | string | `"client_id"` | | +| volumes[1].secret.items[1].key | string | `"client_secret"` | | +| volumes[1].secret.items[1].path | string | `"client_secret"` | | +| volumes[1].secret.secretName | string | `"funnel-oidc-client"` | | +| volumes[2].configMap.name | string | `"funnel-worker-templates"` | | +| volumes[2].name | string | `"worker-templates-volume"` | | +| volumes[3].emptyDir | object | `{}` | | +| volumes[3].name | string | `"plugin-volume"` | | +| volumes[4].emptyDir | object | `{}` | | +| volumes[4].name | string | `"funnel-patched-config-volume"` | | diff --git a/helm/funnel/charts/funnel-0.1.99-rc.36.tgz b/helm/funnel/charts/funnel-0.1.99-rc.36.tgz deleted file mode 100644 index 208c61aac..000000000 Binary files a/helm/funnel/charts/funnel-0.1.99-rc.36.tgz and /dev/null differ diff --git a/helm/funnel/files/executor-job.yaml b/helm/funnel/files/executor-job.yaml new file mode 100644 index 000000000..250a2fad8 --- /dev/null +++ b/helm/funnel/files/executor-job.yaml @@ -0,0 +1,146 @@ +# Task Executor +apiVersion: batch/v1 +kind: Job +metadata: + name: {{`{{.TaskId}}`}}-{{`{{.JobId}}`}} + namespace: {{`{{.JobsNamespace}}`}} + labels: + app: funnel-executor + taskId: {{`{{.TaskId}}`}} + job-name: {{`{{.TaskId}}-{{.JobId}}`}} +spec: + backoffLimit: {{ .Values.Kubernetes.Executor.backoffLimit}} + completions: {{ .Values.Kubernetes.Executor.completions }} + template: + metadata: + + # Annotations + {{- with .Values.Kubernetes.Executor.Annotations }} + annotations: + {{ toYaml . | indent 2 }} + {{- end }} + + labels: + app: funnel-executor + taskId: {{`{{.TaskId}}`}} + job-name: {{`{{.TaskId}}`}}-{{`{{.JobId}}`}} + spec: + + # If priorityClassName is set for the Executor, use that. + # Otherwise, if it's set for the Worker, use that. + # This allows users to set a default priority class for all jobs by setting it for the Worker, + # but also override it for the Executor if needed. + {{- $pc := ""}} + {{- if .Values.Kubernetes.Executor.PriorityClassName }} + {{- $pc = .Values.Kubernetes.Executor.PriorityClassName }} + {{- else if .Values.Kubernetes.Worker.PriorityClassName }} + {{- $pc = .Values.Kubernetes.Worker.PriorityClassName }} + {{- end }} + + {{- if $pc }} + priorityClassName: {{ $pc }} + {{- end }} + + # NodeSelectors + # https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes/#create-a-pod-that-gets-scheduled-to-your-chosen-node + {{` + {{- if .NodeSelector }} + nodeSelector: + {{- range $key, $value := .NodeSelector }} + {{ $key }}: {{ $value }} + {{- end }} + {{- end }} + `}} + + # Tolerations + # https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + {{` + {{- if .Tolerations }} + tolerations: + {{- range .Tolerations }} + - key: {{ .Key }} + operator: {{ .Operator }} + effect: {{ .Effect }} + {{if .Value}}value: {{ .Value }}{{end}} + {{if .TolerationSeconds}}tolerationSeconds: {{ .TolerationSeconds }}{{end}} + {{- end }} + {{- end }} + `}} + + securityContext: + # Use the default seccomp profile, which should block several dangerous syscalls that could be used for privilege escalation or container escape + seccompProfile: + type: RuntimeDefault + + # Kubernetes service account token is not needed for the Executor, and disabling it can help reduce the risk of token theft and misuse. + automountServiceAccountToken: false + restartPolicy: {{ .Values.Kubernetes.Executor.restartPolicy }} + serviceAccountName: {{`{{.ServiceAccountName}}`}} + containers: + - name: funnel-executor-{{`{{.TaskId}}`}} + image: {{`{{.Image}}`}} + imagePullPolicy: {{`{{if .ImagePullPolicy}}{{.ImagePullPolicy}}{{else}}Always{{end}}`}} + securityContext: + # Block the ability to gain more privileges + allowPrivilegeEscalation: false + privileged: false + + # Block all capabilities and then add back only the ones we need + capabilities: + drop: ["ALL"] + add: ["CHOWN", "DAC_OVERRIDE", "SETUID", "SETGID"] + # Command + {{` + {{- if .UseShell}} + command: + - "/bin/sh" + - "-c" + args: + - {{printf "%q" (index .Command 0)}} + {{- else}} + command: + {{- range .Command}} + - {{printf "%q" .}} + {{- end}} + {{- end}} + `}} + + workingDir: {{`{{.Workdir}}`}} + + env: + {{` + {{- range $key, $value := .Env }} + - name: {{ $key }} + value: "{{ $value }}" + {{- end }} + `}} + + resources: + requests: + cpu: {{`{{ .Cpus }}`}} + memory: {{`{{ .RamGb }}`}}Gi + ephemeral-storage: {{`{{ .DiskGb }}`}}Gi + limits: + cpu: {{`{{ .CpusLimit }}`}} + memory: {{`{{ .RamGbLimit }}`}}Gi + ephemeral-storage: {{`{{ .DiskGbLimit }}`}}Gi + + volumeMounts: + {{` + {{- if .NeedsPVC }} + {{- range $idx, $item := .Volumes}} + - name: funnel-storage-{{$.TaskId}} + mountPath: {{$item.ContainerPath}} + subPath: {{$.TaskId}}{{$item.ContainerPath}} + {{- end}} + {{- end }} + `}} + + volumes: + {{` + {{- if .NeedsPVC }} + - name: funnel-storage-{{.TaskId}} + persistentVolumeClaim: + claimName: funnel-worker-pvc-{{.TaskId}} + {{- end }} + `}} diff --git a/helm/funnel/files/role.yaml b/helm/funnel/files/role.yaml new file mode 100644 index 000000000..ef27a85ef --- /dev/null +++ b/helm/funnel/files/role.yaml @@ -0,0 +1,39 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: funnel-worker-sa-{{`{{.Namespace}}`}}-{{`{{.TaskId}}`}}-role + namespace: {{`{{.Namespace}}`}} + labels: + app: funnel + taskId: {{`{{.TaskId}}`}} +rules: +# Job management permissions +- apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "get", "list", "watch", "update", "patch", "delete"] +- apiGroups: ["batch"] + resources: ["jobs/status"] + verbs: ["get", "update", "patch"] + +# Pod management permissions (jobs create pods) +- apiGroups: [""] + resources: ["pods"] + verbs: ["create", "get", "list", "watch", "delete"] +- apiGroups: [""] + resources: ["pods/status", "pods/log"] + verbs: ["get", "list", "watch"] + +# ConfigMap and Secret access (if needed for job configuration) +- apiGroups: [""] + resources: ["configmaps", "secrets"] + verbs: ["get", "list"] + +# PVC management (if using persistent volumes) +- apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["create", "get", "list", "watch", "delete"] + +# Events (for job status reporting) +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] diff --git a/helm/funnel/files/rolebinding.yaml b/helm/funnel/files/rolebinding.yaml new file mode 100644 index 000000000..fa287a897 --- /dev/null +++ b/helm/funnel/files/rolebinding.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: funnel-worker-sa-{{`{{.Namespace}}`}}-{{`{{.TaskId}}`}}-binding + namespace: {{`{{.Namespace}}`}} + labels: + app: funnel + taskId: {{`{{.TaskId}}`}} +subjects: +- kind: ServiceAccount + name: {{`{{.ServiceAccountName}}`}} + namespace: {{`{{.Namespace}}`}} +roleRef: + kind: Role + name: funnel-worker-sa-{{`{{.Namespace}}`}}-{{`{{.TaskId}}`}}-role + apiGroup: rbac.authorization.k8s.io diff --git a/helm/funnel/files/server-config.yaml b/helm/funnel/files/server-config.yaml new file mode 100644 index 000000000..c585576aa --- /dev/null +++ b/helm/funnel/files/server-config.yaml @@ -0,0 +1,201 @@ +Compute: {{ .Values.Compute }} + +Kubernetes: + DisableReconciler: {{ .Values.Kubernetes.DisableReconciler }} + DisableJobCleanup: {{ .Values.Kubernetes.DisableJobCleanup }} + ReconcileRate: {{ .Values.Kubernetes.ReconcileRate }} +{{- with .Values.Kubernetes.ForbiddenPathPrefixes }} + ForbiddenPathPrefixes: +{{ toYaml . | indent 4 }} +{{- end }} + Namespace: {{ .Release.Namespace }} + JobsNamespace: {{ .Values.Kubernetes.JobsNamespace }} + ServiceAccount: {{ .Values.Kubernetes.ServiceAccount }} + Resources: + Defaults: + Cpus: {{ .Values.Kubernetes.Resources.Defaults.Cpus }} + RamGb: {{ .Values.Kubernetes.Resources.Defaults.RamGb }} + DiskGb: {{ .Values.Kubernetes.Resources.Defaults.DiskGb }} + Limits: + Cpus: {{ .Values.Kubernetes.Resources.Limits.Cpus }} + RamGb: {{ .Values.Kubernetes.Resources.Limits.RamGb }} + DiskGb: {{ .Values.Kubernetes.Resources.Limits.DiskGb }} + Timeout: + duration: {{ .Values.Kubernetes.Timeout.duration }} + + NodeSelector: + {{- if .Values.Kubernetes.NodeSelector }} + {{- range $key, $value := .Values.Kubernetes.NodeSelector }} + {{ $key }}: {{ $value }} + {{- end }} + {{- end }} + + Tolerations: + {{- if .Values.Kubernetes.Tolerations }} + {{- range .Values.Kubernetes.Tolerations }} + - Key: {{ .Key }} + Operator: {{ .Operator }} + {{- if .Value }} + Value: {{ .Value }} + {{- end }} + Effect: {{ .Effect }} + {{- if .TolerationSeconds }} + TolerationSeconds: {{ .TolerationSeconds }} + {{- end }} + {{- end }} + {{- end }} + + # Worker Job + WorkerTemplate: | +{{ tpl ((.Values.Kubernetes.WorkerTemplate | default (.Files.Get "files/worker-job.yaml"))) . | indent 4 }} + + # Executor Job + ExecutorTemplate: | +{{ tpl ((.Values.Kubernetes.ExecutorTemplate | default (.Files.Get "files/executor-job.yaml"))) . | indent 4 }} + +# ConfigMap template + ConfigMapTemplate: | +{{ tpl (.Files.Get "files/worker-configmap.yaml") . | indent 4 }} + + # PV template + PVTemplate: | +{{ tpl (.Files.Get "files/worker-pv.yaml") . | indent 4 }} + + # PVC template + PVCTemplate: | +{{ tpl (.Files.Get "files/worker-pvc.yaml") . | indent 4 }} + + # ServiceAccount template + ServiceAccountTemplate: | +{{ tpl (.Files.Get "files/serviceaccount.yaml") . | indent 4 }} + + RoleTemplate: | +{{ tpl (.Files.Get "files/role.yaml") . | indent 4 }} + + RoleBindingTemplate: | +{{ tpl (.Files.Get "files/rolebinding.yaml") . | indent 4 }} + +Database: {{ .Values.Database }} + +EventWriters: + {{- range .Values.EventWriters }} + - {{ . }} + {{- end }} + +Logger: + Level: {{ .Values.Logger.Level }} + OutputFile: "{{ .Values.Logger.OutputFile }}" + Formatter: {{ .Values.Logger.Formatter }} + TextFormat: + ForceColors: {{ .Values.Logger.TextFormat.ForceColors }} + FullTimestamp: {{ .Values.Logger.TextFormat.FullTimestamp }} + TimestampFormat: {{ .Values.Logger.TextFormat.TimestampFormat }} + +Server: + HostName: "{{ .Values.Server.HostName }}" + HTTPPort: "{{ .Values.Server.HTTPPort }}" + RPCPort: "{{ .Values.Server.RPCPort }}" + DisableHTTPCache: {{ .Values.Server.DisableHTTPCache }} + +RPCClient: + ServerAddress: {{ .Values.RPCClient.ServerAddress }} + Timeout: + duration: {{ .Values.RPCClient.Timeout.duration }} + MaxRetries: {{ .Values.RPCClient.MaxRetries }} + +Scheduler: + ScheduleRate: {{ .Values.Scheduler.ScheduleRate }} + ScheduleChunk: {{ .Values.Scheduler.ScheduleChunk }} + NodePingTimeout: + duration: {{ .Values.Scheduler.NodePingTimeout.duration }} + NodeInitTimeout: + duration: {{ .Values.Scheduler.NodeInitTimeout.duration }} + +Node: + ID: {{ .Values.Node.ID }} + Timeout: + disabled: {{ .Values.Node.Timeout.disabled }} + UpdateRate: {{ .Values.Node.UpdateRate }} + Resources: + Cpus: {{ .Values.Node.Resources.Cpus }} + RamGb: {{ .Values.Node.Resources.RamGb }} + DiskGb: {{ .Values.Node.Resources.DiskGb }} + +Worker: + WorkDir: {{ .Values.Worker.WorkDir }} + PollingRate: {{ .Values.Worker.PollingRate }} + LogUpdateRate: {{ .Values.Worker.LogUpdateRate }} + LogTailSize: {{ .Values.Worker.LogTailSize }} + LeaveWorkDir: {{ .Values.Worker.LeaveWorkDir }} + MaxParallelTransfers: {{ .Values.Worker.MaxParallelTransfers }} + +BoltDB: + Path: {{ .Values.BoltDB.Path }} + +AmazonS3: + Disabled: {{ .Values.AmazonS3.Disabled }} + AWSConfig: + MaxRetries: {{ .Values.AmazonS3.AWSConfig.MaxRetries }} + Key: {{ .Values.AmazonS3.AWSConfig.Key }} + Secret: {{ .Values.AmazonS3.AWSConfig.Secret }} + SSE: + CustomerKeyFile: {{ .Values.AmazonS3.SSE.CustomerKeyFile }} + KMSKey: {{ .Values.AmazonS3.SSE.KMSKey }} + +DynamoDB: + TableBasename: {{ .Values.DynamoDB.TableBasename }} + AWSConfig: + Region: {{ .Values.DynamoDB.AWSConfig.Region }} + Key: {{ .Values.DynamoDB.AWSConfig.Key }} + Secret: {{ .Values.DynamoDB.AWSConfig.Secret }} + +Elastic: + IndexPrefix: {{ .Values.Elastic.IndexPrefix }} + URL: {{ .Values.Elastic.URL }} + +Datastore: + Project: {{ .Values.Datastore.Project }} + CredentialsFile: {{ .Values.Datastore.CredentialsFile }} + +Postgres: + Host: {{ .Values.Postgres.Host }} + Database: {{ .Values.Postgres.Database }} + User: {{ .Values.Postgres.User }} + Password: {{ .Values.Postgres.Password }} + AdminUser: {{ .Values.Postgres.AdminUser }} + AdminPassword: {{ .Values.Postgres.AdminPassword }} + Timeout: + duration: {{ .Values.Postgres.Timeout.duration }} + +Kafka: + Servers: + {{- if .Values.Kafka.Servers }} + {{- range .Values.Kafka.Servers }} + - {{ . }} + {{- end }} + {{- else }} + - "" + {{- end }} + Topic: {{ .Values.Kafka.Topic }} + +GenericS3: + {{- range .Values.GenericS3 }} + - Disabled: {{ .Disabled }} + Endpoint: {{ .Endpoint }} + Key: {{ .Key }} + Secret: {{ .Secret }} + Bucket: {{ .Bucket }} + Region: {{ .Region }} + KmsKeyID: {{ .KmsKeyID }} + {{- end }} + +{{- if .Values.Plugins}} +Plugins: + Path: {{ .Values.Plugins.Path }} + Params: + {{- if .Values.Plugins.Params }} + {{- range $key, $value := .Values.Plugins.Params }} + {{ $key }}: {{ $value }} + {{- end }} + {{- end }} +{{- end}} diff --git a/helm/funnel/files/serviceaccount.yaml b/helm/funnel/files/serviceaccount.yaml new file mode 100644 index 000000000..2eaf90cc7 --- /dev/null +++ b/helm/funnel/files/serviceaccount.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + # Note: Only creating ServiceAccount here, not required to create new Role or RoleBinding + # TODO: Set defaults for case no custom values are being supplied from the TES Task Tags + name: {{`{{.ServiceAccountName}}`}} + namespace: {{`{{.Namespace}}`}} + labels: + app: funnel + taskId: {{`{{.TaskId}}`}} + # Only set if provided with Role ARN + {{`{{- if .IamRoleArn}}`}} + annotations: + eks.amazonaws.com/role-arn: {{`{{.IamRoleArn}}`}} + {{`{{- end}}`}} +spec: + # TODO: Should tokens be automatically mounted into pods? + automountServiceAccountToken: false diff --git a/helm/funnel/files/worker-configmap.yaml b/helm/funnel/files/worker-configmap.yaml new file mode 100644 index 000000000..25d5a3264 --- /dev/null +++ b/helm/funnel/files/worker-configmap.yaml @@ -0,0 +1,11 @@ +# This is the config containing the templates used by the Funnel Worker +apiVersion: v1 +kind: ConfigMap +metadata: + name: funnel-worker-config-{{`{{ .TaskId }}`}} + namespace: {{`{{ .Namespace }}`}} + labels: + app: funnel +data: + funnel-worker.yaml: | +{{`{{ .Config | indent 4}}`}} diff --git a/helm/funnel/files/worker-job.yaml b/helm/funnel/files/worker-job.yaml new file mode 100644 index 000000000..7bf13da87 --- /dev/null +++ b/helm/funnel/files/worker-job.yaml @@ -0,0 +1,117 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{`{{.TaskId}}`}} + namespace: {{`{{.JobsNamespace}}`}} + labels: + app: funnel-worker + task-id: {{`{{.TaskId}}`}} + # Custom Labels support + {{` + {{- range $key, $value := .ExtraLabels }} + {{ $key }}: {{ $value }} + {{- end }} + {{- range $key, $value := .NodeSelector }} + {{ $key }}: {{ $value }} + {{- end }} + `}} +spec: + backoffLimit: {{ .Values.Kubernetes.Worker.backoffLimit}} + completions: {{ .Values.Kubernetes.Worker.completions }} + template: + metadata: + labels: + app: funnel-worker + task-id: {{`{{.TaskId}}`}} + + # Annotations + {{- with .Values.Kubernetes.Worker.Annotations }} + annotations: + {{ toYaml . | indent 2 }} + {{- end }} + + spec: + + {{ if .Values.Kubernetes.Worker.PriorityClassName }} + priorityClassName: {{ .Values.Kubernetes.Worker.PriorityClassName }} + {{ end }} + # NodeSelectors + # https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes/#create-a-pod-that-gets-scheduled-to-your-chosen-node + {{` + {{- if .NodeSelector }} + nodeSelector: + {{- range $key, $value := .NodeSelector }} + {{ $key }}: {{ $value }} + {{- end }} + {{- end }} + `}} + + # Tolerations + # https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + {{` + {{- if .Tolerations }} + tolerations: + {{- range .Tolerations }} + - key: {{ .Key }} + operator: {{ .Operator }} + effect: {{ .Effect }} + {{if .Value}}value: {{ .Value }}{{end}} + {{if .TolerationSeconds}}tolerationSeconds: {{ .TolerationSeconds }}{{end}} + {{- end }} + {{- end }} + `}} + + # SecurityContext + # https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod + {{` + {{- if .securityContext }} + securityContext: + fsGroup: {{ .securityContext.fsGroup }} + runAsUser: {{ .securityContext.runAsUser }} + runAsGroup: {{ .securityContext.runAsGroup }} + supplementalGroups: {{ .securityContext.supplementalGroups }} + {{- end }} + `}} + + serviceAccountName: {{`{{.ServiceAccountName}}`}} + restartPolicy: {{ .Values.Kubernetes.Worker.restartPolicy }} + containers: + - name: funnel-worker-{{`{{.TaskId}}`}} + image: {{`{{.Image}}`}} + imagePullPolicy: Always + command: + - /bin/sh + - -c + args: + - | + funnel worker run --config /etc/config/funnel-worker.yaml --taskID {{`{{.TaskId}}`}} + resources: + requests: + cpu: {{ .Values.resources.requests.cpu }} + memory: {{ .Values.resources.requests.memory }} + ephemeral-storage: {{ .Values.resources.requests.ephemeral_storage }} + limits: + cpu: {{ .Values.resources.limits.cpu }} + memory: {{ .Values.resources.limits.memory }} + ephemeral-storage: {{ .Values.resources.limits.ephemeral_storage }} + volumeMounts: + - name: config-volume + mountPath: /etc/config + {{` + {{- if .NeedsPVC }} + - name: funnel-storage-{{.TaskId}} + mountPath: /opt/funnel/funnel-work-dir/{{.TaskId}} + subPath: {{.TaskId}} + {{- end }} + `}} + volumes: + - name: config-volume + configMap: + name: funnel-worker-config-{{`{{.TaskId}}`}} + {{` + {{- if .NeedsPVC }} + - name: funnel-storage-{{.TaskId}} + persistentVolumeClaim: + claimName: funnel-worker-pvc-{{.TaskId}} + {{- end }} + `}} diff --git a/helm/funnel/files/worker-pv.yaml b/helm/funnel/files/worker-pv.yaml new file mode 100644 index 000000000..d118617bf --- /dev/null +++ b/helm/funnel/files/worker-pv.yaml @@ -0,0 +1,64 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: funnel-worker-pv-{{`{{.TaskId}}`}} + labels: + app: funnel + taskId: {{`{{.TaskId}}`}} + namespace: {{`{{.Namespace}}`}} +spec: + storageClassName: "" # required for static provisioning + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + capacity: # placeholder only — capacity is effectively elastic + storage: "10Mi" + + mountOptions: + {{- if eq .Values.Kubernetes.Storage.type "mountpoint_s3" }} + - allow-delete + - allow-overwrite # allow overwriting existing files + - incremental-upload # allow appending to existing files + - allow-other # allow non-root users to access the mount + - region={{`{{.Region}}`}} + - file-mode=0755 + - prefix=funnel-temp-files/ + {{`{{- if .KmsKeyID}}`}} + - sse aws:kms + - sse-kms-key-id={{`{{.KmsKeyID}}`}} + {{`{{- end}}`}} + {{- end }} + + {{- if eq .Values.Kubernetes.Storage.type "s3files" }} + - tls + - noresvport + {{- end }} + + {{- if .Values.endpoint_url }} + - endpoint-url={{ .Values.endpoint_url }} + - force-path-style # avoids DNS resolution issue, see: + # https://github.com/awslabs/mountpoint-s3/blob/v1.22.2/doc/TROUBLESHOOTING.md#invalid-hostname-for-dns-resolution + {{- end }} + + csi: + {{- if eq .Values.Kubernetes.Storage.type "mountpoint_s3" }} + driver: s3.csi.aws.com + volumeHandle: s3-csi-{{`{{.TaskId}}`}} + volumeAttributes: + bucketName: {{`{{.Bucket}}`}} + {{- if .Values.authenticationSource }} + authenticationSource: {{ .Values.authenticationSource }} + stsRegion: {{ .Values.stsRegion }} + {{- end }} + {{- end }} + + {{- if eq .Values.Kubernetes.Storage.type "s3files" }} + driver: efs.csi.aws.com + volumeHandle: s3files:{{`{{.S3FilesystemId}}`}} + volumeAttributes: + encryptInTransit: "true" + {{- end }} + + claimRef: + namespace: {{`{{.Namespace}}`}} + name: funnel-worker-pvc-{{`{{.TaskId}}`}} \ No newline at end of file diff --git a/helm/funnel/files/worker-pvc.yaml b/helm/funnel/files/worker-pvc.yaml new file mode 100644 index 000000000..d8fccb1b4 --- /dev/null +++ b/helm/funnel/files/worker-pvc.yaml @@ -0,0 +1,17 @@ +# Worker/Executor PVC +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: funnel-worker-pvc-{{`{{.TaskId}}`}} + namespace: {{`{{.Namespace}}`}} + labels: + app: funnel + taskId: {{`{{.TaskId}}`}} +spec: + storageClassName: "" # Required for static provisioning + accessModes: + - ReadWriteMany + resources: + requests: + storage: 10Mi + volumeName: funnel-worker-pv-{{`{{.TaskId}}`}} diff --git a/helm/funnel/templates/_helpers.tpl b/helm/funnel/templates/_helpers.tpl index b6ed780bc..20c72b511 100644 --- a/helm/funnel/templates/_helpers.tpl +++ b/helm/funnel/templates/_helpers.tpl @@ -81,4 +81,47 @@ Create the name of the service account to use */}} {{- define "funnel-oidc-client" -}} {{- default "funnel-oidc-client" .Values.externalSecrets.funnelOidcClient }} -{{- end }} \ No newline at end of file +{{- end }} + +{{/* +Parse a Go duration string (e.g. "10s", "90s", "5m", "1h") into whole seconds. +Only the s/m/h suffixes are supported; an unsuffixed value is treated as seconds. +Returns 0 when the input is empty or unparseable. +*/}} +{{- define "funnel.durationSeconds" -}} +{{- $d := . | toString | trim -}} +{{- $seconds := 0 -}} +{{- if hasSuffix "h" $d -}} +{{- $seconds = mul (trimSuffix "h" $d | int) 3600 -}} +{{- else if hasSuffix "m" $d -}} +{{- $seconds = mul (trimSuffix "m" $d | int) 60 -}} +{{- else if hasSuffix "s" $d -}} +{{- $seconds = trimSuffix "s" $d | int -}} +{{- else if $d -}} +{{- $seconds = $d | int -}} +{{- end -}} +{{- $seconds -}} +{{- end }} + +{{/* +Derive the cleanup CronJob schedule from Kubernetes.ReconcileRate unless +cleanup.schedule is set explicitly. +*/}} +{{- define "funnel.cleanupSchedule" -}} +{{- if .Values.cleanup.schedule -}} +{{- .Values.cleanup.schedule -}} +{{- else -}} +{{- $seconds := include "funnel.durationSeconds" .Values.Kubernetes.ReconcileRate | int -}} +{{- $minutes := div (add $seconds 59) 60 -}} +{{- if lt $minutes 1 }}{{- $minutes = 1 -}}{{- end -}} +{{- $offset := .Values.cleanup.scheduleOffsetMinutes | default 0 | int -}} +{{- if and (le $minutes 59) (eq (mod 60 $minutes) 0) -}} +{{- printf "%d-59/%d * * * *" $offset $minutes -}} +{{- else if le $minutes 59 -}} +{{- printf "%d/%d * * * *" $offset $minutes -}} +{{- else -}} +{{- $hours := div (add $minutes 59) 60 -}} +{{- printf "%d */%d * * *" $offset $hours -}} +{{- end -}} +{{- end -}} +{{- end }} diff --git a/helm/funnel/templates/clusterrole.yaml b/helm/funnel/templates/clusterrole.yaml new file mode 100644 index 000000000..4669eca2b --- /dev/null +++ b/helm/funnel/templates/clusterrole.yaml @@ -0,0 +1,65 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: funnel-cluster-role-{{ .Release.Namespace }} +rules: + - apiGroups: [""] + resources: + - "configmaps" + - "pods" + - "pods/log" + - "persistentvolumes" + - "persistentvolumeclaims" + - "secrets" + - "serviceaccounts" + - "events" + - "pods/status" + verbs: + - "get" + - "list" + - "watch" + - "create" + - "update" + - "patch" + - "delete" + + - apiGroups: ["batch", "extensions"] + resources: + - "jobs" + - "jobs/status" # Previously corrected + verbs: + - "get" + - "list" + - "watch" + - "create" + - "update" + - "patch" + - "delete" + + - apiGroups: ["extensions", "apps"] + resources: + - "deployments" + verbs: + - "get" + - "list" + - "watch" + - "create" + - "update" + - "patch" + - "delete" + + - apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +{{- end }} diff --git a/helm/funnel/templates/clusterrolebinding.yaml b/helm/funnel/templates/clusterrolebinding.yaml new file mode 100644 index 000000000..2fd4a87e4 --- /dev/null +++ b/helm/funnel/templates/clusterrolebinding.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: funnel-cluster-rolebinding-{{ .Release.Namespace }} +subjects: + - kind: ServiceAccount + name: funnel-sa-{{ .Release.Namespace }} + namespace: {{ .Release.Namespace }} + {{- if and .Values.Kubernetes.JobsNamespace (ne .Values.Kubernetes.JobsNamespace .Values.Kubernetes.Namespace) }} + - kind: ServiceAccount + name: funnel-sa-{{ .Release.Namespace }} + namespace: {{ .Values.Kubernetes.JobsNamespace }} + {{- end }} +roleRef: + kind: ClusterRole + name: funnel-cluster-role-{{ .Release.Namespace }} + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/helm/funnel/templates/cronjob.yaml b/helm/funnel/templates/cronjob.yaml new file mode 100644 index 000000000..2c92e72d2 --- /dev/null +++ b/helm/funnel/templates/cronjob.yaml @@ -0,0 +1,45 @@ +{{- if .Values.cleanup.enabled }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: funnel-cleanup-{{ .Release.Namespace }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "funnel.labels" . | nindent 4 }} +spec: + # Schedule is derived from Kubernetes.ReconcileRate unless cleanup.schedule is set explicitly. + schedule: {{ include "funnel.cleanupSchedule" . | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + template: + metadata: + labels: + {{- include "funnel.labels" . | nindent 12 }} + spec: + serviceAccountName: funnel-sa-{{ .Release.Namespace }} + restartPolicy: OnFailure + containers: + - name: funnel-cleanup + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: Always + command: + - funnel + - kubernetes + - cleanup + - --config + - /etc/config/funnel-server.yaml + resources: + requests: + cpu: "100m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "256Mi" + volumeMounts: + {{- toYaml .Values.volumeMounts | nindent 14 }} + volumes: + {{- toYaml .Values.volumes | nindent 10 }} +{{- end }} diff --git a/helm/funnel/templates/external-secret.yaml b/helm/funnel/templates/external-secret.yaml index 4cede2583..b705d168d 100644 --- a/helm/funnel/templates/external-secret.yaml +++ b/helm/funnel/templates/external-secret.yaml @@ -1,5 +1,5 @@ {{ if .Values.global.externalSecrets.deploy }} -apiVersion: external-secrets.io/v1beta1 +apiVersion: {{ include "common.externalSecrets.apiVersion" . }} kind: ExternalSecret metadata: name: funnel-oidc-client diff --git a/helm/funnel/templates/job-resources.yaml b/helm/funnel/templates/job-resources.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/helm/funnel/templates/plugin-server.yaml b/helm/funnel/templates/plugin-server.yaml new file mode 100644 index 000000000..747772255 --- /dev/null +++ b/helm/funnel/templates/plugin-server.yaml @@ -0,0 +1,66 @@ +{{ if .Values.deployTestServer }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: funnel-plugin-test-server + namespace: {{ .Release.Namespace }} + labels: + app: funnel-plugin-test-server +spec: + replicas: 1 + selector: + matchLabels: + app: funnel-plugin-test-server + template: + metadata: + labels: + app: funnel-plugin-test-server + spec: + containers: + - name: test-server + image: quay.io/ohsu-comp-bio/funnel-plugins-test-server:testing + imagePullPolicy: Always + ports: + - containerPort: 8080 + volumeMounts: + - name: funnel-plugin-test-users + mountPath: /app/example-users.csv + subPath: example-users.csv + command: ["/bin/sh", "-c"] + args: ["while true; do sleep 3600; done"] + volumes: + - name: funnel-plugin-test-users + configMap: + name: funnel-plugin-test-users + +--- +apiVersion: v1 +kind: Service +metadata: + name: funnel-plugin-test-server + namespace: {{ .Release.Namespace }} +spec: + selector: + app: funnel-plugin-test-server + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: funnel-plugin-test-users + namespace: {{ .Release.Namespace }} +data: + example-users.csv: |- + user,key,secret,GET,PUT,DELETE + example,key1,secret1,1,1,1 + foo,key2,secret2,1,0,1 + bar,key3,secret3,0,0,0 + foobar,key4,secret4,0,1,1 + alpha,key5,secret5,1,1,0 + beta,key6,secret6,1,0,0 + gamma,key7,secret7,0,1,0 + delta,key8,secret8,0,0,1 +{{ end }} diff --git a/helm/funnel/templates/server-configmap.yaml b/helm/funnel/templates/server-configmap.yaml new file mode 100644 index 000000000..11c1a7499 --- /dev/null +++ b/helm/funnel/templates/server-configmap.yaml @@ -0,0 +1,15 @@ +# This is the config for the Funnel Server +# +# The Worker config uses the template in `config/kubernetes-configmap-template.yaml` +# to dynamically create a new ConfigMap for each task (in `compute/kubernetes/backend.go`). +# +# This is done to support per-user task configurations (e.g. S3 credentials). +apiVersion: v1 +kind: ConfigMap +metadata: + name: funnel-server-config + namespace: {{ .Release.Namespace }} + +data: + funnel-server.yaml: |- +{{ tpl (.Files.Get "files/server-config.yaml") . | indent 4 }} diff --git a/helm/funnel/templates/server-deployment.yaml b/helm/funnel/templates/server-deployment.yaml new file mode 100644 index 000000000..04e656b25 --- /dev/null +++ b/helm/funnel/templates/server-deployment.yaml @@ -0,0 +1,96 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.name | default "funnel-server" }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "funnel.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 # This ensures only one pod is unavailable during the update process + maxSurge: 0 # No new pod is created until the old one is fully terminated + selector: + matchLabels: + {{- include "funnel.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "funnel.labels" . | nindent 8 }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/server-configmap.yaml") . | sha256sum }} + checksum/values: {{ .Values | quote | sha256sum }} + spec: + serviceAccountName: funnel-sa-{{ .Release.Namespace }} + {{- if .Values.image.initContainers }} + initContainers: + {{- range .Values.image.initContainers }} + - name: {{ .name | default "initcontainer" }} + image: "{{ .image }}:{{ .tag }}" + imagePullPolicy: {{ .pullPolicy | default "IfNotPresent" }} + {{- if .command }} + command: + {{- range .command }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- if .args }} + args: + {{- range .args }} + - {{ . | quote }} + {{- end }} + {{- end }} + {{- if .volumeMounts }} + volumeMounts: + {{- toYaml .volumeMounts | nindent 10 }} + {{- end }} + {{- if .env }} + env: + {{- toYaml .env | nindent 10 }} + {{- end }} + {{- end }} + {{- end }} + containers: + - name: funnel + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: Always + command: + - 'funnel' + - 'server' + - 'run' + - '--config' + - '/etc/config/funnel-server.yaml' + resources: + requests: + cpu: {{ .Values.resources.requests.cpu }} + memory: {{ .Values.resources.requests.memory }} + limits: + cpu: {{ .Values.resources.limits.cpu }} + memory: {{ .Values.resources.limits.memory }} + ports: + - name: http + containerPort: 8000 + - name: rpc + containerPort: 9090 + readinessProbe: + httpGet: + path: /healthz + port: 8000 + periodSeconds: 10 + timeoutSeconds: 3 + successThreshold: 1 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /healthz + port: 8000 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 3 + volumeMounts: + {{- toYaml .Values.volumeMounts | nindent 10 }} + + volumes: + {{- toYaml .Values.volumes | nindent 6 }} diff --git a/helm/funnel/templates/service.yaml b/helm/funnel/templates/service.yaml new file mode 100644 index 000000000..8780e71fa --- /dev/null +++ b/helm/funnel/templates/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: funnel + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + ports: + - name: http + protocol: TCP + port: 8000 + targetPort: 8000 + - name: rpc + protocol: TCP + port: 9090 + targetPort: 9090 + selector: + {{- include "funnel.selectorLabels" . | nindent 4 }} diff --git a/helm/funnel/templates/serviceaccount.yaml b/helm/funnel/templates/serviceaccount.yaml new file mode 100644 index 000000000..203935c07 --- /dev/null +++ b/helm/funnel/templates/serviceaccount.yaml @@ -0,0 +1,22 @@ +{{- if .Values.rbac.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: funnel-sa-{{ .Release.Namespace }} + namespace: {{ .Values.Kubernetes.Namespace }} + labels: + {{- include "funnel.labels" . | nindent 4 }} +automountServiceAccountToken: true +{{- if and .Values.Kubernetes.JobsNamespace (ne .Values.Kubernetes.JobsNamespace .Values.Kubernetes.Namespace) }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: funnel-sa-{{ .Release.Namespace }} + namespace: {{ .Values.Kubernetes.JobsNamespace }} + labels: + {{- include "funnel.labels" . | nindent 4 }} +automountServiceAccountToken: true +{{- end }} +{{- end }} diff --git a/helm/funnel/templates/tests/test-connection.yaml b/helm/funnel/templates/tests/test-connection.yaml new file mode 100644 index 000000000..cb2d151cc --- /dev/null +++ b/helm/funnel/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "funnel.fullname" . }}-test-connection" + labels: + {{- include "funnel.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "funnel.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/helm/funnel/templates/worker-configmap.yaml b/helm/funnel/templates/worker-configmap.yaml new file mode 100644 index 000000000..5974512c6 --- /dev/null +++ b/helm/funnel/templates/worker-configmap.yaml @@ -0,0 +1,22 @@ +# This is the config containing the templates used by the Funnel Worker +apiVersion: v1 +kind: ConfigMap +metadata: + name: funnel-worker-templates + namespace: {{ .Release.Namespace }} + +data: + +{{ (.Files.Glob "files/worker-pv.yaml").AsConfig | indent 2 }} + +{{ (.Files.Glob "files/worker-pvc.yaml").AsConfig | indent 2 }} + +{{ (.Files.Glob "files/worker-job.yaml").AsConfig | indent 2 }} + +{{ (.Files.Glob "files/executor-job.yaml").AsConfig | indent 2 }} + +{{ (.Files.Glob "files/serviceaccount.yaml").AsConfig | indent 2 }} + +{{ (.Files.Glob "files/role.yaml").AsConfig | indent 2 }} + +{{ (.Files.Glob "files/rolebinding.yaml").AsConfig | indent 2 }} diff --git a/helm/funnel/values.yaml b/helm/funnel/values.yaml index 6549c86a4..e49125757 100644 --- a/helm/funnel/values.yaml +++ b/helm/funnel/values.yaml @@ -1,7 +1,3 @@ -# Default values for funnel. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - global: aws: # -- (string) AWS region for this deployment @@ -116,6 +112,9 @@ netPolicy: egressApps: - gen3-workflow +# Kubernetes-specific settings +replicaCount: 1 + # Values to determine the labels that are used for the deployment, pod, etc. # -- (string) Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". release: "production" @@ -126,118 +125,148 @@ partOf: "Workflow_Execution" # -- (bool) Whether to create a job to generate the OIDC client for Funnel. oidc_job_enabled: true +image: + # -- (string) The Docker image repository for the Funnel service. + repository: quay.io/ohsu-comp-bio/funnel + tag: develop-2026-07-09-19-49-55Z-97d1df55 + + # -- (string) When to pull the image. This value should be "Always" to ensure the latest image is used. + pullPolicy: Always + + # -- (map) Configuration for the Funnel init container. + initContainers: + - name: plugin + # -- (string) The Docker image repository for the Funnel init/plugin container. + image: quay.io/cdis/funnel-gen3-plugin + # -- (string) The Docker image tag for the Funnel init/plugin container. + tag: main-gen3 + # -- (string) When to pull the image. This value should be "Always" to ensure the latest image is used. + pullPolicy: Always + # -- (list) Arguments to pass to the init container. + command: + - cp + - /app/build/plugins/authorizer + - /opt/funnel/plugin-binaries/auth-plugin + volumeMounts: + - name: plugin-volume + mountPath: /opt/funnel/plugin-binaries + - name: config-updater + image: quay.io/cdis/awshelper + tag: master + env: + - name: FUNNEL_OIDC_CLIENT_ID + valueFrom: + secretKeyRef: + name: funnel-oidc-client + key: client_id + optional: false + - name: FUNNEL_OIDC_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: funnel-oidc-client + key: client_secret + optional: false + - name: DB_HOST + valueFrom: + secretKeyRef: + name: funnel-dbcreds + key: host + optional: false + - name: DB_USER + valueFrom: + secretKeyRef: + name: funnel-dbcreds + key: username + optional: false + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: funnel-dbcreds + key: password + optional: false + - name: DB_DATABASE + valueFrom: + secretKeyRef: + name: funnel-dbcreds + key: database + optional: false + volumeMounts: + - name: funnel-patched-config-volume + mountPath: /tmp + - name: funnel-config-volume + mountPath: /etc/config/funnel.conf + subPath: funnel-server.yaml + command: ["/bin/bash"] + args: + - "-c" + - | + # Create a funnel-patched.conf since /etc/config/funnel.conf is readonly + CONFIG=/tmp/funnel-patched.conf + cp /etc/config/funnel.conf $CONFIG + + namespace=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) + JOBS_NAMESPACE=workflow-pods-$namespace + S3_URL=gen3-workflow-service.$namespace.svc.cluster.local + DB_HOST=$DB_HOST:5432 + + # `Kubernetes.JobsNamespace` has to be configured manually because of templating + # limitations. This ensures it is configured to the value that is hardcoded elsewhere. + configured=$(yq -r '.Kubernetes.JobsNamespace' "$CONFIG") + if [[ "$configured" != "$JOBS_NAMESPACE" ]]; then + echo "ERROR: funnel.Kubernetes.JobsNamespace is set to '$configured' instead of '$JOBS_NAMESPACE'. Please fix the configuration" >&2 + exit 1 + fi + + echo "======= Funnel configuration =======" + echo " Kubernetes.JobsNamespace : $JOBS_NAMESPACE" + echo " Plugins.Params.OidcClientId: $FUNNEL_OIDC_CLIENT_ID" + echo " Plugins.Params.S3Url : $S3_URL" + echo " Postgres.Host : $DB_HOST" + echo " Postgres.Database : $DB_DATABASE" + echo " Postgres.User : $DB_USER" + echo "====================================" + + # Replace placeholders with actual values (in-place) + sed -i "s|FUNNEL_PLUGIN_OIDC_CLIENT_ID_PLACEHOLDER|${FUNNEL_OIDC_CLIENT_ID}|g" $CONFIG + sed -i "s|FUNNEL_PLUGIN_OIDC_CLIENT_SECRET_PLACEHOLDER|${FUNNEL_OIDC_CLIENT_SECRET}|g" $CONFIG + sed -i "s|FUNNEL_PLUGIN_S3URL_PLACEHOLDER|${S3_URL}|g" $CONFIG + sed -i "s/FUNNEL_POSTGRES_HOST_PLACEHOLDER/${DB_HOST}/g" $CONFIG + sed -i "s/FUNNEL_POSTGRES_DATABASE_PLACEHOLDER/${DB_DATABASE}/g" $CONFIG + sed -i "s/FUNNEL_POSTGRES_USER_PLACEHOLDER/${DB_USER}/g" $CONFIG + sed -i "s/FUNNEL_POSTGRES_PASSWORD_PLACEHOLDER/${DB_PASSWORD}/g" $CONFIG + + +labels: + app: funnel -funnel: - # -- (map) Configuration for the Funnel container image. - image: - # -- (string) The Docker image repository for the Funnel service. - repository: quay.io/ohsu-comp-bio/funnel - # -- (string) When to pull the image. This value should be "Always" to ensure the latest image is used. - pullPolicy: Always - - # -- (map) Configuration for the Funnel init container. - initContainers: - - name: plugin - # -- (string) The Docker image repository for the Funnel init/plugin container. - image: quay.io/cdis/funnel-gen3-plugin - # -- (string) The Docker image tag for the Funnel init/plugin container. - tag: main-gen3 - # -- (string) When to pull the image. This value should be "Always" to ensure the latest image is used. - pullPolicy: Always - # -- (list) Arguments to pass to the init container. - command: - - cp - - /app/build/plugins/authorizer - - /opt/funnel/plugin-binaries/auth-plugin - volumeMounts: - - name: plugin-volume - mountPath: /opt/funnel/plugin-binaries - - name: config-updater - image: quay.io/cdis/awshelper - tag: master - env: - - name: FUNNEL_OIDC_CLIENT_ID - valueFrom: - secretKeyRef: - name: funnel-oidc-client - key: client_id - optional: false - - name: FUNNEL_OIDC_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: funnel-oidc-client - key: client_secret - optional: false - - name: DB_HOST - valueFrom: - secretKeyRef: - name: funnel-dbcreds - key: host - optional: false - - name: DB_USER - valueFrom: - secretKeyRef: - name: funnel-dbcreds - key: username - optional: false - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: funnel-dbcreds - key: password - optional: false - - name: DB_DATABASE - valueFrom: - secretKeyRef: - name: funnel-dbcreds - key: database - optional: false - volumeMounts: - - name: funnel-patched-config-volume - mountPath: /tmp - - name: funnel-config-volume - mountPath: /etc/config/funnel.conf - subPath: funnel-server.yaml - command: ["/bin/bash"] - args: - - "-c" - - | - # Create a funnel-patched.conf since /etc/config/funnel.conf is readonly - CONFIG=/tmp/funnel-patched.conf - cp /etc/config/funnel.conf $CONFIG - - namespace=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) - JOBS_NAMESPACE=workflow-pods-$namespace - S3_URL=gen3-workflow-service.$namespace.svc.cluster.local - DB_HOST=$DB_HOST:5432 - - # `Kubernetes.JobsNamespace` has to be configured manually because of templating - # limitations. This ensures it is configured to the value that is hardcoded elsewhere. - configured=$(yq -r '.Kubernetes.JobsNamespace' "$CONFIG") - if [[ "$configured" != "$JOBS_NAMESPACE" ]]; then - echo "ERROR: funnel.Kubernetes.JobsNamespace is set to '$configured' instead of '$JOBS_NAMESPACE'. Please fix the configuration" >&2 - exit 1 - fi - - echo "======= Funnel configuration =======" - echo " Kubernetes.JobsNamespace : $JOBS_NAMESPACE" - echo " Plugins.Params.OidcClientId: $FUNNEL_OIDC_CLIENT_ID" - echo " Plugins.Params.S3Url : $S3_URL" - echo " Postgres.Host : $DB_HOST" - echo " Postgres.Database : $DB_DATABASE" - echo " Postgres.User : $DB_USER" - echo "====================================" - - # Replace placeholders with actual values (in-place) - sed -i "s|FUNNEL_PLUGIN_OIDC_CLIENT_ID_PLACEHOLDER|${FUNNEL_OIDC_CLIENT_ID}|g" $CONFIG - sed -i "s|FUNNEL_PLUGIN_OIDC_CLIENT_SECRET_PLACEHOLDER|${FUNNEL_OIDC_CLIENT_SECRET}|g" $CONFIG - sed -i "s|FUNNEL_PLUGIN_S3URL_PLACEHOLDER|${S3_URL}|g" $CONFIG - sed -i "s/FUNNEL_POSTGRES_HOST_PLACEHOLDER/${DB_HOST}/g" $CONFIG - sed -i "s/FUNNEL_POSTGRES_DATABASE_PLACEHOLDER/${DB_DATABASE}/g" $CONFIG - sed -i "s/FUNNEL_POSTGRES_USER_PLACEHOLDER/${DB_USER}/g" $CONFIG - sed -i "s/FUNNEL_POSTGRES_PASSWORD_PLACEHOLDER/${DB_PASSWORD}/g" $CONFIG - - volumes: +rbac: + create: true + +# AWS STS Source + Region for S3 CSI Driver +authenticationSource: pod +stsRegion: us-east-1 + +# Kubernetes Service Settings +service: + type: ClusterIP + httpPort: 8000 + rpcPort: 9090 + + +# Funnel default settings configured for Gen3-managed PostgreSQL. +# +# These are passed into `files/server-config.yaml` and made available to the deployment via `server-configmap.yaml` +# +# - Ref: https://github.com/ohsu-comp-bio/funnel/blob/master/config/default-config.yaml + +# The name of the active compute backend +# Available backends: local, htcondor, slurm, pbs, gridengine, manual, aws-batch, kubernetes +Compute: kubernetes + + +# Overrides the default volumes configured in the server deployment. +# If custom volumes are specified, they will fully replace the default values. +volumes: - name: funnel-config-volume configMap: name: funnel-server-config @@ -260,63 +289,489 @@ funnel: - name: funnel-patched-config-volume emptyDir: {} # Shared volume for config data - volumeMounts: - - name: funnel-patched-config-volume - mountPath: /etc/config/funnel-server.yaml - subPath: funnel-patched.conf +# Overrides the default volumesMounts configured in the server deployment. +# If custom volumesMounts are specified, they will fully replace the default values. +volumeMounts: + - name: funnel-patched-config-volume + mountPath: /etc/config/funnel-server.yaml + subPath: funnel-patched.conf + + - name: "funnel-oidc-volume" + readOnly: true + mountPath: "/etc/config/oidc" - - name: "funnel-oidc-volume" - readOnly: true - mountPath: "/etc/config/oidc" + - name: worker-templates-volume + mountPath: /etc/funnel/templates - - name: worker-templates-volume - mountPath: /etc/funnel/templates + - name: plugin-volume + mountPath: /opt/funnel/plugin-binaries - - name: plugin-volume - mountPath: /opt/funnel/plugin-binaries +# Resource Requests/Limits for worker jobs and server pods. +resources: + requests: + cpu: 100m + memory: "2Gi" + ephemeral_storage: "2Gi" + limits: + cpu: 1000m + memory: "2Gi" + ephemeral_storage: "2Gi" - resources: - requests: - memory: "2Gi" - ephemeral_storage: "2Gi" +# The name of the active server database backend +# Available backends: boltdb, badger, datastore, dynamodb, elastic, mongodb, postgres +Database: postgres +# The name of the active event writer backend(s). +# Available backends: log, boltdb, badger, datastore, dynamodb, elastic, mongodb, postgres, kafka +EventWriters: + - postgres + - log +Postgres: + Host: FUNNEL_POSTGRES_HOST_PLACEHOLDER + Database: FUNNEL_POSTGRES_DATABASE_PLACEHOLDER + User: FUNNEL_POSTGRES_USER_PLACEHOLDER + Password: FUNNEL_POSTGRES_PASSWORD_PLACEHOLDER + AdminUser: postgres + AdminPassword: example + Timeout: + duration: 300s +Logger: + # Logging levels: debug, info, error + Level: debug + # Write logs to this path. If empty, logs are written to stderr. + OutputFile: "" + # Log format: json or text + Formatter: json + # Text format settings + TextFormat: + # Try to force colors/rich format in text output + ForceColors: true + # Include full timestamps in text output + FullTimestamp: true + # Format for timestamps. RFC3339 is default. + TimestampFormat: "2006-01-02T15:04:05Z07:00" +Plugins: + Path: plugin-binaries/auth-plugin + Params: + OidcClientId: FUNNEL_PLUGIN_OIDC_CLIENT_ID_PLACEHOLDER + OidcClientSecret: FUNNEL_PLUGIN_OIDC_CLIENT_SECRET_PLACEHOLDER + S3Url: FUNNEL_PLUGIN_S3URL_PLACEHOLDER + +# Kubernetes describes the configuration for the Kubernetes compute backend. +Kubernetes: + # Timeout for task creation (worker pod creation). On timeout, tasks fail with SYSTEM_ERROR + Timeout: + duration: 300s + ReconcileRate: 120s + Executor: + PriorityClassName: "" + restartPolicy: Never + # Setting backoffLimit to 0 means no retries. + backoffLimit: 0 + Annotations: + karpenter.sh/do-not-disrupt: "true" + # Setting completions to 1 means the job completes after one successful execution. + completions: 1 - Database: postgres - EventWriters: - - postgres - - log - postgresql: # default bitnami postgres pod - enabled: false - Postgres: - Host: FUNNEL_POSTGRES_HOST_PLACEHOLDER - Database: FUNNEL_POSTGRES_DATABASE_PLACEHOLDER - User: FUNNEL_POSTGRES_USER_PLACEHOLDER - Password: FUNNEL_POSTGRES_PASSWORD_PLACEHOLDER - Logger: - Level: info - Plugins: - Path: plugin-binaries/auth-plugin - Params: - OidcClientId: FUNNEL_PLUGIN_OIDC_CLIENT_ID_PLACEHOLDER - OidcClientSecret: FUNNEL_PLUGIN_OIDC_CLIENT_SECRET_PLACEHOLDER - S3Url: FUNNEL_PLUGIN_S3URL_PLACEHOLDER Worker: - # This lets mount-s3 pods unmount and Complete. https://ctds-planx.atlassian.net/browse/MIDRC-1214 - LeaveWorkDir: true - Kubernetes: - # Timeout for task creation (worker pod creation). On timeout, tasks fail with SYSTEM_ERROR - Timeout: - duration: 300s - ReconcileRate: 120s - Executor: - restartPolicy: Never - backoffLimit: 0 - Annotations: - karpenter.sh/do-not-disrupt: "true" - Worker: - restartPolicy: Never - backoffLimit: 1 - Annotations: - karpenter.sh/do-not-disrupt: "true" - # When a new node is ready, worker pods may start before system-level pods do. This ensures - # worker pods are not preempted by new system-level pods on that node. - PriorityClassName: "system-cluster-critical" + + restartPolicy: Never + backoffLimit: 0 + completions: 1 + Annotations: + karpenter.sh/do-not-disrupt: "true" + # When a new node is ready, worker pods may start before system-level pods do. This ensures + # worker pods are not preempted by new system-level pods on that node. + PriorityClassName: "system-cluster-critical" + + # Turn off task state reconciler. When enabled, Funnel communicates with Kubernetes + # to find tasks that are stuck in a queued state or errored and + # updates the task state accordingly. + DisableReconciler: false + DisableJobCleanup: false + # -- Path prefixes that Funnel's Kubernetes backend must reject when creating worker pods. + ForbiddenPathPrefixes: [] + # ReconcileRate is how often the compute backend compares states in Funnel's backend + # to those reported by the backend + # Kubernetes Namespace to spawn jobs within + Namespace: "" + # Kubernetes Namespace to spawn jobs within + JobsNamespace: "" + # Kubernetes ServiceAccount to use for the job + ServiceAccount: "" + + # Master batch job template. See: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#job-v1-batch + WorkerTemplate: "" + # NodeSelector (scheduling) + NodeSelector: {} + # Tolerations (scheduling) + Tolerations: [] + + # Job template used for executing the tasks. + ExecutorTemplate: "" + PVTemplate: "" + PVCTemplate: "" + + Resources: + Defaults: + Cpus: 1000m + RamGb: 512Mi + DiskGb: 512Mi + Limits: + Cpus: 8000m + RamGb: 4096Mi + DiskGb: 4096Mi + + # Funnel Worker + Executor storage (S3 bucket) + Storage: + # type: either "mountpoint_s3" or "s3files" + type: mountpoint_s3 +# cleanup configures an optional CronJob that runs `funnel kubernetes cleanup` +# to delete orphaned Funnel-managed Kubernetes resources. It is intentionally +# turned off by default; set cleanup.enabled=true to enable it. +cleanup: + enabled: false + # Leave empty to derive the schedule from Kubernetes.ReconcileRate. Set a + # standard 5-field cron expression to override, e.g. "*/15 * * * *". + schedule: "" + # Offset the start minute so cleanup jobs in different namespaces do not all + # fire at the same instant. Must be 0-59. + scheduleOffsetMinutes: 0 + +# ------------------------------------------------------------------------------- +# Storage +# ------------------------------------------------------------------------------- + +# If possible, credentials will be automatically discovered +# from the environment. + +# -- Local file system storage configuration. +LocalStorage: + # Whitelist of local directory paths which Funnel is allowed to access. + AllowedDirs: + - ./ + +# HTTPStorage is used to download public files on the web via a GET request. +HTTPStorage: + # Timeout for http(s) GET requests. + Timeout: 30s + +AmazonS3: + Disabled: false + # The maximum number of times that a request will be retried for failures. + AWSConfig: + MaxRetries: 10 + # AWS Access key ID + Key: "" + # AWS Secret Access Key + Secret: "" + # Server Side Encryption (SSE) settings + SSE: + # Customer Provided Key + # ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html + CustomerKeyFile: "" + # KMS Key + # ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html + KMSKey: "" + +# Configure storage backends for S3 providers such as Minio and/or Ceph +# GenericS3: +# - Disabled: true +# Endpoint: "" +# Key: "" +# Secret: "" +# KmsKeyID: "" + +GoogleStorage: + Disabled: false + # Path to account credentials file. + # Optional. If possible, credentials will be automatically discovered + # from the environment. + CredentialsFile: "" +# Available backends: local, htcondor, slurm, pbs, gridengine, manual, aws-batch, kubernetes/Kube +Swift: + Disabled: false + UserName: "" + Password: "" + AuthURL: "" + TenantName: "" + TenantID: "" + RegionName: "" + # 500 MB + ChunkSizeBytes: 500000000 + +FTPStorage: + Disabled: false + Timeout: 10s + User: "anonymous" + Password: "anonymous" + +Server: + # Hostname of the Funnel server. + HostName: funnel + + # Port used for HTTP communication and the web dashboard. + HTTPPort: "8000" + + # Port used for RPC communication. + RPCPort: "9090" + + # Require basic authentication for the server APIs using a password. + # If used, make sure to properly restrict access to the config file + # (e.g. chmod 600 funnel.config.yml) + # BasicAuth: + # - User: user1 + # Password: abc123 + # - User: user2 + # Password: foobar + + # Include a "Cache-Control: no-store" HTTP header in Get/List responses + # to prevent caching by intermediary services. + DisableHTTPCache: true + +RPCClient: + # RPC server address + ServerAddress: localhost:9090 + + # Credentials for Basic authentication for the server APIs using a password. + # If used, make sure to properly restrict access to the config file + # (e.g. chmod 600 funnel.config.yml) + # User: funnel + # Password: abc123 + + # connection timeout. + Timeout: + duration: 60s + + # The maximum number of times that a request will be retried for failures. + # Time between retries follows an exponential backoff starting at 5 seconds + # up to 1 minute + MaxRetries: 10 + +# The scheduler is used for the Manual compute backend. +Scheduler: + # How often to run a scheduler iteration. + ScheduleRate: 1s + # How many tasks to schedule in one iteration. + ScheduleChunk: 10 + # How long to wait between updates before marking a node dead. + NodePingTimeout: + duration: 60s + # How long to wait for a node to start, before marking the node dead. + NodeInitTimeout: + duration: 300s + +Node: + # If empty, a node ID will be automatically generated. + ID: "" + + # If the node has been idle for longer than the timeout, it will shut down. + # -1 means there is no timeout. 0 means timeout immediately after the first task. + Timeout: + disabled: true + + # A Node will automatically try to detect what resources are available to it. + # Defining Resources in the Node configuration overrides this behavior. + Resources: + # CPUs available. + Cpus: 0 + + # RAM available, in GB. + RamGb: 0.0 + + # Disk space available, in GB. + DiskGb: 0.0 + + # For low-level tuning. + # How often to sync with the Funnel server. + UpdateRate: 5s + +Worker: + # Files created during processing will be written in this directory. + WorkDir: ./funnel-work-dir + + # For low-level tuning. + # How often to poll for cancel signals + PollingRate: 5s + + # For low-level tuning. + # How often to send stdout/err task log updates to the Funnel server. + # Setting this to 0 will result in these fields being updated a single time + # after the executor exits. + LogUpdateRate: 5s + + # Max bytes to store for stdout/err in the task log (10 KB) + LogTailSize: 10000 + + # Normally the worker deletes its working directory after executing. + # This option disables that behavior. + # This lets mount-s3 pods unmount and Complete. https://ctds-planx.atlassian.net/browse/MIDRC-1214 + LeaveWorkDir: true + + # Limit the number of concurrent downloads/uploads + MaxParallelTransfers: 10 + +# ------------------------------------------------------------------------------- +# Databases and/or Event Writers/Handlers +# ------------------------------------------------------------------------------- + +# -- Local file database configuration. +BoltDB: + # Path to the database file + Path: ./funnel-work-dir/funnel.db + +DynamoDB: + # Basename to use for dynamodb tables + TableBasename: funnel + AWSConfig: + # AWS region + Region: "" + # AWS Access key ID + Key: "" + # AWS Secret Access Key + Secret: "" + +Elastic: + # Prefix to use for indexes (task, events, nodes) + IndexPrefix: funnel + # URL of the elasticsearch server. + URL: http://localhost:9200 + +# Google Cloud Datastore task database. +Datastore: + Project: "" + # Path to account credentials file. + # Optional. If possible, credentials will be automatically discovered + # from the environment. + CredentialsFile: "" + + Timeout: + duration: 300s + +Kafka: + Topic: funnel + +# ------------------------------------------------------------------------------- +# Compute Backends +# ------------------------------------------------------------------------------- + +# -- HTCondor compute backend configuration. +HTCondor: + # Turn off task state reconciler. When enabled, Funnel communicates with the HPC + # scheduler to find tasks that are stuck in a queued state or errored and + # updates the task state accordingly. + DisableReconciler: true + # ReconcileRate is how often the compute backend compares states in Funnel's backend + # to those reported by the backend + ReconcileRate: 10s + TemplateFile: "" + Template: | + universe = vanilla + getenv = True + executable = {{.Executable}} + arguments = worker run --config {{.Config}} --task-id {{.TaskId}} + log = {{.WorkDir}}/condor-event-log + error = {{.WorkDir}}/funnel-stderr + output = {{.WorkDir}}/funnel-stdout + should_transfer_files = YES + when_to_transfer_output = ON_EXIT_OR_EVICT + {{if ne .Cpus 0 -}} + {{printf "request_cpus = %d" .Cpus}} + {{- end}} + {{if ne .RamGb 0.0 -}} + {{printf "request_memory = %.0f GB" .RamGb}} + {{- end}} + {{if ne .DiskGb 0.0 -}} + {{printf "request_disk = %.0f GB" .DiskGb}} + {{- end}} + + queue + +PBS: + # Turn off task state reconciler. When enabled, Funnel communicates with the HPC + # scheduler to find tasks that are stuck in a queued state or errored and + # updates the task state accordingly. + DisableReconciler: true + # ReconcileRate is how often the compute backend compares states in Funnel's backend + # to those reported by the backend + ReconcileRate: 10s + TemplateFile: "" + Template: | + #!bin/bash + #PBS -N {{.TaskId}} + #PBS -o {{.WorkDir}}/funnel-stdout + #PBS -e {{.WorkDir}}/funnel-stderr + {{if ne .Cpus 0 -}} + {{printf "#PBS -l nodes=1:ppn=%d" .Cpus}} + {{- end}} + {{if ne .RamGb 0.0 -}} + {{printf "#PBS -l mem=%.0fgb" .RamGb}} + {{- end}} + {{if ne .DiskGb 0.0 -}} + {{printf "#PBS -l file=%.0fgb" .DiskGb}} + {{- end}} + + {{.Executable}} worker run --config {{.Config}} --taskID {{.TaskId}} + +GridEngine: + TemplateFile: "" + Template: | + #!bin/bash + #$ -N {{.TaskId}} + #$ -o {{.WorkDir}}/funnel-stdout + #$ -e {{.WorkDir}}/funnel-stderr + #$ -l nodes=1 + {{if ne .Cpus 0 -}} + {{printf "#$ -pe mpi %d" .Cpus}} + {{- end}} + {{if ne .RamGb 0.0 -}} + {{printf "#$ -l h_vmem=%.0fG" .RamGb}} + {{- end}} + {{if ne .DiskGb 0.0 -}} + {{printf "#$ -l h_fsize=%.0fG" .DiskGb}} + {{- end}} + + {{.Executable}} worker run --config {{.Config}} --taskID {{.TaskId}} + +Slurm: + # Turn off task state reconciler. When enabled, Funnel communicates with the HPC + # scheduler to find tasks that are stuck in a queued state or errored and + # updates the task state accordingly. + DisableReconciler: true + # ReconcileRate is how often the compute backend compares states in Funnel's backend + # to those reported by the backend + ReconcileRate: 10s + TemplateFile: "" + Template: | + #!/bin/bash + #SBATCH --job-name {{.TaskId}} + #SBATCH --ntasks 1 + #SBATCH --error {{.WorkDir}}/funnel-stderr + #SBATCH --output {{.WorkDir}}/funnel-stdout + {{if ne .Cpus 0 -}} + {{printf "#SBATCH --cpus-per-task %d" .Cpus}} + {{- end}} + {{if ne .RamGb 0.0 -}} + {{printf "#SBATCH --mem %.0fGB" .RamGb}} + {{- end}} + {{if ne .DiskGb 0.0 -}} + {{printf "#SBATCH --tmp %.0fGB" .DiskGb}} + {{- end}} + + {{.Executable}} worker run --config {{.Config}} --taskID {{.TaskId}} + +# AWSBatch describes the configuration for the AWS Batch compute backend. +AWSBatch: + # Turn off task state reconciler. When enabled, Funnel communicates with AWS Batch + # to find tasks that never started and updates task state accordingly. + DisableReconciler: true + # ReconcileRate is how often the compute backend compares states in Funnel's backend + # to those reported by AWS Batch + ReconcileRate: 10s + # JobDefinition can be either a name or the Amazon Resource Name (ARN). + JobDefinition: "funnel-job-def" + # JobQueue can be either a name or the Amazon Resource Name (ARN). + JobQueue: "funnel-job-queue" + # AWS region of the specified job queue and to create the job definition in + Region: "" + Key: "" + Secret: "" diff --git a/helm/gen3-analysis/Chart.yaml b/helm/gen3-analysis/Chart.yaml index 9c233f3f9..100028e76 100644 --- a/helm/gen3-analysis/Chart.yaml +++ b/helm/gen3-analysis/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.12 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/gen3-analysis/README.md b/helm/gen3-analysis/README.md index 143a09f27..c3da7063a 100644 --- a/helm/gen3-analysis/README.md +++ b/helm/gen3-analysis/README.md @@ -1,14 +1,18 @@ # gen3-analysis -![Version: 0.1.12](https://img.shields.io/badge/Version-0.1.12-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 gen3-analysis Service +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values diff --git a/helm/gen3-embeddings/.helmignore b/helm/gen3-embeddings/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/helm/gen3-embeddings/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/gen3-embeddings/Chart.yaml b/helm/gen3-embeddings/Chart.yaml new file mode 100644 index 000000000..6c0d56329 --- /dev/null +++ b/helm/gen3-embeddings/Chart.yaml @@ -0,0 +1,32 @@ +apiVersion: v2 +name: gen3-embeddings +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.0.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "main" +dependencies: + - name: common + version: "*" + repository: file://../common + - name: postgresql + version: 11.9.13 + repository: "https://charts.bitnami.com/bitnami" + condition: postgres.separate diff --git a/helm/gen3-embeddings/README.md b/helm/gen3-embeddings/README.md new file mode 100644 index 000000000..408e6e90d --- /dev/null +++ b/helm/gen3-embeddings/README.md @@ -0,0 +1,101 @@ +# gen3-embeddings + +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: main](https://img.shields.io/badge/AppVersion-main-informational?style=flat-square) + +A Helm chart for Kubernetes + +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + +## Requirements + +| Repository | Name | +|------------|------| +| file://../common | common | +| https://charts.bitnami.com/bitnami | postgresql | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| automountServiceAccountToken | bool | `false` | | +| autoscaling | object | `{}` | | +| commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | +| criticalService | string | `"false"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | +| debug | bool | `false` | | +| env | list | `[{"name":"GEN3_DEBUG","value":"false"},{"name":"ARBORIST_URL","valueFrom":{"configMapKeyRef":{"key":"arborist_url","name":"manifest-global","optional":true}}},{"name":"PGPOOL_MIN_SIZE","value":"1"},{"name":"PGPOOL_MAX_SIZE","value":"5"},{"name":"GUNICORN_WORKERS","value":"2"}]` | Environment variables to pass to the container | +| externalSecrets | map | `{"createK8sGen3EmbeddingsSecret":false,"dbcreds":null,"gen3EmbeddingsG3auto":null,"pushSecret":false}` | External Secrets settings. | +| externalSecrets.createK8sGen3EmbeddingsSecret | string | `false` | Will create the Helm "gen3Embeddings-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. | +| externalSecrets.dbcreds | string | `nil` | Will override the name of the aws secrets manager secret. Default is "Values.global.environment-.Chart.Name-creds" | +| externalSecrets.gen3EmbeddingsG3auto | string | `nil` | Will override the name of the aws secrets manager secret. Default is "gen3UserDataLibrary-g3auto" | +| externalSecrets.pushSecret | bool | `false` | Whether to create the database and Secrets Manager secrets via PushSecret. | +| fullnameOverride | string | `""` | | +| global.autoscaling.averageCPUValue | string | `"500m"` | | +| global.autoscaling.averageMemoryValue | string | `"500Mi"` | | +| global.autoscaling.enabled | bool | `false` | | +| global.autoscaling.maxReplicas | int | `10` | | +| global.autoscaling.minReplicas | int | `1` | | +| global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false,"externalSecrets":{"enabled":false,"externalSecretAwsCreds":null}}` | AWS configuration | +| global.aws.awsAccessKeyId | string | `nil` | Credentials for AWS stuff. | +| global.aws.awsSecretAccessKey | string | `nil` | Credentials for AWS stuff. | +| global.aws.enabled | bool | `false` | Set to true if deploying to AWS. Controls ingress annotations. | +| global.aws.externalSecrets.enabled | bool | `false` | Whether to use External Secrets for aws config. | +| global.aws.externalSecrets.externalSecretAwsCreds | String | `nil` | Name of Secrets Manager secret. | +| global.dev | bool | `true` | Whether the deployment is for development purposes. | +| global.externalSecrets | map | `{"deploy":false,"separateSecretStore":false}` | External Secrets settings. | +| global.externalSecrets.deploy | bool | `false` | Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any metadata secrets you have deployed. | +| global.externalSecrets.separateSecretStore | string | `false` | Will deploy a separate External Secret Store for this service. | +| global.postgres.dbCreate | bool | `true` | Whether the database should be created. | +| global.postgres.externalSecret | string | `""` | Name of external secret. Disabled if empty | +| global.postgres.master | map | `{"host":null,"password":null,"port":"5432","username":"postgres"}` | Master credentials to postgres. This is going to be the default postgres server being used for each service, unless each service specifies their own postgres | +| global.postgres.master.host | string | `nil` | hostname of postgres server | +| global.postgres.master.password | string | `nil` | password for superuser in postgres. This is used to create or restore databases | +| global.postgres.master.port | string | `"5432"` | Port for Postgres. | +| global.postgres.master.username | string | `"postgres"` | username of superuser in postgres. This is used to create or restore databases | +| global.topologySpread | map | `{"enabled":false,"maxSkew":1,"topologyKey":"topology.kubernetes.io/zone"}` | Karpenter topology spread configuration. | +| global.topologySpread.enabled | bool | `false` | Whether to enable topology spread constraints for all subcharts that support it. | +| global.topologySpread.maxSkew | int | `1` | The maxSkew to use for topology spread constraints. Defaults to 1. | +| global.topologySpread.topologyKey | string | `"topology.kubernetes.io/zone"` | The topology key to use for spreading. Defaults to "topology.kubernetes.io/zone". | +| gunicornWorkers | int | `1` | | +| image.pullPolicy | string | `"Always"` | | +| image.repository | string | `"quay.io/cdis/gen3_embeddings"` | | +| image.tag | string | `"main"` | | +| ingress.annotations | object | `{}` | | +| ingress.className | string | `""` | | +| ingress.enabled | bool | `false` | | +| ingress.hosts[0].host | string | `"chart-example.local"` | | +| ingress.hosts[0].paths[0].path | string | `"/"` | | +| ingress.hosts[0].paths[0].pathType | string | `"ImplementationSpecific"` | | +| ingress.tls | list | `[]` | | +| livenessProbe.httpGet.path | string | `"/"` | | +| livenessProbe.httpGet.port | string | `"http"` | | +| metricsEnabled | bool | `nil` | Whether Metrics are enabled. | +| nameOverride | string | `""` | | +| partOf | string | `"Embeddings"` | Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. | +| postgres | map | `{"database":null,"dbCreate":null,"dbRestore":false,"host":null,"password":null,"port":"5432","separate":false,"username":null}` | Postgres database configuration. If db does not exist in postgres cluster and dbCreate is set ot true then these databases will be created for you | +| postgres.database | string | `nil` | Database name for postgres. This is a service override, defaults to - | +| postgres.dbCreate | bool | `nil` | Whether the database should be created. Default to global.postgres.dbCreate | +| postgres.host | string | `nil` | Hostname for postgres server. This is a service override, defaults to global.postgres.host | +| postgres.password | string | `nil` | Password for Postgres. Will be autogenerated if left empty. | +| postgres.port | string | `"5432"` | Port for Postgres. | +| postgres.separate | string | `false` | Will create a Database for the individual service to help with developing it. | +| postgres.username | string | `nil` | Username for postgres. This is a service override, defaults to - | +| postgresql | map | `{"primary":{"persistence":{"enabled":false}}}` | Postgresql subchart settings if deployed separately option is set to "true". Disable persistence by default so we can spin up and down ephemeral environments | +| postgresql.primary.persistence.enabled | bool | `false` | Option to persist the dbs data. | +| readinessProbe.httpGet.path | string | `"/"` | | +| readinessProbe.httpGet.port | string | `"http"` | | +| release | string | `"production"` | Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". | +| replicaCount | int | `1` | | +| resources | object | `{}` | | +| secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null}` | Secret information to access the db restore job S3 bucket. | +| secrets.awsAccessKeyId | str | `nil` | AWS access key ID. Overrides global key. | +| secrets.awsSecretAccessKey | str | `nil` | AWS secret access key ID. Overrides global key. | +| selectorLabels | map | `nil` | Will completely override the selectorLabels defined in the common chart's _label_setup.tpl | +| service.port | int | `80` | | +| service.type | string | `"ClusterIP"` | | +| volumeMounts[0].mountPath | string | `"/services/gen3_embeddings/.env"` | | +| volumeMounts[0].name | string | `"gen3-embeddings-g3auto-volume"` | | +| volumeMounts[0].readOnly | bool | `true` | | +| volumeMounts[0].subPath | string | `"gen3-embeddings.env"` | | diff --git a/helm/gen3-embeddings/templates/NOTES.txt b/helm/gen3-embeddings/templates/NOTES.txt new file mode 100644 index 000000000..1aabeafed --- /dev/null +++ b/helm/gen3-embeddings/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "gen3-embeddings.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch its status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "gen3-embeddings.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "gen3-embeddings.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "gen3-embeddings.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/helm/gen3-embeddings/templates/_helpers.tpl b/helm/gen3-embeddings/templates/_helpers.tpl new file mode 100644 index 000000000..23381414d --- /dev/null +++ b/helm/gen3-embeddings/templates/_helpers.tpl @@ -0,0 +1,76 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "gen3-embeddings.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "gen3-embeddings.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "gen3-embeddings.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "gen3-embeddings.labels" -}} +{{- if .Values.commonLabels }} + {{- with .Values.commonLabels }} + {{- toYaml . }} + {{- end }} +{{- else }} + {{- (include "common.commonLabels" .)}} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "gen3-embeddings.selectorLabels" -}} +{{- if .Values.selectorLabels }} + {{- with .Values.selectorLabels }} + {{- toYaml . }} + {{- end }} +{{- else }} + {{- (include "common.selectorLabels" .)}} +{{- end }} +{{- end }} + +{{/* + Postgres Password lookup +*/}} +{{- define "gen3-embeddings.postgres.password" -}} +{{- $localpass := (lookup "v1" "Secret" "postgres" "postgres-postgresql" ) -}} +{{- if $localpass }} +{{- default (index $localpass.data "postgres-password" | b64dec) }} +{{- else }} +{{- default .Values.postgres.password }} +{{- end }} +{{- end }} + +{{/* + Gen3Embeddings g3 Auto Secrets Manager Name +*/}} +{{- define "gen3embeddings-g3auto" -}} +{{- default "gen3embeddings-g3auto" .Values.externalSecrets.gen3EmbeddingsG3auto }} +{{- end }} diff --git a/helm/gen3-embeddings/templates/db-init.yaml b/helm/gen3-embeddings/templates/db-init.yaml new file mode 100644 index 000000000..95297b084 --- /dev/null +++ b/helm/gen3-embeddings/templates/db-init.yaml @@ -0,0 +1,15 @@ +{{ include "common.db-secret" . }} +--- +{{ include "common.db_setup_sa" . }} +--- +{{- if .Values.dbRestore }} +{{ include "common.s3_pg_restore" . }} +{{- else }} +{{ include "common.db_setup_job" . }} +{{- end -}} +{{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} +--- +{{ include "common.db-push-secret" . }} +--- +{{ include "common.secret.db.bootstrap" . }} +{{- end }} diff --git a/helm/gen3-embeddings/templates/deployment.yaml b/helm/gen3-embeddings/templates/deployment.yaml new file mode 100644 index 000000000..aaaa229bf --- /dev/null +++ b/helm/gen3-embeddings/templates/deployment.yaml @@ -0,0 +1,215 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gen3-embeddings-deployment + labels: + {{- include "gen3-embeddings.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "gen3-embeddings.selectorLabels" . | nindent 6 }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- with .Values.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "gen3-embeddings.selectorLabels" . | nindent 8 }} + {{- include "common.extraLabels" . | nindent 8 }} + # gen3 networkpolicy labels + netnolimit: 'yes' + public: 'yes' + userhelper: 'yes' + annotations: + checksum/config: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} + {{- $metricsEnabled := .Values.metricsEnabled }} + {{- if eq $metricsEnabled nil }} + {{- $metricsEnabled = .Values.global.metricsEnabled }} + {{- end }} + {{- if eq $metricsEnabled nil }} + {{- $metricsEnabled = true }} + {{- end }} + + {{- if $metricsEnabled }} + {{- include "common.grafanaAnnotations" . | nindent 8 }} + {{- end }} + spec: + {{- if .Values.global.topologySpread.enabled }} + {{- include "common.TopologySpread" . | nindent 6 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + automountServiceAccountToken: {{ .Values.automountServiceAccountToken }} + volumes: + - name: gen3-embeddings-g3auto-volume + secret: + secretName: gen3embeddings-g3auto + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + env: + {{- toYaml .Values.env | nindent 12 }} + - name: PGHOST + valueFrom: + secretKeyRef: + name: gen3-embeddings-dbcreds + key: host + optional: false + - name: PGPORT + valueFrom: + secretKeyRef: + name: gen3-embeddings-dbcreds + key: port + optional: false + - name: PGUSER + valueFrom: + secretKeyRef: + name: gen3-embeddings-dbcreds + key: username + optional: false + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: gen3-embeddings-dbcreds + key: password + optional: false + - name: PGDATABASE + valueFrom: + secretKeyRef: + name: gen3-embeddings-dbcreds + key: database + optional: false + - name: CONFIG_PATH + value: /services/gen3_embeddings/.env + {{- if eq .Values.global.dev false }} + - name: FENCE_URL + value: https://{{ .Values.global.hostname }}/user + {{- else }} + - name: FENCE_URL + value: {{ default "http://fence-service" .Values.global.fenceURL | quote }} + {{- end }} + - name: DBREADY + valueFrom: + secretKeyRef: + name: gen3-embeddings-dbcreds + key: dbcreated + optional: false + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - containerPort: 4142 + name: http + livenessProbe: + httpGet: + path: /_status + port: 4142 + initialDelaySeconds: 30 + periodSeconds: 60 + timeoutSeconds: 30 + readinessProbe: + httpGet: + path: /_status + port: 4142 + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + initContainers: + - name: gen3-embeddings-init + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: GEN3_DEBUG + value: "False" + # Admin/master connection – same pattern as db_setup_job + - name: PGPASSWORD + {{- if $.Values.global.dev }} + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-postgresql + key: postgres-password + optional: false + {{- else if $.Values.global.postgres.externalSecret }} + valueFrom: + secretKeyRef: + name: {{ $.Values.global.postgres.externalSecret }} + key: password + optional: false + {{- else }} + value: {{ .Values.global.postgres.master.password | quote }} + {{- end }} + - name: PGUSER + {{- if $.Values.global.postgres.externalSecret }} + valueFrom: + secretKeyRef: + name: {{ $.Values.global.postgres.externalSecret }} + key: username + optional: false + {{- else }} + value: {{ .Values.global.postgres.master.username | quote }} + {{- end }} + - name: PGPORT + {{- if $.Values.global.postgres.externalSecret }} + valueFrom: + secretKeyRef: + name: {{ $.Values.global.postgres.externalSecret }} + key: port + optional: false + {{- else }} + value: {{ .Values.global.postgres.master.port | quote }} + {{- end }} + - name: PGHOST + {{- if $.Values.global.dev }} + value: "{{ .Release.Name }}-postgresql" + {{- else if $.Values.global.postgres.externalSecret }} + valueFrom: + secretKeyRef: + name: {{ $.Values.global.postgres.externalSecret }} + key: host + optional: false + {{- else }} + value: {{ .Values.global.postgres.master.host | quote }} + {{- end }} + + - name: PGDATABASE + valueFrom: + secretKeyRef: + name: gen3-embeddings-dbcreds + key: database + optional: false + - name: DBREADY + valueFrom: + secretKeyRef: + name: gen3-embeddings-dbcreds + key: dbcreated + optional: false + + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + command: ["/bin/bash", "-c"] + args: + - | + set -euo pipefail + echo "Running gen3-embeddings migrations..." + DATABASE_URL="postgresql://${PGUSER}:${PGPASSWORD}@${PGHOST}:${PGPORT}/${PGDATABASE}?sslmode=disable" + dbmate -u "$DATABASE_URL" -d "/services/gen3_embeddings/db/migrations" migrate + echo "Migrations completed." \ No newline at end of file diff --git a/helm/gen3-embeddings/templates/external-secret.yaml b/helm/gen3-embeddings/templates/external-secret.yaml new file mode 100644 index 000000000..e77eaea6e --- /dev/null +++ b/helm/gen3-embeddings/templates/external-secret.yaml @@ -0,0 +1,34 @@ +{{ if .Values.global.externalSecrets.deploy }} +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: gen3embeddings-g3auto +spec: + refreshInterval: 5m + secretStoreRef: + name: {{include "common.SecretStore" .}} + kind: SecretStore + target: + name: gen3embeddings-g3auto + creationPolicy: Owner + data: + - secretKey: base64Authz.txt + remoteRef: + #name of secret in secrets manager + key: {{include "gen3embeddings-g3auto" .}} + property: base64Authz.txt + - secretKey: dbcreds.json + remoteRef: + #name of secret in secrets manager + key: {{include "gen3embeddings-g3auto" .}} + property: dbcreds.json + - secretKey: gen3-embeddings.env + remoteRef: + #name of secret in secrets manager + key: {{include "gen3embeddings-g3auto" .}} + property: gen3-embeddings.env +{{- end }} +--- +{{- if and .Values.global.externalSecrets.deploy (not .Values.global.externalSecrets.createLocalK8sSecret) }} +{{ include "common.externalSecret.db" . }} +{{- end }} \ No newline at end of file diff --git a/helm/gen3-embeddings/templates/hpa.yaml b/helm/gen3-embeddings/templates/hpa.yaml new file mode 100644 index 000000000..c3dee2ad8 --- /dev/null +++ b/helm/gen3-embeddings/templates/hpa.yaml @@ -0,0 +1,3 @@ +{{- if default .Values.global.autoscaling.enabled .Values.autoscaling.enabled }} +{{ include "common.hpa" . }} +{{- end }} \ No newline at end of file diff --git a/helm/gen3-embeddings/templates/pdb.yaml b/helm/gen3-embeddings/templates/pdb.yaml new file mode 100644 index 000000000..2ef2de13d --- /dev/null +++ b/helm/gen3-embeddings/templates/pdb.yaml @@ -0,0 +1,3 @@ +{{- if and .Values.global.pdb (gt (int .Values.replicaCount) 1) }} +{{ include "common.pod_disruption_budget" . }} +{{- end }} \ No newline at end of file diff --git a/helm/gen3-embeddings/templates/secret-store.yaml b/helm/gen3-embeddings/templates/secret-store.yaml new file mode 100644 index 000000000..771c7760d --- /dev/null +++ b/helm/gen3-embeddings/templates/secret-store.yaml @@ -0,0 +1,3 @@ +{{ if .Values.global.externalSecrets.separateSecretStore }} +{{ include "common.secretstore" . }} +{{- end }} \ No newline at end of file diff --git a/helm/gen3-embeddings/templates/secrets.yaml b/helm/gen3-embeddings/templates/secrets.yaml new file mode 100644 index 000000000..1c2245ae5 --- /dev/null +++ b/helm/gen3-embeddings/templates/secrets.yaml @@ -0,0 +1,17 @@ +{{- if or (not .Values.global.externalSecrets.deploy) (and .Values.global.externalSecrets.deploy .Values.externalSecrets.createK8sGen3EmbeddingsSecret) }} +apiVersion: v1 +kind: Secret +metadata: + name: gen3embeddings-g3auto +stringData: + {{- $randomPass := printf "%s%s" "gateway:" (randAlphaNum 32) }} + base64Authz.txt: {{ $randomPass | quote | b64enc }} + gen3-embeddings.env: | + DEBUG={{ .Values.debug}} + DB_HOST={{ .Values.postgres.host }} + DB_USER={{ .Values.postgres.username }} + GUNICORN_WORKERS={{ .Values.gunicornWorkers}} + DB_PASSWORD={{ include "gen3-embeddings.postgres.password" . }} + DB_DATABASE={{ .Values.postgres.dbname }} + ADMIN_LOGINS={{ $randomPass }} +{{- end }} \ No newline at end of file diff --git a/helm/gen3-embeddings/templates/service.yaml b/helm/gen3-embeddings/templates/service.yaml new file mode 100644 index 000000000..9d470fd62 --- /dev/null +++ b/helm/gen3-embeddings/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: "gen3-embeddings-service" + labels: + {{- include "gen3-embeddings.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: 4142 + protocol: TCP + name: http + selector: + {{- include "gen3-embeddings.selectorLabels" . | nindent 4 }} diff --git a/helm/gen3-embeddings/values.yaml b/helm/gen3-embeddings/values.yaml new file mode 100644 index 000000000..6a8d4fea4 --- /dev/null +++ b/helm/gen3-embeddings/values.yaml @@ -0,0 +1,206 @@ +# Default values for gen3-embeddings. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +global: + # -- (map) AWS configuration + aws: + # -- (bool) Set to true if deploying to AWS. Controls ingress annotations. + enabled: false + # -- (string) Credentials for AWS stuff. + awsAccessKeyId: + # -- (string) Credentials for AWS stuff. + awsSecretAccessKey: + externalSecrets: + # -- (bool) Whether to use External Secrets for aws config. + enabled: false + # -- (String) Name of Secrets Manager secret. + externalSecretAwsCreds: + # -- (bool) Whether the deployment is for development purposes. + dev: true + postgres: + # -- (bool) Whether the database should be created. + dbCreate: true + # -- (string) Name of external secret. Disabled if empty + externalSecret: "" + # -- (map) Master credentials to postgres. This is going to be the default postgres server being used for each service, unless each service specifies their own postgres + master: + # -- (string) hostname of postgres server + host: + # -- (string) username of superuser in postgres. This is used to create or restore databases + username: postgres + # -- (string) password for superuser in postgres. This is used to create or restore databases + password: + # -- (string) Port for Postgres. + port: "5432" + # -- (map) External Secrets settings. + externalSecrets: + # -- (bool) Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any metadata secrets you have deployed. + deploy: false + # -- (string) Will deploy a separate External Secret Store for this service. + separateSecretStore: false + # -- (map) This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/ + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 10 + averageCPUValue: 500m + averageMemoryValue: 500Mi + # -- (map) Karpenter topology spread configuration. + topologySpread: + # -- (bool) Whether to enable topology spread constraints for all subcharts that support it. + enabled: false + # -- (string) The topology key to use for spreading. Defaults to "topology.kubernetes.io/zone". + topologyKey: "topology.kubernetes.io/zone" + # -- (int) The maxSkew to use for topology spread constraints. Defaults to 1. + maxSkew: 1 + +# -- (map) This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/ +autoscaling: {} + +# This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ +replicaCount: 1 +debug: false + +# This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ +image: + repository: quay.io/cdis/gen3_embeddings + # This sets the pull policy for images. + pullPolicy: Always + # Overrides the image tag whose default is the chart appVersion. + tag: main + +# This is to override the chart name. +nameOverride: "" +fullnameOverride: "" + +# This is for setting up a service more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/ +service: + # This sets the service type more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: ClusterIP + # This sets the ports more information can be found here: https://kubernetes.io/docs/concepts/services-networking/service/#field-spec-ports + port: 80 + +# This block is for setting up the ingress for more information can be found here: https://kubernetes.io/docs/concepts/services-networking/ingress/ +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi +# -- (list) Environment variables to pass to the container +env: + - name: GEN3_DEBUG + value: "false" + - name: ARBORIST_URL + valueFrom: + configMapKeyRef: + name: manifest-global + key: arborist_url + optional: true + - name: PGPOOL_MIN_SIZE + value: "1" + - name: PGPOOL_MAX_SIZE + value: "5" + - name: GUNICORN_WORKERS + value: "2" +# This is to setup the liveness and readiness probes more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ +livenessProbe: + httpGet: + path: / + port: http +readinessProbe: + httpGet: + path: / + port: http + +# Additional volumeMounts on the output Deployment definition. +volumeMounts: + - mountPath: /services/gen3_embeddings/.env + name: gen3-embeddings-g3auto-volume + readOnly: true + subPath: gen3-embeddings.env + +affinity: {} +automountServiceAccountToken: false + +# -- (map) Postgres database configuration. If db does not exist in postgres cluster and dbCreate is set ot true then these databases will be created for you +postgres: + # (bool) Whether the database should be restored from s3. Default to global.postgres.dbRestore + dbRestore: false + # -- (bool) Whether the database should be created. Default to global.postgres.dbCreate + dbCreate: + # -- (string) Hostname for postgres server. This is a service override, defaults to global.postgres.host + host: + # -- (string) Database name for postgres. This is a service override, defaults to - + database: + # -- (string) Username for postgres. This is a service override, defaults to - + username: + # -- (string) Port for Postgres. + port: "5432" + # -- (string) Password for Postgres. Will be autogenerated if left empty. + password: + # -- (string) Will create a Database for the individual service to help with developing it. + separate: false +# -- (map) Postgresql subchart settings if deployed separately option is set to "true". +# Disable persistence by default so we can spin up and down ephemeral environments +postgresql: + primary: + persistence: + # -- (bool) Option to persist the dbs data. + enabled: false + +# Values to determine the labels that are used for the deployment, pod, etc. +# -- (string) Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". +release: "production" +# -- (string) Valid options are "true" or "false". If invalid option is set- the value will default to "false". +criticalService: "false" +# -- (string) Label to help organize pods and their use. Any value is valid, but use "_" or "-" to divide words. +partOf: "Embeddings" +# -- (map) Will completely override the selectorLabels defined in the common chart's _label_setup.tpl +selectorLabels: +# -- (map) Will completely override the commonLabels defined in the common chart's _label_setup.tpl +commonLabels: + +# -- (map) External Secrets settings. +externalSecrets: + # -- (string) Will create the Helm "gen3Embeddings-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. + createK8sGen3EmbeddingsSecret: false + # -- (string) Will override the name of the aws secrets manager secret. Default is "gen3UserDataLibrary-g3auto" + gen3EmbeddingsG3auto: + # -- (bool) Whether to create the database and Secrets Manager secrets via PushSecret. + pushSecret: false + # -- (string) Will override the name of the aws secrets manager secret. Default is "Values.global.environment-.Chart.Name-creds" + dbcreds: + +# -- (bool) Whether Metrics are enabled. +metricsEnabled: + +# -- (map) Secret information to access the db restore job S3 bucket. +secrets: + # -- (str) AWS access key ID. Overrides global key. + awsAccessKeyId: + # -- (str) AWS secret access key ID. Overrides global key. + awsSecretAccessKey: +gunicornWorkers: 1 diff --git a/helm/gen3-network-policies/Chart.yaml b/helm/gen3-network-policies/Chart.yaml index f998a269b..e822ba394 100644 --- a/helm/gen3-network-policies/Chart.yaml +++ b/helm/gen3-network-policies/Chart.yaml @@ -4,6 +4,6 @@ description: A Helm chart that holds network policies needed to run Gen3 type: application -version: 0.1.4 +version: 0.0.0 appVersion: "0.1.2" diff --git a/helm/gen3-network-policies/README.md b/helm/gen3-network-policies/README.md index 22498af89..a1cc65439 100644 --- a/helm/gen3-network-policies/README.md +++ b/helm/gen3-network-policies/README.md @@ -1,9 +1,13 @@ # gen3-network-policies -![Version: 0.1.4](https://img.shields.io/badge/Version-0.1.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.2](https://img.shields.io/badge/AppVersion-0.1.2-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.1.2](https://img.shields.io/badge/AppVersion-0.1.2-informational?style=flat-square) A Helm chart that holds network policies needed to run Gen3 +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Values | Key | Type | Default | Description | diff --git a/helm/gen3-user-data-library/Chart.yaml b/helm/gen3-user-data-library/Chart.yaml index f9ac80c11..6a8765a3d 100644 --- a/helm/gen3-user-data-library/Chart.yaml +++ b/helm/gen3-user-data-library/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.15 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -24,7 +24,7 @@ version: 0.1.15 appVersion: "main" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common - name: postgresql version: 11.9.13 diff --git a/helm/gen3-user-data-library/README.md b/helm/gen3-user-data-library/README.md index 81f7fa1af..77cce4b7f 100644 --- a/helm/gen3-user-data-library/README.md +++ b/helm/gen3-user-data-library/README.md @@ -1,15 +1,19 @@ # gen3-user-data-library -![Version: 0.1.15](https://img.shields.io/badge/Version-0.1.15-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: main](https://img.shields.io/badge/AppVersion-main-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: main](https://img.shields.io/badge/AppVersion-main-informational?style=flat-square) A Helm chart for Kubernetes +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | -| https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | +| Repository | Name | +|------------|------| +| file://../common | common | +| https://charts.bitnami.com/bitnami | postgresql | ## Values diff --git a/helm/gen3-user-data-library/templates/db-init.yaml b/helm/gen3-user-data-library/templates/db-init.yaml index 0393aa732..95297b084 100644 --- a/helm/gen3-user-data-library/templates/db-init.yaml +++ b/helm/gen3-user-data-library/templates/db-init.yaml @@ -6,4 +6,10 @@ {{ include "common.s3_pg_restore" . }} {{- else }} {{ include "common.db_setup_job" . }} -{{- end -}} \ No newline at end of file +{{- end -}} +{{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} +--- +{{ include "common.db-push-secret" . }} +--- +{{ include "common.secret.db.bootstrap" . }} +{{- end }} diff --git a/helm/gen3-workflow/Chart.yaml b/helm/gen3-workflow/Chart.yaml index 09a68d39b..e9dc652ee 100644 --- a/helm/gen3-workflow/Chart.yaml +++ b/helm/gen3-workflow/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.22 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -24,5 +24,5 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/gen3-workflow/README.md b/helm/gen3-workflow/README.md index 3c3b8e7e1..fff704990 100644 --- a/helm/gen3-workflow/README.md +++ b/helm/gen3-workflow/README.md @@ -1,19 +1,24 @@ # gen3-workflow -![Version: 0.1.22](https://img.shields.io/badge/Version-0.1.22-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for Kubernetes +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| +| GEN3_WORKFLOW_CONFIG | map | `{}` | Passed straight through to the service's own configuration file, so any setting gen3-workflow supports can be set here using its real ALL_UPPER name, without the chart needing a matching key. Rendered after the `gen3WorkflowConfig` values below, so setting a key that the chart already templates does override it, but leaves both lines in the rendered config file. See https://github.com/uc-cdis/gen3-workflow/blob/master/gen3workflow/config-default.yaml | | affinity | map | `{"nodeAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"preference":{"matchExpressions":[{"key":"karpenter.sh/capacity-type","operator":"In","values":["spot"]}]},"weight":100},{"preference":{"matchExpressions":[{"key":"eks.amazonaws.com/capacityType","operator":"In","values":["SPOT"]}]},"weight":99}]},"podAntiAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"podAffinityTerm":{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["gen3-workflow"]}]},"topologyKey":"kubernetes.io/hostname"},"weight":25}]}}` | Affinity to use for the deployment. | | affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution | map | `[{"preference":{"matchExpressions":[{"key":"karpenter.sh/capacity-type","operator":"In","values":["spot"]}]},"weight":100},{"preference":{"matchExpressions":[{"key":"eks.amazonaws.com/capacityType","operator":"In","values":["SPOT"]}]},"weight":99}]` | Option for scheduling to be required or preferred. | | affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[0] | int | `{"preference":{"matchExpressions":[{"key":"karpenter.sh/capacity-type","operator":"In","values":["spot"]}]},"weight":100}` | Weight value for preferred scheduling. | @@ -42,8 +47,10 @@ A Helm chart for Kubernetes | fullnameOverride | string | `""` | Override the full name of the chart, which is used as the name of resources created by the chart | | gen3WorkflowConfig.arboristUrl | string | `""` | Custom Arborist URL. Ignored if already set via environment variable. | | gen3WorkflowConfig.debug | bool | `false` | Enables debug mode for the application. | +| gen3WorkflowConfig.eksSecurityGroupNames | list | `[]` | Names of the EKS security groups that Karpenter attaches to an EKS worker node in the cluster (needed for S3Files) | | gen3WorkflowConfig.enableOptimizedNodeScheduling | bool | `true` | When enabled, jobs are configured to run on specific nodes through Kubernetes NodeSelector and Tolerations. Disable this if using a cluster that does not support nodepools. | | gen3WorkflowConfig.enablePrometheusMetrics | bool | `false` | Enables Prometheus metrics for the workflow service. | +| gen3WorkflowConfig.enableS3Files | bool | `false` | Set it to true to create S3Files resources (default - false) | | gen3WorkflowConfig.hostname | string | `""` | Override hostname where the workflow service runs. If empty, gen3-workflow falls back to values.global.hostname | | gen3WorkflowConfig.httpxDebug | bool | `false` | Enables verbose logging specifically for httpx requests. | | gen3WorkflowConfig.kmsEncryptionEnabled | bool | `true` | Enables KMS encryption for S3 uploads. | diff --git a/helm/gen3-workflow/templates/crossplane.yaml b/helm/gen3-workflow/templates/crossplane.yaml index 4edc9a438..98cc916b6 100644 --- a/helm/gen3-workflow/templates/crossplane.yaml +++ b/helm/gen3-workflow/templates/crossplane.yaml @@ -60,13 +60,20 @@ spec: "s3:GetObject", "s3:PutObject", "s3:DeleteObject", + "s3:ListBucketMultipartUploads", "s3:AbortMultipartUpload", "s3:DeleteBucketPolicy", "s3:GetEncryptionConfiguration", "s3:PutEncryptionConfiguration", "s3:GetBucketPolicy", "s3:PutBucketPolicy", - "s3:PutLifecycleConfiguration" + "s3:PutLifecycleConfiguration", + "s3:GetBucketVersioning", + "s3:PutBucketVersioning", + "s3:ListBucketVersions", + "s3:DeleteObjectVersion", + "s3:GetBucketNotification", + "s3:PutBucketNotification" ], "Resource": [ "arn:aws:s3:::gen3wf-*", @@ -119,6 +126,61 @@ spec: "kms:TagResource" ], "Resource": "*" + }, + { + "Sid": "AllowPassRoleToS3FilesService", + "Effect": "Allow", + "Action": "iam:PassRole", + "Resource": "arn:aws:iam::{{ .Values.global.crossplane.accountId }}:role/gen3wf-*-s3files-role", + "Condition": { + "StringEquals": { + "iam:PassedToService": "elasticfilesystem.amazonaws.com" + } + } + }, + { + "Sid": "S3FilesSystemAndMountManagement", + "Effect": "Allow", + "Action": [ + "s3files:CreateFileSystem", + "s3files:GetFileSystem", + "s3files:CreateMountTarget", + "s3files:ListFileSystems", + "s3files:TagResource", + "s3files:ListMountTargets" + ], + "Resource": "*" + }, + { + "Sid": "EC2NetworkDiscoveryAndSecurityControlForS3Files", + "Effect": "Allow", + "Action": [ + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeTags", + "ec2:CreateSecurityGroup", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:AuthorizeSecurityGroupEgress", + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeVpcs" + ], + "Resource": "*" + }, + { + "Sid": "AddEventBridgeRulesForS3Files", + "Effect": "Allow", + "Action": [ + "events:ListRules", + "events:PutRule", + "events:PutTargets", + "events:DescribeRule", + "events:ListTargetsByRule" + ], + "Resource": "arn:aws:events:*:{{ .Values.global.crossplane.accountId }}:rule/*" } ] } @@ -134,4 +196,4 @@ spec: roleName: "{{ .Values.global.environment }}-{{ .Release.Namespace }}-{{ include "gen3workflow.serviceAccountName" . }}" policyArnRef: name: "{{ .Values.global.environment }}-{{ .Release.Namespace }}-gen3-workflow-policy" -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm/gen3-workflow/templates/external-secret.yaml b/helm/gen3-workflow/templates/external-secret.yaml index 9c36e5f09..bbdefab82 100644 --- a/helm/gen3-workflow/templates/external-secret.yaml +++ b/helm/gen3-workflow/templates/external-secret.yaml @@ -1,6 +1,6 @@ {{ if .Values.global.externalSecrets.deploy }} {{- if not .Values.externalSecrets.createK8sGen3WorkflowSecret}} -apiVersion: external-secrets.io/v1beta1 +apiVersion: {{ include "common.externalSecrets.apiVersion" . }} kind: ExternalSecret metadata: name: gen3workflow-g3auto diff --git a/helm/gen3-workflow/templates/secrets.yaml b/helm/gen3-workflow/templates/secrets.yaml index dba07eb98..faea3c62c 100644 --- a/helm/gen3-workflow/templates/secrets.yaml +++ b/helm/gen3-workflow/templates/secrets.yaml @@ -28,6 +28,7 @@ stringData: S3_ENDPOINTS_AWS_ACCESS_KEY_ID: {{ .Values.gen3WorkflowConfig.s3AccessKeyId }} #NOTE: This is not used when using IRSA S3_ENDPOINTS_AWS_SECRET_ACCESS_KEY: {{ .Values.gen3WorkflowConfig.s3SecretAccessKey }} #NOTE: This is not used when using IRSA KMS_ENCRYPTION_ENABLED: {{ .Values.gen3WorkflowConfig.kmsEncryptionEnabled }} + ENABLE_S3_FILES: {{ .Values.gen3WorkflowConfig.enableS3Files }} ############# # GA4GH TES # @@ -53,4 +54,11 @@ stringData: EKS_CLUSTER_NAME: {{ .Values.global.clusterName }} EKS_CLUSTER_REGION: {{ .Values.global.aws.region }} {{- end }} + EKS_SECURITY_GROUP_NAMES: {{ .Values.gen3WorkflowConfig.eksSecurityGroupNames | toJson }} + {{- with .Values.GEN3_WORKFLOW_CONFIG }} + + # Any setting the service supports, under its real ALL_UPPER name. Rendered last, so a key + # repeated from above overrides it (at the cost of appearing twice in this file). + {{- toYaml . | nindent 6 }} + {{- end }} {{- end }} diff --git a/helm/gen3-workflow/values.yaml b/helm/gen3-workflow/values.yaml index 79fe81c6b..04f4e74f8 100644 --- a/helm/gen3-workflow/values.yaml +++ b/helm/gen3-workflow/values.yaml @@ -285,6 +285,14 @@ extraLabels: # for network policy netnolimit: "yes" +# -- (map) Passed straight through to the service's own configuration file, so any setting +# gen3-workflow supports can be set here using its real ALL_UPPER name, without the chart +# needing a matching key. Rendered after the `gen3WorkflowConfig` values below, so setting a key +# that the chart already templates does override it, but leaves both lines in the rendered +# config file. +# See https://github.com/uc-cdis/gen3-workflow/blob/master/gen3workflow/config-default.yaml +GEN3_WORKFLOW_CONFIG: {} + gen3WorkflowConfig: # -- (string) Override hostname where the workflow service runs. If empty, gen3-workflow falls back to values.global.hostname hostname: "" @@ -314,6 +322,10 @@ gen3WorkflowConfig: s3SecretAccessKey: "" # -- (bool) Enables KMS encryption for S3 uploads. kmsEncryptionEnabled: true + # -- (bool) Set it to true to create S3Files resources (default - false) + enableS3Files: false + # -- (list) Names of the EKS security groups that Karpenter attaches to an EKS worker node in the cluster (needed for S3Files) + eksSecurityGroupNames: [] # -- (string) TES server URL to which workflow tasks are forwarded. tesServerUrl: http://funnel:8000 # -- (list) Whitelist of container image patterns allowed for workflow tasks. diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml index c21a8f708..08c4b2da1 100644 --- a/helm/gen3/Chart.yaml +++ b/helm/gen3/Chart.yaml @@ -5,162 +5,170 @@ description: Helm chart to deploy Gen3 Data Commons # Dependencies dependencies: - name: access-backend - version: 0.1.20 + version: "*" repository: "file://../access-backend" condition: access-backend.enabled - name: ambassador - version: 0.1.37 + version: "*" repository: "file://../ambassador" condition: ambassador.enabled - name: arborist - version: 0.1.34 + version: "*" repository: "file://../arborist" condition: arborist.enabled - name: argo-wrapper - version: 0.1.30 + version: "*" repository: "file://../argo-wrapper" condition: argo-wrapper.enabled - name: audit - version: 0.1.42 + version: "*" repository: "file://../audit" condition: audit.enabled - name: aws-es-proxy - version: 0.1.41 + version: "*" repository: "file://../aws-es-proxy" condition: aws-es-proxy.enabled - name: aws-sigv4-proxy - version: 0.1.2 + version: "*" repository: "file://../aws-sigv4-proxy" condition: aws-sigv4-proxy.enabled - name: cedar - version: 0.1.26 + version: "*" repository: "file://../cedar" condition: cedar.enabled - name: cohort-middleware - version: 0.1.22 + version: "*" repository: "file://../cohort-middleware" condition: cohort-middleware.enabled - name: common - version: 0.1.36 + version: "*" repository: file://../common - name: dashboard - version: 0.1.20 + version: "*" repository: file://../dashboard condition: dashboard.enabled - name: datareplicate - version: 0.1.21 + version: "*" repository: "file://../datareplicate" condition: datareplicate.enabled - name: data-upload-cron - version: 0.1.6 + version: "*" repository: "file://../data-upload-cron" condition: data-upload-cron.enabled - name: embedding-management-service - version: 0.1.7 + version: "*" repository: file://../embedding-management-service condition: embedding-management-service.enabled - name: etl - version: 0.1.23 + version: "*" repository: file://../etl condition: etl.enabled - name: frontend-framework - version: 0.1.29 + version: "*" repository: "file://../frontend-framework" condition: frontend-framework.enabled - name: fence - version: 0.1.78 + version: "*" repository: "file://../fence" condition: fence.enabled - name: funnel - version: 0.1.26 + version: "*" repository: "file://../funnel" condition: gen3-workflow.enabled + - name: gen3-embeddings + version: "*" + repository: "file://../gen3-embeddings" + condition: gen3-embeddings.enabled - name: gen3-user-data-library - version: 0.1.15 + version: "*" repository: "file://../gen3-user-data-library" condition: gen3-user-data-library.enabled - name: gen3-workflow - version: 0.1.22 + version: "*" repository: "file://../gen3-workflow" condition: gen3-workflow.enabled - name: guppy - version: 0.1.37 + version: "*" repository: "file://../guppy" condition: guppy.enabled - name: hatchery - version: 0.1.69 + version: "*" repository: "file://../hatchery" condition: hatchery.enabled - name: indexd - version: 0.1.47 + version: "*" repository: "file://../indexd" condition: indexd.enabled - name: manifestservice - version: 0.1.42 + version: "*" repository: "file://../manifestservice" condition: manifestservice.enabled - name: metadata - version: 0.1.44 + version: "*" repository: "file://../metadata" condition: metadata.enabled - name: peregrine - version: 0.1.42 + version: "*" repository: "file://../peregrine" condition: peregrine.enabled - name: portal - version: 0.1.60 + version: "*" repository: "file://../portal" condition: portal.enabled - name: requestor - version: 0.1.34 + version: "*" repository: "file://../requestor" condition: requestor.enabled - name: revproxy - version: 0.1.61 + version: "*" repository: "file://../revproxy" condition: revproxy.enabled - name: sheepdog - version: 0.1.42 + version: "*" repository: "file://../sheepdog" condition: sheepdog.enabled - name: ssjdispatcher - version: 0.1.47 + version: "*" repository: "file://../ssjdispatcher" condition: ssjdispatcher.enabled - name: sower - version: 0.1.46 + version: "*" condition: sower.enabled repository: "file://../sower" - name: wts - version: 0.1.40 + version: "*" repository: "file://../wts" condition: wts.enabled + - name: zendesk-wrapper + version: "*" + repository: "file://../zendesk-wrapper" + condition: zendesk-wrapper.enabled - name: gen3-network-policies - version: 0.1.4 + version: "*" repository: "file://../gen3-network-policies" condition: global.netPolicy.enabled - name: dicom-server - version: 0.1.31 + version: "*" repository: file://../dicom-server condition: dicom-server.enabled - name: ohif-viewer - version: 0.1.14 + version: "*" repository: file://../ohif-viewer condition: ohif-viewer.enabled - name: orthanc - version: 0.1.15 + version: "*" repository: file://../orthanc condition: orthanc.enabled - name: gen3-analysis - version: 0.1.12 + version: "*" repository: file://../gen3-analysis condition: gen3-analysis.enabled - name: ohdsi-atlas - version: 0.1.2 + version: "*" repository: file://../ohdsi-atlas condition: ohdsi-atlas.enabled - name: ohdsi-webapi - version: 0.1.5 + version: "*" repository: file://../ohdsi-webapi condition: ohdsi-webapi.enabled @@ -179,9 +187,21 @@ dependencies: # Reference: https://github.com/neuvector/neuvector-helm # For more information, please use the Gen3 community Slack. - name: neuvector - version: "0.1.2" + version: "*" repository: "file://../neuvector" condition: neuvector.enabled + - name: jeg + version: "*" + repository: "file://../jeg" + condition: jeg.enabled + - name: workspace-proxy + version: "*" + repository: "file://../workspace-proxy" + condition: workspace-proxy.enabled + - name: vectis-overlays + version: "*" + repository: "file://../vectis-overlays" + condition: vectis-overlays.enabled # A chart can be either an 'application' or a 'library' chart. # @@ -197,7 +217,7 @@ type: application # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.3.59 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/gen3/README.md b/helm/gen3/README.md index b9a48b561..62073e924 100644 --- a/helm/gen3/README.md +++ b/helm/gen3/README.md @@ -1,9 +1,13 @@ # gen3 -![Version: 0.3.59](https://img.shields.io/badge/Version-0.3.59-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) Helm chart to deploy Gen3 Data Commons +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Maintainers | Name | Email | Url | @@ -16,51 +20,56 @@ Helm chart to deploy Gen3 Data Commons ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../access-backend | access-backend | 0.1.20 | -| file://../ambassador | ambassador | 0.1.37 | -| file://../arborist | arborist | 0.1.34 | -| file://../argo-wrapper | argo-wrapper | 0.1.30 | -| file://../audit | audit | 0.1.42 | -| file://../aws-es-proxy | aws-es-proxy | 0.1.41 | -| file://../aws-sigv4-proxy | aws-sigv4-proxy | 0.1.2 | -| file://../cedar | cedar | 0.1.26 | -| file://../cohort-middleware | cohort-middleware | 0.1.22 | -| file://../common | common | 0.1.36 | -| file://../dashboard | dashboard | 0.1.20 | -| file://../data-upload-cron | data-upload-cron | 0.1.6 | -| file://../datareplicate | datareplicate | 0.1.21 | -| file://../dicom-server | dicom-server | 0.1.31 | -| file://../embedding-management-service | embedding-management-service | 0.1.7 | -| file://../etl | etl | 0.1.23 | -| file://../fence | fence | 0.1.78 | -| file://../frontend-framework | frontend-framework | 0.1.29 | -| file://../funnel | funnel | 0.1.26 | -| file://../gen3-analysis | gen3-analysis | 0.1.12 | -| file://../gen3-network-policies | gen3-network-policies | 0.1.4 | -| file://../gen3-user-data-library | gen3-user-data-library | 0.1.15 | -| file://../gen3-workflow | gen3-workflow | 0.1.22 | -| file://../guppy | guppy | 0.1.37 | -| file://../hatchery | hatchery | 0.1.69 | -| file://../indexd | indexd | 0.1.47 | -| file://../manifestservice | manifestservice | 0.1.42 | -| file://../metadata | metadata | 0.1.44 | -| file://../neuvector | neuvector | 0.1.2 | -| file://../ohdsi-atlas | ohdsi-atlas | 0.1.2 | -| file://../ohdsi-webapi | ohdsi-webapi | 0.1.5 | -| file://../ohif-viewer | ohif-viewer | 0.1.14 | -| file://../orthanc | orthanc | 0.1.15 | -| file://../peregrine | peregrine | 0.1.42 | -| file://../portal | portal | 0.1.60 | -| file://../requestor | requestor | 0.1.34 | -| file://../revproxy | revproxy | 0.1.61 | -| file://../sheepdog | sheepdog | 0.1.42 | -| file://../sower | sower | 0.1.46 | -| file://../ssjdispatcher | ssjdispatcher | 0.1.47 | -| file://../wts | wts | 0.1.40 | -| https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | -| https://helm.elastic.co | elasticsearch | 7.10.2 | +| Repository | Name | +|------------|------| +| file://../access-backend | access-backend | +| file://../ambassador | ambassador | +| file://../arborist | arborist | +| file://../argo-wrapper | argo-wrapper | +| file://../audit | audit | +| file://../aws-es-proxy | aws-es-proxy | +| file://../aws-sigv4-proxy | aws-sigv4-proxy | +| file://../cedar | cedar | +| file://../cohort-middleware | cohort-middleware | +| file://../common | common | +| file://../dashboard | dashboard | +| file://../data-upload-cron | data-upload-cron | +| file://../datareplicate | datareplicate | +| file://../dicom-server | dicom-server | +| file://../embedding-management-service | embedding-management-service | +| file://../etl | etl | +| file://../fence | fence | +| file://../frontend-framework | frontend-framework | +| file://../funnel | funnel | +| file://../gen3-analysis | gen3-analysis | +| file://../gen3-embeddings | gen3-embeddings | +| file://../gen3-network-policies | gen3-network-policies | +| file://../gen3-user-data-library | gen3-user-data-library | +| file://../gen3-workflow | gen3-workflow | +| file://../guppy | guppy | +| file://../hatchery | hatchery | +| file://../indexd | indexd | +| file://../jeg | jeg | +| file://../manifestservice | manifestservice | +| file://../metadata | metadata | +| file://../neuvector | neuvector | +| file://../ohdsi-atlas | ohdsi-atlas | +| file://../ohdsi-webapi | ohdsi-webapi | +| file://../ohif-viewer | ohif-viewer | +| file://../orthanc | orthanc | +| file://../peregrine | peregrine | +| file://../portal | portal | +| file://../requestor | requestor | +| file://../revproxy | revproxy | +| file://../sheepdog | sheepdog | +| file://../sower | sower | +| file://../ssjdispatcher | ssjdispatcher | +| file://../vectis-overlays | vectis-overlays | +| file://../workspace-proxy | workspace-proxy | +| file://../wts | wts | +| file://../zendesk-wrapper | zendesk-wrapper | +| https://charts.bitnami.com/bitnami | postgresql | +| https://helm.elastic.co | elasticsearch | ## Values @@ -122,6 +131,7 @@ Helm chart to deploy Gen3 Data Commons | frontend-framework.image.tag | string | `"main"` | Overrides the image tag whose default is the chart appVersion. | | gen3-analysis | map | `{"enabled":false}` | Configurations for gen3-analysis chart. | | gen3-analysis.enabled | bool | `false` | Whether to deploy the gen3-analysis subchart. | +| gen3-embeddings | map | `{"enabled":false}` | Configurations for gen3-embeddings chart. | | gen3-user-data-library | map | `{"enabled":false}` | Configurations for gen3-user-data-library chart. | | gen3-user-data-library.enabled | bool | `false` | Whether to deploy the gen3-user-data-library subchart. | | gen3-workflow | map | `{"enabled":false}` | Configurations for gen3-workflow chart. | @@ -155,7 +165,8 @@ Helm chart to deploy Gen3 Data Commons | global.dictionaryUrl | string | `"https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json"` | URL of the data dictionary. | | global.dispatcherJobNum | int | `"10"` | Number of dispatcher jobs. | | global.environment | string | `"default"` | Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces in same cluster. | -| global.externalSecrets | map | `{"clusterSecretStoreRef":"","createLocalK8sSecret":false,"createSlackWebhookSecret":false,"deploy":false,"slackWebhookSecretName":""}` | External Secrets settings. | +| global.externalSecrets | map | `{"apiVersion":"external-secrets.io/v1","clusterSecretStoreRef":"","createLocalK8sSecret":false,"createSlackWebhookSecret":false,"deploy":false,"slackWebhookSecretName":""}` | External Secrets settings. | +| global.externalSecrets.apiVersion | string | `"external-secrets.io/v1"` | API version to use for External Secrets resources. Defaults to v1beta1 when unset. | | global.externalSecrets.createLocalK8sSecret | bool | `false` | Will create the databases and store the creds in Kubernetes Secrets even if externalSecrets is deployed. Useful if you want to use ExternalSecrets for other secrets besides db secrets. | | global.externalSecrets.createSlackWebhookSecret | bool | `false` | Will create a Kubernetes Secret for the slack webhook. | | global.externalSecrets.deploy | bool | `false` | Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override secrets you have deployed. | @@ -231,6 +242,8 @@ Helm chart to deploy Gen3 Data Commons | hatchery.hatchery.sidecarContainer.memory-limit | string | `"256Mi"` | The maximum amount of memory the sidecar container can use | | indexd.defaultPrefix | string | `"PREFIX/"` | the default prefix for indexd records | | indexd.enabled | bool | `true` | Whether to deploy the indexd subchart. | +| jeg | map | `{"enabled":false}` | Jupyter Enterprise Gateway for vectis workspaces. | +| jeg.enabled | bool | `false` | Whether to deploy the jeg subchart. | | manifestservice.enabled | bool | `true` | Whether to deploy the manifest service subchart. | | metadata.enabled | bool | `true` | Whether to deploy the metadata subchart. | | mutatingWebhook.enabled | bool | `false` | Whether to deploy the mutating webhook service. | @@ -250,7 +263,7 @@ Helm chart to deploy Gen3 Data Commons | peregrine.enabled | bool | `true` | Whether to deploy the peregrine subchart. | | pidgin.enabled | bool | `false` | Whether to deploy the pidgin subchart. | | portal.enabled | bool | `true` | Whether to deploy the portal subchart. | -| postgresql | map | `{"image":{"repository":"bitnamilegacy/postgresql","tag":"16.6.0-debian-12-r2"},"primary":{"persistence":{"enabled":false}}}` | To configure postgresql subchart Disable persistence by default so we can spin up and down ephemeral environments | +| postgresql | map | `{"image":{"registry":"quay.io","repository":"cdis/docker-bitnami-pgvector","tag":16},"primary":{"persistence":{"enabled":false}}}` | To configure postgresql subchart Disable persistence by default so we can spin up and down ephemeral environments | | postgresql.primary.persistence.enabled | bool | `false` | Option to persist the dbs data. | | requestor.enabled | bool | `false` | Whether to deploy the requestor subchart. | | revproxy.enabled | bool | `true` | Whether to deploy the revproxy subchart. | @@ -267,4 +280,9 @@ Helm chart to deploy Gen3 Data Commons | tests | map | `{"SERVICE_TO_TEST":null,"TEST_LABEL":null,"image":{"tag":"master"},"resources":{"limits":{"memory":"10G"},"requests":{"memory":"6G"}}}` | Environment variables that control which tests are run. | | tests.SERVICE_TO_TEST | str | `nil` | Name of the service we are testing. Default is empty as GH workflow automatically sets this. | | tests.TEST_LABEL | str | `nil` | Name of the test that will run. Default is empty as GH workflow automatically sets this. | +| vectis-overlays | map | `{"enabled":false}` | vectis-overlays — guppy-compat, siem-service, search-auth-proxy. | +| vectis-overlays.enabled | bool | `false` | Whether to deploy the vectis-overlays subchart. | +| workspace-proxy | map | `{"enabled":false}` | workspace-proxy — per-user workspace HTTP/WebSocket router. | +| workspace-proxy.enabled | bool | `false` | Whether to deploy the workspace-proxy subchart. | | wts.enabled | bool | `true` | Whether to deploy the wts subchart. | +| zendesk-wrapper.enabled | bool | `false` | Whether to deploy the zendesk-wrapper subchart. | diff --git a/helm/gen3/values.yaml b/helm/gen3/values.yaml index e8d0d1f73..f76c1e5d2 100644 --- a/helm/gen3/values.yaml +++ b/helm/gen3/values.yaml @@ -130,7 +130,7 @@ global: # -- (map) External Secrets settings. externalSecrets: # -- (string) API version to use for External Secrets resources. Defaults to v1beta1 when unset. - apiVersion: "external-secrets.io/v1beta1" + apiVersion: "external-secrets.io/v1" # -- (bool) Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override secrets you have deployed. deploy: false # -- (bool) Will create the databases and store the creds in Kubernetes Secrets even if externalSecrets is deployed. Useful if you want to use ExternalSecrets for other secrets besides db secrets. @@ -251,6 +251,10 @@ gen3-analysis: data-upload-cron: enabled: false +# -- (map) Configurations for gen3-embeddings chart. +gen3-embeddings: + enabled: false + # -- (map) Configurations for gen3-user-data-library chart. gen3-user-data-library: # -- (bool) Whether to deploy the gen3-user-data-library subchart. @@ -400,6 +404,10 @@ wts: # -- (bool) Whether to deploy the wts subchart. enabled: true +zendesk-wrapper: + # -- (bool) Whether to deploy the zendesk-wrapper subchart. + enabled: false + sower: # -- (bool) Whether to deploy the sower subchart. enabled: false @@ -420,8 +428,9 @@ access-backend: # Disable persistence by default so we can spin up and down ephemeral environments postgresql: image: - repository: bitnamilegacy/postgresql - tag: 16.6.0-debian-12-r2 + registry: quay.io + repository: "cdis/docker-bitnami-pgvector" + tag: 16 primary: persistence: # -- (bool) Option to persist the dbs data. @@ -474,6 +483,21 @@ neuvector: # hostname/service name for our ElasitcSearch instance, used to allow egress from containers ES_HOST: gen3-elasticsearch-master +# -- (map) Jupyter Enterprise Gateway for vectis workspaces. +jeg: + # -- (bool) Whether to deploy the jeg subchart. + enabled: false + +# -- (map) workspace-proxy — per-user workspace HTTP/WebSocket router. +workspace-proxy: + # -- (bool) Whether to deploy the workspace-proxy subchart. + enabled: false + +# -- (map) vectis-overlays — guppy-compat, siem-service, search-auth-proxy. +vectis-overlays: + # -- (bool) Whether to deploy the vectis-overlays subchart. + enabled: false + # -- (map) Secret information for External Secrets and DB Secrets. secrets: # -- (str) AWS access key ID. Overrides global key. diff --git a/helm/guppy/Chart.yaml b/helm/guppy/Chart.yaml index d12367a03..347eb6841 100644 --- a/helm/guppy/Chart.yaml +++ b/helm/guppy/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.37 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/guppy/README.md b/helm/guppy/README.md index c4e08b164..36a12c8af 100644 --- a/helm/guppy/README.md +++ b/helm/guppy/README.md @@ -1,14 +1,18 @@ # guppy -![Version: 0.1.37](https://img.shields.io/badge/Version-0.1.37-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 Guppy Service +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values @@ -88,6 +92,7 @@ A Helm chart for gen3 Guppy Service | secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null}` | Secret information to access the db restore job S3 bucket. | | secrets.awsAccessKeyId | str | `nil` | AWS access key ID. Overrides global key. | | secrets.awsSecretAccessKey | str | `nil` | AWS secret access key ID. Overrides global key. | +| securityContext | map | `{}` | Security context for the containers in the pod | | selectorLabels | map | `nil` | Will completely override the selectorLabels defined in the common chart's _label_setup.tpl | | service | map | `{"port":[{"name":"http","port":80,"protocol":"TCP","targetPort":8000}],"targetPort":8000,"type":"ClusterIP"}` | Kubernetes service information. | | service.port | int | `[{"name":"http","port":80,"protocol":"TCP","targetPort":8000}]` | The port number that the service exposes. | diff --git a/helm/guppy/templates/deployment.yaml b/helm/guppy/templates/deployment.yaml index 8e6f7c0a5..680454931 100644 --- a/helm/guppy/templates/deployment.yaml +++ b/helm/guppy/templates/deployment.yaml @@ -52,6 +52,8 @@ spec: {{- end }} containers: - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" livenessProbe: httpGet: diff --git a/helm/guppy/values.yaml b/helm/guppy/values.yaml index 763e93256..45ed2c3a2 100644 --- a/helm/guppy/values.yaml +++ b/helm/guppy/values.yaml @@ -178,6 +178,16 @@ resources: # -- (string) The maximum amount of memory the container can use memory: 2Gi +# -- (map) Security context for the containers in the pod +securityContext: + {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + # -- (map) Kubernetes service information. service: # -- (string) Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". diff --git a/helm/hatchery/Chart.yaml b/helm/hatchery/Chart.yaml index a555dade4..a4344400b 100644 --- a/helm/hatchery/Chart.yaml +++ b/helm/hatchery/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.69 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/hatchery/README.md b/helm/hatchery/README.md index b9c3f3306..e83cb74c5 100644 --- a/helm/hatchery/README.md +++ b/helm/hatchery/README.md @@ -1,14 +1,18 @@ # hatchery -![Version: 0.1.69](https://img.shields.io/badge/Version-0.1.69-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 Hatchery +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values @@ -115,6 +119,7 @@ A Helm chart for gen3 Hatchery | resources.limits.memory | string | `"512Mi"` | The maximum amount of memory the container can use | | resources.requests | map | `{"memory":"12Mi"}` | The amount of resources that the container requests | | resources.requests.memory | string | `"12Mi"` | The amount of memory requested | +| securityContext | map | `{}` | Security context for the containers in the pod | | selectorLabels | map | `nil` | Will completely override the selectorLabels defined in the common chart's _label_setup.tpl | | service | map | `{"port":80,"targetPort":8000,"type":"ClusterIP"}` | Kubernetes service information. | | service.port | int | `80` | The port number that the service exposes. | diff --git a/helm/hatchery/templates/deployment.yaml b/helm/hatchery/templates/deployment.yaml index 0e077bf40..742352cd1 100644 --- a/helm/hatchery/templates/deployment.yaml +++ b/helm/hatchery/templates/deployment.yaml @@ -33,6 +33,7 @@ spec: netnolimit: "yes" public: "yes" userhelper: "yes" + netvpc: "yes" {{- include "hatchery.selectorLabels" . | nindent 8 }} {{- include "common.extraLabels" . | nindent 8 }} spec: diff --git a/helm/hatchery/values.yaml b/helm/hatchery/values.yaml index f06e47659..ed2256acb 100644 --- a/helm/hatchery/values.yaml +++ b/helm/hatchery/values.yaml @@ -142,6 +142,16 @@ nameOverride: "" # -- (string) Override the full name of the deployment. fullnameOverride: "" +# -- (map) Security context for the containers in the pod +securityContext: + {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + # -- (map) Kubernetes service information. service: # -- (string) Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". diff --git a/helm/indexd/Chart.yaml b/helm/indexd/Chart.yaml index 8645d8899..4de36fdf7 100644 --- a/helm/indexd/Chart.yaml +++ b/helm/indexd/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.47 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,7 +25,7 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common - name: postgresql version: 11.9.13 diff --git a/helm/indexd/README.md b/helm/indexd/README.md index d6fedcd3a..1ab7ac1c0 100644 --- a/helm/indexd/README.md +++ b/helm/indexd/README.md @@ -1,15 +1,19 @@ # indexd -![Version: 0.1.47](https://img.shields.io/badge/Version-0.1.47-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 indexd +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | -| https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | +| Repository | Name | +|------------|------| +| file://../common | common | +| https://charts.bitnami.com/bitnami | postgresql | ## Values @@ -114,9 +118,10 @@ A Helm chart for gen3 indexd | serviceAccount.create | bool | `false` | Specifies whether a service account should be created. | | serviceAccount.name | string | `""` | The name of the service account | | tolerations | list | `[]` | Tolerations for the pods | -| trustedIssuers | map | `{"defaultBearerIssuer":"","defaultPassportIssuer":""}` | Maps an arborist resource to the issuers for passports and bearer tokens. | +| trustedIssuers | map | `{"defaultBearerIssuer":"","defaultPassportIssuer":"","defaultPreferredType":"BearerAuth"}` | Maps an arborist resource to the issuers for passports and bearer tokens. | | trustedIssuers.defaultBearerIssuer | string | `""` | The default issuer for bearer tokens to be used if a given auth resource isn't provided. Defaults to the fence token issuer | | trustedIssuers.defaultPassportIssuer | string | `""` | The default issuer for passports to be used if a given auth resource isn't provided. | +| trustedIssuers.defaultPreferredType | string | `"BearerAuth"` | The default preferred supported_type. This will ensure indexd return this supported_type before the other. | | useSingleTable | string | `"False"` | | | uwsgi | map | `{"listen":1024}` | Values for overriding uwsgi settings | | volumeMounts | list | `[{"mountPath":"/etc/uwsgi/uwsgi.ini","name":"uwsgi-config","subPath":"uwsgi.ini"},{"mountPath":"/var/www/indexd/local_settings.py","name":"config-volume","readOnly":true,"subPath":"local_settings.py"},{"mountPath":"/indexd/deployment/wsgi/gunicorn.conf.py","name":"gunicorn-conf","readOnly":true,"subPath":"gunicorn.conf.py"}]` | Volumes to mount to the container. | diff --git a/helm/indexd/indexd-settings/local_settings.py b/helm/indexd/indexd-settings/local_settings.py index b87b72414..766684fca 100644 --- a/helm/indexd/indexd-settings/local_settings.py +++ b/helm/indexd/indexd-settings/local_settings.py @@ -106,4 +106,9 @@ if default_passport_issuer: CONFIG["DEFAULT_PASSPORT_ISSUER"] = default_passport_issuer +default_preferred_type = environ.get("DEFAULT_PREFERRED_TYPE", None) + +if default_preferred_type: + CONFIG["DEFAULT_PREFERRED_TYPE"] = default_preferred_type + settings = {"config": CONFIG, "auth": AUTH} diff --git a/helm/indexd/templates/deployment.yaml b/helm/indexd/templates/deployment.yaml index a9dcfb0c8..ab999c487 100644 --- a/helm/indexd/templates/deployment.yaml +++ b/helm/indexd/templates/deployment.yaml @@ -94,6 +94,8 @@ spec: value: {{ default (printf "https://%s/user" .Values.global.hostname) .Values.trustedIssuers.defaultBearerIssuer | quote }} - name: DEFAULT_PASSPORT_ISSUER value: {{ .Values.trustedIssuers.defaultPassportIssuer | quote }} + - name: DEFAULT_PREFERRED_TYPE + value: {{ .Values.trustedIssuers.defaultPreferredType | quote }} - name: CLOUD_PROVIDER_MAP value: {{ .Values.cloudProviderMap | toJson | quote }} {{- toYaml .Values.env | nindent 12 }} diff --git a/helm/indexd/values.yaml b/helm/indexd/values.yaml index b916f2417..f5ce02485 100644 --- a/helm/indexd/values.yaml +++ b/helm/indexd/values.yaml @@ -297,12 +297,15 @@ cloudProviderMap: # -- (map) Maps an arborist resource to the issuers for passports and bearer tokens. trustedIssuers: + # -- (string) The default preferred supported_type. This will ensure indexd return this supported_type before the other. + defaultPreferredType: "BearerAuth" # -- (string) The default issuer for bearer tokens to be used if a given auth resource isn't provided. Defaults to the fence token issuer defaultBearerIssuer: "" # -- (string) The default issuer for passports to be used if a given auth resource isn't provided. defaultPassportIssuer: "" # Example: # "/programs/parent": + # preferred_type: "PassportAuth" # passport_auth_issuers: # - "https://stsstg.nih.gov" # bearer_auth_issuers: diff --git a/helm/jeg/Chart.yaml b/helm/jeg/Chart.yaml new file mode 100644 index 000000000..cfb94ab22 --- /dev/null +++ b/helm/jeg/Chart.yaml @@ -0,0 +1,14 @@ +apiVersion: v2 +name: jeg +description: > + Jupyter Enterprise Gateway for gen3 vectis workspaces. + Launches ephemeral kernel pods in the workspace namespace on behalf of + user Jupyter sessions proxied through workspace-proxy. +type: application +version: 0.0.0 +appVersion: "3.2.3" + +dependencies: + - name: common + version: "*" + repository: file://../common diff --git a/helm/jeg/README.md b/helm/jeg/README.md new file mode 100644 index 000000000..05b95f0b1 --- /dev/null +++ b/helm/jeg/README.md @@ -0,0 +1,41 @@ +# jeg + +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.2.3](https://img.shields.io/badge/AppVersion-3.2.3-informational?style=flat-square) + +Jupyter Enterprise Gateway for gen3 vectis workspaces. Launches ephemeral kernel pods in the workspace namespace on behalf of user Jupyter sessions proxied through workspace-proxy. + +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + +## Requirements + +| Repository | Name | +|------------|------| +| file://../common | common | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| env.EG_AUTH_TOKEN | string | `""` | | +| env.EG_CULL_CONNECTED | string | `"True"` | | +| env.EG_CULL_IDLE_TIMEOUT | string | `"14400"` | | +| env.EG_DEFAULT_KERNEL_NAME | string | `"python3"` | | +| env.EG_KERNEL_IMAGE_PULL_POLICY | string | `"IfNotPresent"` | | +| env.EG_KERNEL_LAUNCH_TIMEOUT | string | `"120"` | | +| env.EG_KERNEL_WHITELIST_ENVS | string | `"ACCESS_TOKEN"` | | +| env.EG_LIST_KERNELS | string | `"True"` | | +| env.EG_MAX_KERNELS_PER_USER | string | `"2"` | | +| env.EG_MIRROR_WORKING_DIRS | string | `"False"` | | +| env.EG_NAMESPACE | string | `"jupyter-pods"` | | +| env.EG_SHARED_NAMESPACE | string | `"False"` | | +| image.pullPolicy | string | `"Always"` | | +| image.repository | string | `"quay.io/cdis/gen3-vectis"` | | +| image.tag | string | `"qa-jeg"` | | +| replicaCount | int | `1` | | +| resources.limits.cpu | string | `"500m"` | | +| resources.limits.memory | string | `"512Mi"` | | +| resources.requests.cpu | string | `"100m"` | | +| resources.requests.memory | string | `"256Mi"` | | +| workspaceNamespace | string | `"jupyter-pods"` | | diff --git a/helm/jeg/templates/deployment.yaml b/helm/jeg/templates/deployment.yaml new file mode 100644 index 000000000..903c59920 --- /dev/null +++ b/helm/jeg/templates/deployment.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: jupyter-enterprise-gateway + namespace: {{ .Values.workspaceNamespace }} + labels: + app: enterprise-gateway +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: enterprise-gateway + template: + metadata: + labels: + app: enterprise-gateway + spec: + serviceAccountName: enterprise-gateway + automountServiceAccountToken: true + containers: + - name: enterprise-gateway + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 8888 + protocol: TCP + env: + {{- range $key, $val := .Values.env }} + - name: {{ $key }} + value: {{ $val | quote }} + {{- end }} + livenessProbe: + httpGet: + path: /api + port: http + initialDelaySeconds: 15 + periodSeconds: 30 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /api + port: http + initialDelaySeconds: 10 + periodSeconds: 15 + timeoutSeconds: 5 + resources: + {{- toYaml .Values.resources | nindent 12 }} + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault diff --git a/helm/jeg/templates/rbac.yaml b/helm/jeg/templates/rbac.yaml new file mode 100644 index 000000000..c620fcf80 --- /dev/null +++ b/helm/jeg/templates/rbac.yaml @@ -0,0 +1,35 @@ +# JEG needs permission to create/delete kernel pods and services in the +# workspace namespace. Scoped to a Role (not ClusterRole) for least-privilege. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: enterprise-gateway + namespace: {{ .Values.workspaceNamespace }} + labels: + app: enterprise-gateway +rules: + - apiGroups: [""] + resources: ["pods", "pods/log", "services", "configmaps", "secrets"] + verbs: ["get", "list", "watch", "create", "delete", "patch", "update"] + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "create", "delete", "patch", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: enterprise-gateway + namespace: {{ .Values.workspaceNamespace }} + labels: + app: enterprise-gateway +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: enterprise-gateway +subjects: + - kind: ServiceAccount + name: enterprise-gateway + namespace: {{ .Values.workspaceNamespace }} diff --git a/helm/jeg/templates/service.yaml b/helm/jeg/templates/service.yaml new file mode 100644 index 000000000..1c0179636 --- /dev/null +++ b/helm/jeg/templates/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: jupyter-enterprise-gateway + namespace: {{ .Values.workspaceNamespace }} + labels: + app: enterprise-gateway +spec: + type: ClusterIP + selector: + app: enterprise-gateway + ports: + - name: http + port: 8888 + targetPort: http + protocol: TCP diff --git a/helm/jeg/templates/serviceaccount.yaml b/helm/jeg/templates/serviceaccount.yaml new file mode 100644 index 000000000..793d0b228 --- /dev/null +++ b/helm/jeg/templates/serviceaccount.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: enterprise-gateway + namespace: {{ .Values.workspaceNamespace }} + labels: + app: enterprise-gateway diff --git a/helm/jeg/values.yaml b/helm/jeg/values.yaml new file mode 100644 index 000000000..343c7980e --- /dev/null +++ b/helm/jeg/values.yaml @@ -0,0 +1,35 @@ +# Jupyter Enterprise Gateway default values. +# Override via your environment's jeg-values.yaml + +# Namespace where JEG pod runs AND where it launches kernel pods. +# CRITICAL: must match hatchery user-namespace and workspace-proxy workspaceNamespace. +workspaceNamespace: jupyter-pods + +replicaCount: 1 + +image: + repository: quay.io/cdis/gen3-vectis + tag: qa-jeg + pullPolicy: Always + +env: + EG_NAMESPACE: jupyter-pods + EG_KERNEL_LAUNCH_TIMEOUT: "120" + EG_MAX_KERNELS_PER_USER: "2" + EG_CULL_IDLE_TIMEOUT: "14400" + EG_CULL_CONNECTED: "True" + EG_AUTH_TOKEN: "" + EG_DEFAULT_KERNEL_NAME: python3 + EG_LIST_KERNELS: "True" + EG_KERNEL_IMAGE_PULL_POLICY: IfNotPresent + EG_SHARED_NAMESPACE: "False" + EG_MIRROR_WORKING_DIRS: "False" + EG_KERNEL_WHITELIST_ENVS: "ACCESS_TOKEN" # propagate workspace token into kernel pods + +resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "500m" + memory: "512Mi" diff --git a/helm/manifestservice/Chart.yaml b/helm/manifestservice/Chart.yaml index b14ea2619..8e4ea6a0e 100644 --- a/helm/manifestservice/Chart.yaml +++ b/helm/manifestservice/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.42 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/manifestservice/README.md b/helm/manifestservice/README.md index e761bd077..93cd2a86b 100644 --- a/helm/manifestservice/README.md +++ b/helm/manifestservice/README.md @@ -1,14 +1,18 @@ # manifestservice -![Version: 0.1.42](https://img.shields.io/badge/Version-0.1.42-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for Kubernetes +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values @@ -54,6 +58,7 @@ A Helm chart for Kubernetes | global.externalSecrets | map | `{"deploy":false,"separateSecretStore":false}` | External Secrets settings. | | global.externalSecrets.deploy | bool | `false` | Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any manifestservice secrets you have deployed. | | global.externalSecrets.separateSecretStore | string | `false` | Will deploy a separate External Secret Store for this service. | +| global.hostname | string | `""` | Hostname for the deployment. | | global.minAvailable | int | `1` | The minimum amount of pods that are available at all times if the PDB is deployed. | | global.pdb | bool | `false` | If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. | | global.topologySpread | map | `{"enabled":false,"maxSkew":1,"topologyKey":"topology.kubernetes.io/zone"}` | Karpenter topology spread configuration. | @@ -82,6 +87,7 @@ A Helm chart for Kubernetes | secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null}` | Secret information for External Secrets. | | secrets.awsAccessKeyId | str | `nil` | AWS access key ID. Overrides global key. | | secrets.awsSecretAccessKey | str | `nil` | AWS secret access key ID. Overrides global key. | +| securityContext | map | `{}` | Security context for the containers in the pod | | selectorLabels | map | `nil` | Will completely override the selectorLabels defined in the common chart's _label_setup.tpl | | service | map | `{"port":80,"targetPort":80,"type":"ClusterIP"}` | Kubernetes service information. | | service.port | int | `80` | The port number that the service exposes. | diff --git a/helm/manifestservice/templates/deployment.yaml b/helm/manifestservice/templates/deployment.yaml index 1b0de1072..6fd27f563 100644 --- a/helm/manifestservice/templates/deployment.yaml +++ b/helm/manifestservice/templates/deployment.yaml @@ -52,6 +52,8 @@ spec: - name: manifestservice image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} env: {{- toYaml .Values.env | nindent 12 }} {{- if and .Values.manifestserviceG3auto.awsaccesskey .Values.manifestserviceG3auto.awssecretkey }} @@ -85,6 +87,8 @@ spec: {{- end }} key: secret-access-key {{- end }} + - name: FENCE_URL + value: {{ default "http://fence-service" (printf "https://%s/user" .Values.global.hostname) }} volumeMounts: {{- toYaml .Values.volumeMounts | nindent 12 }} resources: diff --git a/helm/manifestservice/values.yaml b/helm/manifestservice/values.yaml index 721082f6d..5375e29ee 100644 --- a/helm/manifestservice/values.yaml +++ b/helm/manifestservice/values.yaml @@ -26,6 +26,8 @@ global: localSecretName: # -- (string) Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. environment: default + # -- (string) Hostname for the deployment. + hostname: "" # -- (bool) If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. pdb: false # -- (int) The minimum amount of pods that are available at all times if the PDB is deployed. @@ -101,6 +103,16 @@ image: # -- (string) Overrides the image tag whose default is the chart appVersion. tag: "" +# -- (map) Security context for the containers in the pod +securityContext: + {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + # -- (map) Kubernetes service information. service: # -- (string) Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". diff --git a/helm/metadata/Chart.yaml b/helm/metadata/Chart.yaml index 92caa5d85..76710fee8 100644 --- a/helm/metadata/Chart.yaml +++ b/helm/metadata/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.44 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,7 +25,7 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common - name: postgresql version: 11.9.13 diff --git a/helm/metadata/README.md b/helm/metadata/README.md index c78de6546..2e2337929 100644 --- a/helm/metadata/README.md +++ b/helm/metadata/README.md @@ -1,16 +1,20 @@ # metadata -![Version: 0.1.44](https://img.shields.io/badge/Version-0.1.44-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 Metadata Service +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | -| https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | -| https://helm.elastic.co | elasticsearch | 7.17.1 | +| Repository | Name | +|------------|------| +| file://../common | common | +| https://charts.bitnami.com/bitnami | postgresql | +| https://helm.elastic.co | elasticsearch | ## Values @@ -117,6 +121,7 @@ A Helm chart for gen3 Metadata Service | secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null}` | Secret information to access the db restore job S3 bucket. | | secrets.awsAccessKeyId | str | `nil` | AWS access key ID. Overrides global key. | | secrets.awsSecretAccessKey | str | `nil` | AWS secret access key ID. Overrides global key. | +| securityContext | map | `{}` | Security context for the containers in the pod | | selectorLabels | map | `nil` | Will completely override the selectorLabels defined in the common chart's _label_setup.tpl | | service | map | `{"port":[{"name":"http","port":80,"protocol":"TCP","targetPort":80}],"targetPort":80,"type":"ClusterIP"}` | Kubernetes service information. | | service.port | int | `[{"name":"http","port":80,"protocol":"TCP","targetPort":80}]` | The port number that the service exposes. | diff --git a/helm/metadata/templates/deployment.yaml b/helm/metadata/templates/deployment.yaml index 3968899c8..94eb88809 100644 --- a/helm/metadata/templates/deployment.yaml +++ b/helm/metadata/templates/deployment.yaml @@ -56,6 +56,8 @@ spec: optional: true containers: - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" env: - name: GEN3_DEBUG @@ -127,6 +129,8 @@ spec: {{- end }} initContainers: - name: {{ .Values.initContainerName }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} {{- with .Values.initVolumeMounts }} diff --git a/helm/metadata/values.yaml b/helm/metadata/values.yaml index 9a7159d9a..1f13b5382 100644 --- a/helm/metadata/values.yaml +++ b/helm/metadata/values.yaml @@ -329,6 +329,16 @@ serviceAnnotations: prefix: /index/ service: http://metadata-service:80 +# -- (map) Security context for the containers in the pod +securityContext: + {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + # -- (map) Kubernetes service information. service: # -- (string) Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". diff --git a/helm/neuvector/Chart.yaml b/helm/neuvector/Chart.yaml index 845c17972..66d966074 100644 --- a/helm/neuvector/Chart.yaml +++ b/helm/neuvector/Chart.yaml @@ -19,7 +19,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.2 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/neuvector/README.md b/helm/neuvector/README.md index 928cd747d..4de9b4582 100644 --- a/helm/neuvector/README.md +++ b/helm/neuvector/README.md @@ -1,9 +1,13 @@ # neuvector -![Version: 0.1.2](https://img.shields.io/badge/Version-0.1.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.16.0](https://img.shields.io/badge/AppVersion-1.16.0-informational?style=flat-square) NeuVector Kubernetes Security Policy templates to protect Gen3 +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Values | Key | Type | Default | Description | diff --git a/helm/observability/Chart.yaml b/helm/observability/Chart.yaml index 0d0317ef0..fca128e29 100644 --- a/helm/observability/Chart.yaml +++ b/helm/observability/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.3 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/observability/README.md b/helm/observability/README.md index 72b9951d2..7ada6ba59 100644 --- a/helm/observability/README.md +++ b/helm/observability/README.md @@ -1,14 +1,18 @@ # lgtma-chart -![Version: 0.1.3](https://img.shields.io/badge/Version-0.1.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.0.0](https://img.shields.io/badge/AppVersion-1.0.0-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.0.0](https://img.shields.io/badge/AppVersion-1.0.0-informational?style=flat-square) A Helm chart for deploying the LGTM stack with additional resources +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| https://grafana.github.io/helm-charts | lgtm(lgtm-distributed) | 2.1.0 | +| Repository | Name | +|------------|------| +| https://grafana.github.io/helm-charts | lgtm-distributed | ## Values diff --git a/helm/observability/SETUP.md b/helm/observability/SETUP.md index d9c849778..6c9c942bc 100644 --- a/helm/observability/SETUP.md +++ b/helm/observability/SETUP.md @@ -2,7 +2,7 @@ ## Overview -The Observability Helm chart provides an all-in-one solution for deploying Mimir, Loki, and Grafana to your Kubernetes cluster, enabling a complete observability stack for metrics, logs, and visualization. +The Observability Helm chart provides an all-in-one solution for deploying Mimir, Loki, and Grafana to your Kubernetes cluster, enabling a complete observability stack for metrics, logs, and visualization. Traces and profiles are supported by the wider Gen3 observability setup, but their backends live outside this chart: Tempo is disabled here (`lgtm.tempo.enabled: false`), and Pyroscope is not part of the underlying `lgtm-distributed` chart at all. ### Grafana: A leading open-source platform for data visualization and monitoring. Grafana allows you to create rich, interactive dashboards from a variety of data sources, making it easy to analyze metrics and logs from your systems. @@ -15,6 +15,11 @@ Grafana Loki is a log aggregation system designed to efficiently collect, store, By deploying this Helm chart, you'll set up these three components together, allowing you to monitor your systems and applications comprehensively with metrics from Mimir, logs from Loki, and dashboards and alerts in Grafana. +### Pyroscope: +Grafana Pyroscope stores continuous profiling data - the CPU and memory profiles pushed by services that ship a Pyroscope SDK - and Grafana renders them as flame graphs next to the metrics from Mimir and the logs from Loki. + +***Note: Pyroscope is not a component of the `lgtm-distributed` chart and is therefore not deployed by this chart. Services push profiles directly to the Pyroscope ingest endpoint they are given in `PYROSCOPE_SERVER_ADDRESS`, rather than through Alloy, so a cluster that wants profiles needs its own Pyroscope release ([`grafana/pyroscope`](https://github.com/grafana/pyroscope/tree/main/operations/pyroscope/helm/pyroscope)) or an external one, plus a Pyroscope datasource in Grafana. For local development, [docs/local-observability.md](../../docs/local-observability.md) runs one inside the single-pod LGTM image. + ### Alloy: Grafana Alloy is a powerful observability tool that collects and ships logs and metrics from your services to Grafana Loki and Mimir for storage and analysis. diff --git a/helm/ohdsi-atlas/Chart.yaml b/helm/ohdsi-atlas/Chart.yaml index 94f3694a0..83a01321a 100644 --- a/helm/ohdsi-atlas/Chart.yaml +++ b/helm/ohdsi-atlas/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.2 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/ohdsi-atlas/README.md b/helm/ohdsi-atlas/README.md index f74fbf621..1558a2f4c 100644 --- a/helm/ohdsi-atlas/README.md +++ b/helm/ohdsi-atlas/README.md @@ -1,9 +1,13 @@ # ohdsi-atlas -![Version: 0.1.2](https://img.shields.io/badge/Version-0.1.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.15.0](https://img.shields.io/badge/AppVersion-2.15.0-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.15.0](https://img.shields.io/badge/AppVersion-2.15.0-informational?style=flat-square) A Helm chart for OHDSI Atlas +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Values | Key | Type | Default | Description | diff --git a/helm/ohdsi-webapi/Chart.yaml b/helm/ohdsi-webapi/Chart.yaml index 1f809e8fe..bff9fa4d3 100644 --- a/helm/ohdsi-webapi/Chart.yaml +++ b/helm/ohdsi-webapi/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.5 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,7 +25,7 @@ appVersion: "2.15.0" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common - name: postgresql version: 11.9.13 diff --git a/helm/ohdsi-webapi/README.md b/helm/ohdsi-webapi/README.md index 81b80e7e4..313b46fd9 100644 --- a/helm/ohdsi-webapi/README.md +++ b/helm/ohdsi-webapi/README.md @@ -1,15 +1,19 @@ # ohdsi-webapi -![Version: 0.1.5](https://img.shields.io/badge/Version-0.1.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.15.0](https://img.shields.io/badge/AppVersion-2.15.0-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.15.0](https://img.shields.io/badge/AppVersion-2.15.0-informational?style=flat-square) A Helm chart for OHDSI WebAPI +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | -| https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | +| Repository | Name | +|------------|------| +| file://../common | common | +| https://charts.bitnami.com/bitnami | postgresql | ## Values diff --git a/helm/ohif-viewer/Chart.yaml b/helm/ohif-viewer/Chart.yaml index 8422373ce..a578cab27 100644 --- a/helm/ohif-viewer/Chart.yaml +++ b/helm/ohif-viewer/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.14 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/ohif-viewer/README.md b/helm/ohif-viewer/README.md index 144fd1937..622b950db 100644 --- a/helm/ohif-viewer/README.md +++ b/helm/ohif-viewer/README.md @@ -1,14 +1,18 @@ # ohif-viewer -![Version: 0.1.14](https://img.shields.io/badge/Version-0.1.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 Ohif Viewer +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values diff --git a/helm/orthanc/Chart.yaml b/helm/orthanc/Chart.yaml index dd0e5e5be..b554e001e 100644 --- a/helm/orthanc/Chart.yaml +++ b/helm/orthanc/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.15 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/orthanc/README.md b/helm/orthanc/README.md index 6dd73a6ad..02b70fb1b 100644 --- a/helm/orthanc/README.md +++ b/helm/orthanc/README.md @@ -1,14 +1,18 @@ # orthanc -![Version: 0.1.15](https://img.shields.io/badge/Version-0.1.15-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 Dicom Server +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values diff --git a/helm/peregrine/Chart.yaml b/helm/peregrine/Chart.yaml index 4b00c460f..25dd1f366 100644 --- a/helm/peregrine/Chart.yaml +++ b/helm/peregrine/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.42 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,7 +25,7 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common - name: postgresql version: 11.9.13 diff --git a/helm/peregrine/README.md b/helm/peregrine/README.md index c3d1d8700..98962e9f9 100644 --- a/helm/peregrine/README.md +++ b/helm/peregrine/README.md @@ -1,15 +1,19 @@ # peregrine -![Version: 0.1.42](https://img.shields.io/badge/Version-0.1.42-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 Peregrine service +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | -| https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | +| Repository | Name | +|------------|------| +| file://../common | common | +| https://charts.bitnami.com/bitnami | postgresql | ## Values diff --git a/helm/portal/Chart.yaml b/helm/portal/Chart.yaml index 691b48c51..f333e6a67 100644 --- a/helm/portal/Chart.yaml +++ b/helm/portal/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.60 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/portal/README.md b/helm/portal/README.md index 8ed456fe5..888e67d92 100644 --- a/helm/portal/README.md +++ b/helm/portal/README.md @@ -1,14 +1,18 @@ # portal -![Version: 0.1.60](https://img.shields.io/badge/Version-0.1.60-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 data-portal +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values diff --git a/helm/portal/templates/deployment-cached.yaml b/helm/portal/templates/deployment-cached.yaml index 2a3beaa94..489128297 100644 --- a/helm/portal/templates/deployment-cached.yaml +++ b/helm/portal/templates/deployment-cached.yaml @@ -411,7 +411,8 @@ spec: resources: {{- toYaml .Values.resources | nindent 12 }} ports: - - containerPort: 80 + - containerPort: {{ .Values.service.targetPort }} + name: http - containerPort: 443 env: - name: HOSTNAME diff --git a/helm/portal/templates/secret.yaml b/helm/portal/templates/secret.yaml index 81e98d4f7..04c96de10 100644 --- a/helm/portal/templates/secret.yaml +++ b/helm/portal/templates/secret.yaml @@ -480,9 +480,80 @@ stringData: sed -n '1,20p' "/data-portal/data/config/${APP}.json" || true fi + echo "[INFO] Preparing cached-build theme assets" + + mkdir -p /data-portal/src/css + mkdir -p /data-portal/public/src/css + + CUSTOM_CSS="/data-portal/custom/css/${APP}.css" + THEME_CSS="/data-portal/src/css/themeoverrides.css" + PUBLIC_THEME_CSS="/data-portal/public/src/css/themeoverrides.css" + + if [ -f "${CUSTOM_CSS}" ]; then + echo "[INFO] Found custom CSS: ${CUSTOM_CSS}" + + cp -f "${CUSTOM_CSS}" "${THEME_CSS}" + cp -f "${CUSTOM_CSS}" "${PUBLIC_THEME_CSS}" + + echo "[INFO] Created:" + ls -l "${THEME_CSS}" "${PUBLIC_THEME_CSS}" + else + echo "[WARN] Custom CSS not found at ${CUSTOM_CSS}" + echo "[INFO] Creating empty theme override" + + printf '/* generated file - no custom theme configured */\n' \ + > "${THEME_CSS}" + + printf '/* generated file - no custom theme configured */\n' \ + > "${PUBLIC_THEME_CSS}" + fi + + echo "[INFO] Starting webpack build to ${WEBPACK_OUTDIR}" + # Run the webpack build bash runWebpack.sh + + echo "[INFO] Staging runtime static assets" + + mkdir -p "${WEBPACK_OUTDIR}/src/css" + + # themeoverrides.css is created/modified outside webpack output, + # so explicitly copy it into the cached artifact. + if [ -f "/data-portal/src/css/themeoverrides.css" ]; then + cp -f \ + "/data-portal/src/css/themeoverrides.css" \ + "${WEBPACK_OUTDIR}/src/css/themeoverrides.css" + else + echo "[WARN] /data-portal/src/css/themeoverrides.css does not exist" + fi + + # graphiql.css and any other directly-served src/css assets + if [ -d "/data-portal/src/css" ]; then + cp -a /data-portal/src/css/. "${WEBPACK_OUTDIR}/src/css/" + fi + + # Static node_modules CSS/LESS referenced directly by index.html + if [ -d "/data-portal/node_modules/@gen3/ui-component/dist/css" ]; then + mkdir -p "${WEBPACK_OUTDIR}/node_modules/@gen3/ui-component/dist/css" + cp -a \ + /data-portal/node_modules/@gen3/ui-component/dist/css/. \ + "${WEBPACK_OUTDIR}/node_modules/@gen3/ui-component/dist/css/" + fi + + echo "[INFO] Verifying cached runtime assets" + + for required in \ + "${WEBPACK_OUTDIR}/index.html" \ + "${WEBPACK_OUTDIR}/src/css/themeoverrides.css" \ + "${WEBPACK_OUTDIR}/src/css/graphiql.css" + do + if [ ! -f "$required" ]; then + echo "[ERROR] Required cached artifact missing: $required" + exit 1 + fi + done + # Stage assets that nginx will serve directly echo "[INFO] Staging extra assets into ${WEBPACK_OUTDIR}" mkdir -p "${WEBPACK_OUTDIR}/src" diff --git a/helm/requestor/Chart.yaml b/helm/requestor/Chart.yaml index 8096c4287..d933b6102 100644 --- a/helm/requestor/Chart.yaml +++ b/helm/requestor/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.34 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,7 +25,7 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common - name: postgresql version: 11.9.13 diff --git a/helm/requestor/README.md b/helm/requestor/README.md index 6d30a95e0..a8806504e 100644 --- a/helm/requestor/README.md +++ b/helm/requestor/README.md @@ -1,15 +1,19 @@ # requestor -![Version: 0.1.34](https://img.shields.io/badge/Version-0.1.34-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 Requestor Service +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | -| https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | +| Repository | Name | +|------------|------| +| file://../common | common | +| https://charts.bitnami.com/bitnami | postgresql | ## Values @@ -29,9 +33,11 @@ A Helm chart for gen3 Requestor Service | command | list | `["/bin/sh"]` | Command to run for the init container. | | commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | | criticalService | string | `"false"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | -| externalSecrets | map | `{"dbcreds":null,"pushSecret":false}` | External Secrets settings. | +| externalSecrets | map | `{"createK8sRequestorSecret":false,"dbcreds":null,"pushSecret":false,"requestorG3auto":null}` | External Secrets settings. | +| externalSecrets.createK8sRequestorSecret | string | `false` | Will create the Helm "requestor-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. | | externalSecrets.dbcreds | string | `nil` | Will override the name of the aws secrets manager secret. Default is "Values.global.environment-.Chart.Name-creds" | | externalSecrets.pushSecret | bool | `false` | Whether to create the database and Secrets Manager secrets via PushSecret. | +| externalSecrets.requestorG3auto | string | `nil` | Will override the name of the aws secrets manager secret. Default is "requestor-g3auto" | | global.autoscaling.averageCPUValue | string | `"500m"` | | | global.autoscaling.averageMemoryValue | string | `"500Mi"` | | | global.autoscaling.enabled | bool | `false` | | @@ -99,6 +105,9 @@ A Helm chart for gen3 Requestor Service | release | string | `"production"` | Valid options are "production" or "dev". If invalid option is set- the value will default to "dev". | | releaseLabel | string | `"production"` | | | replicaCount | int | `1` | Number of replicas for the deployment. | +| requestorConfig | map | `{"configYaml":"","enabled":false}` | Requestor runtime configuration. leave enabled=false and provide the secret externally. | +| requestorConfig.configYaml | str | `""` | ontents of requestor config.yaml | +| requestorConfig.enabled | bool | `false` | Create local Kubernetes secret from configYaml | | resources | map | `{"limits":{"memory":"512Mi"},"requests":{"memory":"12Mi"}}` | Resource requests and limits for the containers in the pod | | resources.limits | map | `{"memory":"512Mi"}` | The maximum amount of resources that the container is allowed to use | | resources.limits.memory | string | `"512Mi"` | The maximum amount of memory the container can use | @@ -112,6 +121,10 @@ A Helm chart for gen3 Requestor Service | service | map | `{"port":[{"name":"http","port":80,"protocol":"TCP","targetPort":80}],"targetPort":80,"type":"ClusterIP"}` | Kubernetes service information. | | service.port | int | `[{"name":"http","port":80,"protocol":"TCP","targetPort":80}]` | The port number that the service exposes. | | service.type | string | `"ClusterIP"` | Type of service. Valid values are "ClusterIP", "NodePort", "LoadBalancer", "ExternalName". | +| serviceAccount.annotations | map | `{"eks.amazonaws.com/role-arn":null}` | Annotations to add to the service account. | +| serviceAccount.annotations."eks.amazonaws.com/role-arn" | string | `nil` | The Amazon Resource Name (ARN) of the role to associate with the service account | +| serviceAccount.create | bool | `true` | Specifies whether a service account should be created. | +| serviceAccount.name | string | `"requestor-sa"` | The name of the service account | | strategy | map | `{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0},"type":"RollingUpdate"}` | Rolling update deployment strategy | | strategy.rollingUpdate.maxSurge | int | `1` | Number of additional replicas to add during rollout. | | strategy.rollingUpdate.maxUnavailable | int | `0` | Maximum amount of pods that can be unavailable during the update. | diff --git a/helm/requestor/templates/_helpers.tpl b/helm/requestor/templates/_helpers.tpl index 899b723ce..3502e57ea 100644 --- a/helm/requestor/templates/_helpers.tpl +++ b/helm/requestor/templates/_helpers.tpl @@ -77,4 +77,11 @@ Create the name of the service account to use {{- else }} {{- default .Values.secrets.password }} {{- end }} +{{- end }} + +{{/* + Requestor g3auto Secrets Manager Name +*/}} +{{- define "requestor-g3auto" -}} +{{- default "requestor-g3auto" .Values.externalSecrets.requestorG3auto }} {{- end }} \ No newline at end of file diff --git a/helm/requestor/templates/deployment.yaml b/helm/requestor/templates/deployment.yaml index 0c5346080..3fa14bcce 100644 --- a/helm/requestor/templates/deployment.yaml +++ b/helm/requestor/templates/deployment.yaml @@ -45,6 +45,7 @@ spec: affinity: {{- toYaml . | nindent 8 }} {{- end }} + serviceAccountName: {{ include "requestor.serviceAccountName" . }} automountServiceAccountToken: {{ .Values.automountServiceAccountToken }} volumes: - name: config-volume diff --git a/helm/requestor/templates/external-secret.yaml b/helm/requestor/templates/external-secret.yaml index 4a8f59d38..1072a6c9b 100644 --- a/helm/requestor/templates/external-secret.yaml +++ b/helm/requestor/templates/external-secret.yaml @@ -1,3 +1,24 @@ {{- if and .Values.global.externalSecrets.deploy (not .Values.global.externalSecrets.createLocalK8sSecret) }} {{ include "common.externalSecret.db" . }} +{{- end }} + +{{- if and .Values.global.externalSecrets.deploy (not .Values.externalSecrets.createK8sRequestorSecret) }} +--- +apiVersion: {{ include "common.externalSecrets.apiVersion" . }} +kind: ExternalSecret +metadata: + name: requestor-g3auto +spec: + refreshInterval: 5m + secretStoreRef: + name: {{ include "common.SecretStore" . }} + kind: SecretStore + target: + name: requestor-g3auto + creationPolicy: Owner + data: + - secretKey: config.yaml + remoteRef: + key: {{ include "requestor-g3auto" . }} + property: config.yaml {{- end }} \ No newline at end of file diff --git a/helm/requestor/templates/secrets.yaml b/helm/requestor/templates/secrets.yaml new file mode 100644 index 000000000..797200873 --- /dev/null +++ b/helm/requestor/templates/secrets.yaml @@ -0,0 +1,12 @@ +{{- if or (not .Values.global.externalSecrets.deploy) .Values.externalSecrets.createK8sRequestorSecret }} +{{- if .Values.requestorConfig.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: requestor-g3auto +type: Opaque +stringData: + config.yaml: | +{{ .Values.requestorConfig.configYaml | indent 4 }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/requestor/templates/serviceaccount.yaml b/helm/requestor/templates/serviceaccount.yaml new file mode 100644 index 000000000..8161fb049 --- /dev/null +++ b/helm/requestor/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "requestor.serviceAccountName" . }} + labels: + {{- include "requestor.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/requestor/values.yaml b/helm/requestor/values.yaml index edb3ab203..cc1d3d013 100644 --- a/helm/requestor/values.yaml +++ b/helm/requestor/values.yaml @@ -104,6 +104,10 @@ externalSecrets: pushSecret: false # -- (string) Will override the name of the aws secrets manager secret. Default is "Values.global.environment-.Chart.Name-creds" dbcreds: + # -- (string) Will create the Helm "requestor-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. + createK8sRequestorSecret: false + # -- (string) Will override the name of the aws secrets manager secret. Default is "requestor-g3auto" + requestorG3auto: # -- (map) Secret information for External Secrets. secrets: # -- (str) AWS access key ID. Overrides global key. @@ -111,6 +115,13 @@ secrets: # -- (str) AWS secret access key ID. Overrides global key. awsSecretAccessKey: +# -- (map) Requestor runtime configuration. leave enabled=false and provide the secret externally. +requestorConfig: + # -- (bool) Create local Kubernetes secret from configYaml + enabled: false + # -- (str) ontents of requestor config.yaml + configYaml: "" + # -- (map) Postgres database configuration. If db does not exist in postgres cluster and dbCreate is set ot true then these databases will be created for you postgres: # (bool) Whether the database should be restored from s3. Default to global.postgres.dbRestore @@ -179,6 +190,17 @@ affinity: # -- (bool) Automount the default service account token automountServiceAccountToken: false +serviceAccount: + # -- (bool) Specifies whether a service account should be created. + create: true + # -- (map) Annotations to add to the service account. + annotations: + # -- (string) The Amazon Resource Name (ARN) of the role to associate with the service account + eks.amazonaws.com/role-arn: + # If not set and create is true, a name is generated using the fullname template + # -- (string) The name of the service account + name: "requestor-sa" + # -- (map) Docker image information. image: # -- (string) Docker repository. diff --git a/helm/revproxy/Chart.yaml b/helm/revproxy/Chart.yaml index 37cb46c45..04ec47faf 100644 --- a/helm/revproxy/Chart.yaml +++ b/helm/revproxy/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.61 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/revproxy/README.md b/helm/revproxy/README.md index 6fc8ada53..4fd02f186 100644 --- a/helm/revproxy/README.md +++ b/helm/revproxy/README.md @@ -1,19 +1,24 @@ # revproxy -![Version: 0.1.61](https://img.shields.io/badge/Version-0.1.61-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 revproxy +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| +| additionalConfigs | map | `{}` | Raw nginx location blocks to add or override entries in the revproxy-nginx-subconf ConfigMap. Keys are the conf filename (e.g. "guppy-service.conf"). A key matching a built-in static conf file will replace that file's content, allowing disabled services to be suppressed or routes redirected to alternative upstreams without modifying the chart. | | affinity | map | `{}` | Affinity to use for the deployment. | | autoscaling | object | `{}` | | | commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | diff --git a/helm/revproxy/gen3.nginx.conf/fence-service-ga4gh.conf b/helm/revproxy/gen3.nginx.conf/fence-service-ga4gh.conf index af8c1d3d6..401c03b8b 100644 --- a/helm/revproxy/gen3.nginx.conf/fence-service-ga4gh.conf +++ b/helm/revproxy/gen3.nginx.conf/fence-service-ga4gh.conf @@ -11,3 +11,17 @@ location ~ \/ga4gh\/drs\/v1\/objects\/(.*)\/access { proxy_connect_timeout 400; proxy_pass $upstream; } + +location ~ \/ga4gh\/drs\/v1\/objects\/access { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } + + set $proxy_service "presigned-url-fence"; + set $upstream http://presigned-url-fence-service$des_domain; + rewrite ^/user/(.*) /$1 break; + proxy_read_timeout 400; + proxy_send_timeout 400; + proxy_connect_timeout 400; + proxy_pass $upstream; +} diff --git a/helm/revproxy/gen3.nginx.conf/fence-service.conf b/helm/revproxy/gen3.nginx.conf/fence-service.conf index e76dbfcba..c8a99787c 100644 --- a/helm/revproxy/gen3.nginx.conf/fence-service.conf +++ b/helm/revproxy/gen3.nginx.conf/fence-service.conf @@ -20,6 +20,47 @@ location /authn-proxy { client_max_body_size 0; } +# Hits the fence API and stops the original request with a 403 if the provided +# token is denylisted. Usage: `auth_request /block-denylisted-token;` +location /block-denylisted-token { + internal; + set $proxy_service "fence"; + error_page 400 =403 @block_denylisted_token; + error_page 500 =403 @block_denylisted_token; + # Previous versions of Fence that don't have the "/token/denylisted" endpoint return 405. + # Returning 403 forces the deployment of a recent Fence for endpoints that require this check. + error_page 405 =403 @block_denylisted_token; + + proxy_pass http://fence-service${des_domain}/credentials/token/denylisted; + proxy_method POST; + proxy_pass_request_body off; + proxy_set_header Authorization "$access_token"; + proxy_set_header Content-Length "0"; + proxy_set_header X-Forwarded-For "$realip"; + proxy_set_header X-UserId "$userid"; + proxy_set_header X-ReqId "$request_id"; + proxy_set_header X-SessionId "$session_id"; + proxy_set_header X-VisitorId "$visitor_id"; + proxy_set_header X-Original-URI $request_uri; + + # 4xx and 5xx errors return 403 (see `error_page` above) + proxy_intercept_errors on; + + # nginx bug that it checks even if request_body off + client_max_body_size 0; +} + +location @block_denylisted_token { + internal; + return 403 "unable to check if token is denylisted"; +} + +location /user/credentials/token/denylisted { + # Not meant to be called by users, + # but by the `auth_request /block-denylisted-token` location above. + deny all; +} + location /user/ { if ($csrf_check !~ ^ok-\S.+$) { return 403 "failed csrf check"; diff --git a/helm/revproxy/gen3.nginx.conf/gen3-discovery-ai-service.conf b/helm/revproxy/gen3.nginx.conf/gen3-discovery-ai-service.conf index 6e0827dd8..eaf41d89b 100644 --- a/helm/revproxy/gen3.nginx.conf/gen3-discovery-ai-service.conf +++ b/helm/revproxy/gen3.nginx.conf/gen3-discovery-ai-service.conf @@ -1,6 +1,32 @@ -location /ai { +location /ai/ask { if ($csrf_check !~ ^ok-\S.+$) { - return 403 "failed csrf check"; + return 403 "failed csrf check"; + } + + set $proxy_service "gen3-discovery-ai-service"; + set $upstream http://gen3-discovery-ai-service$des_domain; + rewrite ^/ai/ask/(.*) /ask/$1 break; + proxy_pass $upstream; + proxy_redirect http://$host/ https://$host/ai/ask/; + client_max_body_size 0; +} + +location /ai/topics { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } + + set $proxy_service "gen3-discovery-ai-service"; + set $upstream http://gen3-discovery-ai-service$des_domain; + rewrite ^/ai/topics/(.*) /topics/$1 break; + proxy_pass $upstream; + proxy_redirect http://$host/ https://$host/ai/topics/; + client_max_body_size 0; +} + +location /ai/discovery { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; } set $proxy_service "gen3-discovery-ai-service"; diff --git a/helm/revproxy/gen3.nginx.conf/gen3-embeddings-service.conf b/helm/revproxy/gen3.nginx.conf/gen3-embeddings-service.conf new file mode 100644 index 000000000..00fa18667 --- /dev/null +++ b/helm/revproxy/gen3.nginx.conf/gen3-embeddings-service.conf @@ -0,0 +1,56 @@ +# The Gen3 Embeddings service is deployed as part of the overall Gen3 AI +# support. Everything in Gen3 AI is available at the `/ai` route. +# This file defines routing from there to the Gen3 Embeddings service +# for functionality it handles. + +location /ai/vectorstore { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } + + set $proxy_service "gen3-embeddings-service"; + set $upstream http://gen3-embeddings-service$des_domain; + rewrite ^/ai/vectorstore/(.*) /vectorstore/$1 break; + proxy_pass $upstream; + proxy_redirect http://$host/ https://$host/ai/vectorstore/; + client_max_body_size 0; +} + +location /ai/embeddings { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } + + set $proxy_service "gen3-embeddings-service"; + set $upstream http://gen3-embeddings-service$des_domain; + rewrite ^/ai/embeddings/(.*) /embeddings/$1 break; + proxy_pass $upstream; + proxy_redirect http://$host/ https://$host/ai/embeddings/; + client_max_body_size 0; +} + +location /ai/embeddings/_version { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } + + set $proxy_service "gen3-embeddings-service"; + set $upstream http://gen3-embeddings-service$des_domain; + rewrite ^/ai/embeddings/_version /_version break; + proxy_pass $upstream; + proxy_redirect http://$host/ https://$host/ai/embeddings/_version; + client_max_body_size 0; +} + +location /ai/embeddings/_status { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } + + set $proxy_service "gen3-embeddings-service"; + set $upstream http://gen3-embeddings-service$des_domain; + rewrite ^/ai/embeddings/_status /_status break; + proxy_pass $upstream; + proxy_redirect http://$host/ https://$host/ai/embeddings/_status; + client_max_body_size 0; +} diff --git a/helm/revproxy/gen3.nginx.conf/gen3-workflow-service.conf b/helm/revproxy/gen3.nginx.conf/gen3-workflow-service.conf index 859a68bff..14a2bee34 100644 --- a/helm/revproxy/gen3.nginx.conf/gen3-workflow-service.conf +++ b/helm/revproxy/gen3.nginx.conf/gen3-workflow-service.conf @@ -2,6 +2,7 @@ location /ga4gh/tes/v1/ { if ($csrf_check !~ ^ok-\S.+$) { return 403 "failed csrf check"; } + auth_request /block-denylisted-token; set $proxy_service "gen3-workflow"; set $upstream http://gen3-workflow-service.$namespace.svc.cluster.local; @@ -12,9 +13,10 @@ location /workflows/ { if ($csrf_check !~ ^ok-\S.+$) { return 403 "failed csrf check"; } + auth_request /block-denylisted-token; set $proxy_service "gen3-workflow"; set $upstream http://gen3-workflow-service.$namespace.svc.cluster.local; rewrite ^/workflows/(.*) /$1 break; proxy_pass $upstream; -} \ No newline at end of file +} diff --git a/helm/revproxy/gen3.nginx.conf/requestor-service.conf b/helm/revproxy/gen3.nginx.conf/requestor-service.conf index 4f38b625b..1a5fbf110 100644 --- a/helm/revproxy/gen3.nginx.conf/requestor-service.conf +++ b/helm/revproxy/gen3.nginx.conf/requestor-service.conf @@ -1,11 +1,18 @@ - location /requestor/ { - if ($csrf_check !~ ^ok-\S.+$) { - return 403 "failed csrf check"; - } - - set $proxy_service "requestor-service"; - set $upstream http://requestor-service$des_domain; - rewrite ^/requestor/(.*) /$1 break; - proxy_pass $upstream; - proxy_redirect http://$host/ https://$host/requestor/; - } +location /requestor/api/v1/rems-webhook { + set $proxy_service "requestor-service"; + set $upstream http://requestor-service$des_domain; + rewrite ^/requestor/(.*) /$1 break; + proxy_pass $upstream; + proxy_redirect http://$host/ https://$host/requestor/; +} + +location /requestor/ { + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } + set $proxy_service "requestor-service"; + set $upstream http://requestor-service$des_domain; + rewrite ^/requestor/(.*) /$1 break; + proxy_pass $upstream; + proxy_redirect http://$host/ https://$host/requestor/; +} \ No newline at end of file diff --git a/helm/revproxy/gen3.nginx.conf/zendesk-wrapper-service.conf b/helm/revproxy/gen3.nginx.conf/zendesk-wrapper-service.conf new file mode 100644 index 000000000..962f89931 --- /dev/null +++ b/helm/revproxy/gen3.nginx.conf/zendesk-wrapper-service.conf @@ -0,0 +1,32 @@ + + location /zendesk/ { + + if ($csrf_check !~ ^ok-\S.+$) { + return 403 "failed csrf check"; + } + + # this is the old policy, retaining it for backward compatibility reason + set $authz_resource "/kayako"; + set $authz_method "create"; + set $authz_service "kayako"; + # # be careful - sub-request runs in same context as this request + auth_request_set $remoteUser $upstream_http_REMOTE_USER; + auth_request_set $saved_set_cookie $upstream_http_set_cookie; + auth_request /gen3-authz; + + proxy_set_header REMOTE_USER $remoteUser; + set $proxy_service "zendesk-wrapper-service"; + set $upstream http://zendesk-wrapper-service$des_domain; + rewrite ^/zendesk/(.*) /$1 break; + proxy_pass $upstream; + proxy_redirect http://$host/ https://$host/zendesk/; + + + proxy_set_header Authorization "$access_token"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + client_max_body_size 0; + } diff --git a/helm/revproxy/templates/configMaps.yaml b/helm/revproxy/templates/configMaps.yaml index 590b733f0..b09666eb0 100644 --- a/helm/revproxy/templates/configMaps.yaml +++ b/helm/revproxy/templates/configMaps.yaml @@ -3,9 +3,13 @@ kind: ConfigMap metadata: name: revproxy-nginx-subconf data: +{{- $additionalConfigs := .Values.additionalConfigs | default dict }} {{- range $path, $bytes := .Files.Glob "gen3.nginx.conf/*.conf" }} - {{ ($a := split "/" $path)._1 }}: | +{{- $filename := ($a := split "/" $path)._1 }} +{{- if not (hasKey $additionalConfigs $filename) }} + {{ $filename }}: | {{- $bytes | toString | nindent 4 }} +{{- end }} {{- end}} {{- if eq "gen3ff" .Values.global.frontendRoot }} {{ "frontend-framework-service.conf" }}: | @@ -22,6 +26,10 @@ data: {{ "robots-txt.conf" }}: | {{- .Files.Get "gen3.nginx.conf/robots/robots-txt.conf" | nindent 4}} {{- end }} +{{- range $filename, $content := $additionalConfigs }} + {{ $filename }}: | + {{- $content | nindent 4 }} +{{- end }} {{- range .Values.extraServices }} {{ printf "%s-service.conf" .name }}: | location {{ .path }}/ { @@ -40,7 +48,14 @@ data: set $proxy_service "{{ .name }}"; set $upstream http://{{ .serviceName }}$des_domain; + {{- if .customNginxConfigs }} +{{ "\n " }}{{- .customNginxConfigs | trim | replace "\n" "\n " }} +{{- end }} + + {{- if ne .rewritePath false }} rewrite ^{{ .path }}/(.*) /$1 break; + {{- end }} + proxy_pass $upstream; proxy_redirect http://$host/ https://$host{{ .path }}/; } diff --git a/helm/revproxy/values.yaml b/helm/revproxy/values.yaml index 7d84f02c3..4a3e1b3c9 100644 --- a/helm/revproxy/values.yaml +++ b/helm/revproxy/values.yaml @@ -270,6 +270,18 @@ extraServices: # authzPolicy: "protein-paint" # authzService: "protein-paint" # csrfCheck: true +# rewritePath: true +# customNginxConfigs: | +# proxy_buffering off; +# proxy_cache off; +# proxy_read_timeout 1h; +# proxy_send_timeout 1h; + +# -- (map) Raw nginx location blocks to add or override entries in the revproxy-nginx-subconf ConfigMap. +# Keys are the conf filename (e.g. "guppy-service.conf"). A key matching a built-in static conf file +# will replace that file's content, allowing disabled services to be suppressed or routes +# redirected to alternative upstreams without modifying the chart. +additionalConfigs: {} nginx: user: nginx diff --git a/helm/sheepdog/Chart.yaml b/helm/sheepdog/Chart.yaml index 93c897e0a..97e23b8f3 100644 --- a/helm/sheepdog/Chart.yaml +++ b/helm/sheepdog/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.42 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,7 +25,7 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common - name: postgresql version: 11.9.13 diff --git a/helm/sheepdog/README.md b/helm/sheepdog/README.md index 828400373..d8970578c 100644 --- a/helm/sheepdog/README.md +++ b/helm/sheepdog/README.md @@ -1,15 +1,19 @@ # sheepdog -![Version: 0.1.42](https://img.shields.io/badge/Version-0.1.42-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 Sheepdog Service +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | -| https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | +| Repository | Name | +|------------|------| +| file://../common | common | +| https://charts.bitnami.com/bitnami | postgresql | ## Values diff --git a/helm/sheepdog/sheepdog-secret/settings.py b/helm/sheepdog/sheepdog-secret/settings.py index ac896e523..f2e9b0f64 100644 --- a/helm/sheepdog/sheepdog-secret/settings.py +++ b/helm/sheepdog/sheepdog-secret/settings.py @@ -74,6 +74,7 @@ } config['USER_API'] = environ.get('FENCE_URL') or 'http://fence-service/' +config["AUTHZ_AUDIENCE"] = "gen3" # for use by authutils # use the USER_API URL instead of the public issuer URL to accquire JWT keys config['FORCE_ISSUER'] = True app_init(app) diff --git a/helm/sheepdog/templates/deployment.yaml b/helm/sheepdog/templates/deployment.yaml index 111be725e..e7ed92b63 100644 --- a/helm/sheepdog/templates/deployment.yaml +++ b/helm/sheepdog/templates/deployment.yaml @@ -62,6 +62,8 @@ spec: name: config-helper initContainers: - name: sheepdog-init + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} env: @@ -121,6 +123,8 @@ spec: fi containers: - name: sheepdog + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} ports: diff --git a/helm/sower/Chart.yaml b/helm/sower/Chart.yaml index 96408614e..3b053c336 100644 --- a/helm/sower/Chart.yaml +++ b/helm/sower/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.46 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/sower/README.md b/helm/sower/README.md index 00ba738c5..def59a16b 100644 --- a/helm/sower/README.md +++ b/helm/sower/README.md @@ -1,14 +1,18 @@ # sower -![Version: 0.1.46](https://img.shields.io/badge/Version-0.1.46-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 sower +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values diff --git a/helm/ssjdispatcher/Chart.yaml b/helm/ssjdispatcher/Chart.yaml index ee4f425c1..c6e19ffb1 100644 --- a/helm/ssjdispatcher/Chart.yaml +++ b/helm/ssjdispatcher/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.47 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,5 +25,5 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common diff --git a/helm/ssjdispatcher/README.md b/helm/ssjdispatcher/README.md index c606a0ff0..58adcbe1f 100644 --- a/helm/ssjdispatcher/README.md +++ b/helm/ssjdispatcher/README.md @@ -1,14 +1,18 @@ # ssjdispatcher -![Version: 0.1.47](https://img.shields.io/badge/Version-0.1.47-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 ssjdispatcher +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | +| Repository | Name | +|------------|------| +| file://../common | common | ## Values diff --git a/helm/ssjdispatcher/templates/serviceaccount.yaml b/helm/ssjdispatcher/templates/serviceaccount.yaml index 210ce30d1..69ceb6295 100644 --- a/helm/ssjdispatcher/templates/serviceaccount.yaml +++ b/helm/ssjdispatcher/templates/serviceaccount.yaml @@ -9,12 +9,11 @@ metadata: annotations: eks.amazonaws.com/role-arn: arn:aws:iam::{{ .Values.global.crossplane.accountId }}:role/{{ .Values.global.environment }}-{{ .Release.Namespace }}-{{ include "ssjdispatcher.serviceAccountName" . }} {{- else }} - {{- with .Values.serviceAccount.annotations }} + {{- with .Values.serviceAccount.annotations }} annotations: -{{ toYaml . | indent 4 }} - {{- end }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} - --- apiVersion: v1 kind: ServiceAccount @@ -24,11 +23,11 @@ metadata: {{- include "ssjdispatcher.labels" . | nindent 4 }} {{- if and .Values.global.crossplane.enabled .Values.global.aws.enabled }} annotations: - {{- if and .Values.global.crossplane.enabled .Values.global.aws.enabled }} - eks.amazonaws.com/role-arn: arn:aws:iam::{{ .Values.global.crossplane.accountId }}:role/{{ .Values.global.environment }}-{{ include "ssjdispatcher.serviceAccountName" . }} - {{- else }} + eks.amazonaws.com/role-arn: arn:aws:iam::{{ .Values.global.crossplane.accountId }}:role/{{ .Values.global.environment }}-{{ include "ssjdispatcher.serviceAccountName" . }} + {{- else }} {{- with .Values.jobServiceAccount.annotations }} - {{ toYaml . | nindent 4 }} - {{- end }} + annotations: + {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} {{- end }} diff --git a/helm/vectis-overlays/Chart.yaml b/helm/vectis-overlays/Chart.yaml new file mode 100644 index 000000000..49c90b686 --- /dev/null +++ b/helm/vectis-overlays/Chart.yaml @@ -0,0 +1,11 @@ +apiVersion: v2 +name: vectis-overlays +description: Vectis overlay API services (guppy-compat, siem, search-auth-proxy) +type: application +version: 0.0.0 +appVersion: "1.0" + +dependencies: + - name: common + version: "*" + repository: file://../common diff --git a/helm/vectis-overlays/README.md b/helm/vectis-overlays/README.md new file mode 100644 index 000000000..129d96d4e --- /dev/null +++ b/helm/vectis-overlays/README.md @@ -0,0 +1,112 @@ +# vectis-overlays + +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.0](https://img.shields.io/badge/AppVersion-1.0-informational?style=flat-square) + +Vectis overlay API services (guppy-compat, siem, search-auth-proxy) + +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + +## Requirements + +| Repository | Name | +|------------|------| +| file://../common | common | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| global.externalSecrets.clusterSecretStoreRef | string | `""` | | +| guppyCompat.enabled | bool | `false` | | +| guppyCompat.env.GUPPY_COMPAT_MAX_LIMIT | string | `"5000"` | | +| guppyCompat.image.pullPolicy | string | `"Always"` | | +| guppyCompat.image.repository | string | `"quay.io/cdis/gen3-vectis"` | | +| guppyCompat.image.tag | string | `"guppy-compat-v1"` | | +| guppyCompat.replicaCount | int | `1` | | +| guppyCompat.resources.limits.cpu | string | `"500m"` | | +| guppyCompat.resources.limits.memory | string | `"512Mi"` | | +| guppyCompat.resources.requests.cpu | string | `"50m"` | | +| guppyCompat.resources.requests.memory | string | `"128Mi"` | | +| guppyCompat.serviceAccount.annotations | object | `{}` | | +| guppyCompat.serviceAccount.create | bool | `true` | | +| guppyCompat.serviceAccount.name | string | `"guppy-compat-service"` | | +| migrations.enabled | bool | `false` | | +| rds.database | string | `"postgres"` | | +| rds.endpoint | string | `""` | | +| rds.port | string | `"5432"` | | +| rds.secretName | string | `""` | | +| s3Monitor.aws.region | string | `"us-east-1"` | | +| s3Monitor.db.database | string | `"postgres"` | | +| s3Monitor.db.host | string | `""` | | +| s3Monitor.db.port | string | `"5432"` | | +| s3Monitor.db.schema | string | `"vectis"` | | +| s3Monitor.db.secretName | string | `""` | | +| s3Monitor.db.secretTargetName | string | `"s3-monitor-db-creds"` | | +| s3Monitor.db.table | string | `"s3_metadata"` | | +| s3Monitor.enabled | bool | `false` | | +| s3Monitor.failedJobsHistoryLimit | int | `1` | | +| s3Monitor.image.pullPolicy | string | `"IfNotPresent"` | | +| s3Monitor.image.repository | string | `"python"` | | +| s3Monitor.image.tag | string | `"3.11-slim"` | | +| s3Monitor.initImage.pullPolicy | string | `"IfNotPresent"` | | +| s3Monitor.initImage.repository | string | `"busybox"` | | +| s3Monitor.initImage.tag | string | `"1.36"` | | +| s3Monitor.resources.limits.cpu | string | `"500m"` | | +| s3Monitor.resources.limits.memory | string | `"512Mi"` | | +| s3Monitor.resources.requests.cpu | string | `"50m"` | | +| s3Monitor.resources.requests.memory | string | `"128Mi"` | | +| s3Monitor.s3.bucket | string | `""` | | +| s3Monitor.s3.prefix | string | `""` | | +| s3Monitor.schedule | string | `"*/5 * * * *"` | | +| s3Monitor.serviceAccount.annotations | object | `{}` | | +| s3Monitor.serviceAccount.create | bool | `true` | | +| s3Monitor.serviceAccount.name | string | `"s3-monitor-sa"` | | +| s3Monitor.successfulJobsHistoryLimit | int | `3` | | +| searchAuthProxy.apiGateway.apiId | string | `""` | | +| searchAuthProxy.apiGateway.region | string | `"us-east-1"` | | +| searchAuthProxy.appDsnSecretName | string | `""` | | +| searchAuthProxy.enabled | bool | `true` | | +| searchAuthProxy.env.SEARCH_API_TLS_VERIFY | string | `""` | | +| searchAuthProxy.env.SEARCH_PROXY_DEFAULT_LIMIT | string | `"50"` | | +| searchAuthProxy.env.SEARCH_PROXY_MAX_LIMIT | string | `"1000"` | | +| searchAuthProxy.hostAliases | list | `[]` | | +| searchAuthProxy.image.pullPolicy | string | `"Always"` | | +| searchAuthProxy.image.repository | string | `"quay.io/cdis/gen3-vectis"` | | +| searchAuthProxy.image.tag | string | `"search-auth-proxy-v1"` | | +| searchAuthProxy.postgresDsnSecretRef.key | string | `"postgres_dsn"` | | +| searchAuthProxy.postgresDsnSecretRef.name | string | `"vectis-search-auth-proxy"` | | +| searchAuthProxy.replicaCount | int | `1` | | +| searchAuthProxy.resources.limits.cpu | string | `"500m"` | | +| searchAuthProxy.resources.limits.memory | string | `"512Mi"` | | +| searchAuthProxy.resources.requests.cpu | string | `"50m"` | | +| searchAuthProxy.resources.requests.memory | string | `"128Mi"` | | +| searchAuthProxy.serviceAccount.annotations | object | `{}` | | +| searchAuthProxy.serviceAccount.create | bool | `true` | | +| searchAuthProxy.serviceAccount.name | string | `"search-auth-proxy"` | | +| searchAuthProxy.serviceName | string | `"search-auth-proxy"` | | +| searchAuthProxy.siemBackend.baseUrl | string | `""` | | +| searchAuthProxy.siemBackend.indexes | string | `"security_event,audit_event,threat_indicator"` | | +| searchAuthProxy.siemBackend.searchApiRequireSigv4 | string | `"true"` | | +| searchAuthProxy.siemBackend.siemRequireSigv4 | string | `"auto"` | | +| searchAuthProxy.snapshotsBucket | string | `""` | | +| searchAuthProxy.snapshotsKeyPrefix | string | `"snapshots/"` | | +| siemService.enabled | bool | `true` | | +| siemService.env.DB_SCHEMA | string | `"vectis"` | | +| siemService.env.SIEM_DEFAULT_LIMIT | string | `"250"` | | +| siemService.env.SIEM_MAX_LIMIT | string | `"2000"` | | +| siemService.env.SIEM_VIEWS_S3_BUCKET | string | `""` | | +| siemService.env.SIEM_VIEWS_S3_ENABLED | string | `"false"` | | +| siemService.env.SIEM_VIEWS_S3_KEY | string | `"siem/views.json"` | | +| siemService.image.pullPolicy | string | `"Always"` | | +| siemService.image.repository | string | `"quay.io/cdis/gen3-vectis"` | | +| siemService.image.tag | string | `"siem-service-v1"` | | +| siemService.replicaCount | int | `1` | | +| siemService.resources.limits.cpu | string | `"500m"` | | +| siemService.resources.limits.memory | string | `"512Mi"` | | +| siemService.resources.requests.cpu | string | `"50m"` | | +| siemService.resources.requests.memory | string | `"128Mi"` | | +| siemService.serviceAccount.annotations | object | `{}` | | +| siemService.serviceAccount.create | bool | `true` | | +| siemService.serviceAccount.name | string | `"siem-service"` | | diff --git a/helm/vectis-overlays/templates/S3monitor_db.yaml b/helm/vectis-overlays/templates/S3monitor_db.yaml new file mode 100644 index 000000000..5cb2014d7 --- /dev/null +++ b/helm/vectis-overlays/templates/S3monitor_db.yaml @@ -0,0 +1,177 @@ +{{- if .Values.s3Monitor.enabled }} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: s3-monitor + namespace: {{ .Release.Namespace }} +spec: + schedule: {{ .Values.s3Monitor.schedule | quote }} + successfulJobsHistoryLimit: {{ .Values.s3Monitor.successfulJobsHistoryLimit }} + failedJobsHistoryLimit: {{ .Values.s3Monitor.failedJobsHistoryLimit }} + jobTemplate: + spec: + template: + spec: + serviceAccountName: {{ .Values.s3Monitor.serviceAccount.name }} + restartPolicy: OnFailure + initContainers: + - name: download-rds-cert + image: "{{ .Values.s3Monitor.initImage.repository }}:{{ .Values.s3Monitor.initImage.tag }}" + imagePullPolicy: {{ .Values.s3Monitor.initImage.pullPolicy }} + command: ["/bin/sh", "-c"] + args: + - | + wget -q https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem -O /ssl/global-bundle.pem + volumeMounts: + - name: ssl-cert + mountPath: /ssl + containers: + - name: s3-monitor + image: "{{ .Values.s3Monitor.image.repository }}:{{ .Values.s3Monitor.image.tag }}" + imagePullPolicy: {{ .Values.s3Monitor.image.pullPolicy }} + env: + - name: AWS_REGION + value: {{ .Values.s3Monitor.aws.region | quote }} + - name: S3_BUCKET + value: {{ .Values.s3Monitor.s3.bucket | quote }} + - name: S3_PREFIX + value: {{ .Values.s3Monitor.s3.prefix | quote }} + - name: DB_SCHEMA + value: {{ .Values.s3Monitor.db.schema | quote }} + - name: DB_TABLE + value: {{ .Values.s3Monitor.db.table | quote }} + - name: PGHOST + valueFrom: + secretKeyRef: + name: {{ .Values.s3Monitor.db.secretTargetName }} + key: PGHOST + optional: false + - name: PGPORT + valueFrom: + secretKeyRef: + name: {{ .Values.s3Monitor.db.secretTargetName }} + key: PGPORT + optional: false + - name: PGDATABASE + valueFrom: + secretKeyRef: + name: {{ .Values.s3Monitor.db.secretTargetName }} + key: PGDATABASE + optional: false + - name: PGUSER + valueFrom: + secretKeyRef: + name: {{ .Values.s3Monitor.db.secretTargetName }} + key: PGUSER + optional: false + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.s3Monitor.db.secretTargetName }} + key: PGPASSWORD + optional: false + command: ["/bin/sh", "-c"] + args: + - | + pip install -q boto3 psycopg2-binary + python - <<'PYEOF' + import os + from datetime import datetime, timezone, timedelta + + import boto3 + import psycopg2 + from psycopg2 import sql + + region = os.environ["AWS_REGION"] + bucket = os.environ["S3_BUCKET"] + prefix = os.environ.get("S3_PREFIX", "") + db_schema = os.environ.get("DB_SCHEMA", "vectis") + db_table = os.environ.get("DB_TABLE", "s3_metadata") + ssl_cert = "/ssl/global-bundle.pem" + + conn = psycopg2.connect( + host=os.environ["PGHOST"], + port=os.environ["PGPORT"], + dbname=os.environ["PGDATABASE"], + user=os.environ["PGUSER"], + password=os.environ["PGPASSWORD"], + sslmode="verify-full", + sslrootcert=ssl_cert, + connect_timeout=10, + ) + conn.autocommit = True + cur = conn.cursor() + + cur.execute(sql.SQL("CREATE SCHEMA IF NOT EXISTS {};").format(sql.Identifier(db_schema))) + cur.execute( + sql.SQL( + """ + CREATE TABLE IF NOT EXISTS {}.{} ( + id SERIAL PRIMARY KEY, + bucket_name TEXT NOT NULL, + file_key TEXT NOT NULL, + current_size_bytes BIGINT NOT NULL, + previous_size_bytes BIGINT, + event_type TEXT NOT NULL, + s3_last_modified TIMESTAMPTZ NOT NULL, + detected_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (bucket_name, file_key, detected_at) + ); + """ + ).format(sql.Identifier(db_schema), sql.Identifier(db_table)) + ) + + s3 = boto3.client("s3", region_name=region) + paginator = s3.get_paginator("list_objects_v2") + all_objects = [] + for page in paginator.paginate(Bucket=bucket, Prefix=prefix): + all_objects.extend(page.get("Contents", [])) + + cutoff = datetime.now(timezone.utc) - timedelta(minutes=5) + recent = [obj for obj in all_objects if obj["LastModified"] >= cutoff] + + for obj in recent: + key = obj["Key"] + size = obj["Size"] + last_modified = obj["LastModified"] + + cur.execute( + sql.SQL( + """ + SELECT current_size_bytes FROM {}.{} + WHERE bucket_name = %s AND file_key = %s + ORDER BY detected_at DESC LIMIT 1 + """ + ).format(sql.Identifier(db_schema), sql.Identifier(db_table)), + (bucket, key), + ) + row = cur.fetchone() + prev = row[0] if row else None + event_type = "CREATED" if row is None else "UPDATED" + + cur.execute( + sql.SQL( + """ + INSERT INTO {}.{} ( + bucket_name, file_key, current_size_bytes, + previous_size_bytes, event_type, s3_last_modified + ) VALUES (%s, %s, %s, %s, %s, %s) + """ + ).format(sql.Identifier(db_schema), sql.Identifier(db_table)), + (bucket, key, size, prev, event_type, last_modified), + ) + + cur.close() + conn.close() + + print(f"Processed {len(recent)} updated objects from s3://{bucket}/{prefix}") + PYEOF + volumeMounts: + - name: ssl-cert + mountPath: /ssl + resources: + {{- toYaml .Values.s3Monitor.resources | nindent 16 }} + volumes: + - name: ssl-cert + emptyDir: {} +{{- end }} diff --git a/helm/vectis-overlays/templates/guppy-compat.yaml b/helm/vectis-overlays/templates/guppy-compat.yaml new file mode 100644 index 000000000..b37214aba --- /dev/null +++ b/helm/vectis-overlays/templates/guppy-compat.yaml @@ -0,0 +1,62 @@ +{{- if .Values.guppyCompat.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: guppy-compat-service + namespace: {{ .Release.Namespace }} + labels: + app: guppy-compat-service +spec: + replicas: {{ .Values.guppyCompat.replicaCount }} + selector: + matchLabels: + app: guppy-compat-service + template: + metadata: + labels: + app: guppy-compat-service + spec: + serviceAccountName: {{ .Values.guppyCompat.serviceAccount.name }} + containers: + - name: guppy-compat-service + image: "{{ .Values.guppyCompat.image.repository }}:{{ .Values.guppyCompat.image.tag }}" + imagePullPolicy: {{ .Values.guppyCompat.image.pullPolicy }} + ports: + - containerPort: 8000 + env: + - name: SEARCH_BASE + value: "http://search-auth-proxy.{{ .Release.Namespace }}.svc.cluster.local:8000/search" + - name: GUPPY_COMPAT_MAX_LIMIT + value: {{ .Values.guppyCompat.env.GUPPY_COMPAT_MAX_LIMIT | quote }} + - name: ARBORIST_URL + value: "http://arborist-service.{{ .Release.Namespace }}.svc.cluster.local" + readinessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 20 + resources: + {{- toYaml .Values.guppyCompat.resources | nindent 12 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: guppy-compat-service + namespace: {{ .Release.Namespace }} + labels: + app: guppy-compat-service +spec: + selector: + app: guppy-compat-service + ports: + - name: http + port: 8000 + targetPort: 8000 +{{- end }} diff --git a/helm/vectis-overlays/templates/migration-configmap.yaml b/helm/vectis-overlays/templates/migration-configmap.yaml new file mode 100644 index 000000000..55e96a33c --- /dev/null +++ b/helm/vectis-overlays/templates/migration-configmap.yaml @@ -0,0 +1,260 @@ +{{- if .Values.migrations.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: vectis-db-migrations + namespace: {{ .Release.Namespace }} + labels: + app: vectis-db-migrations +data: + 000_create_schema.sql: | + CREATE SCHEMA IF NOT EXISTS vectis; + CREATE TABLE IF NOT EXISTS "vectis"."program" ( + "id" UUID NOT NULL, + "submitter_id" TEXT NOT NULL, + "name" TEXT NOT NULL, + PRIMARY KEY ("id") , + UNIQUE ("submitter_id") + ); + CREATE TABLE IF NOT EXISTS "vectis"."project" ( + "id" UUID NOT NULL, + "submitter_id" TEXT NOT NULL, + "code" TEXT NOT NULL, + "name" TEXT, + "cloud_account_id" TEXT, + "cloud_provider" TEXT, + "program_id" UUID NOT NULL, + PRIMARY KEY ("id") , + UNIQUE ("submitter_id") + ); + CREATE TABLE IF NOT EXISTS "vectis"."subject" ( + "id" UUID NOT NULL, + "submitter_id" TEXT NOT NULL, + "asset_type" TEXT, + "cloud_account_id" TEXT, + "cloud_provider" TEXT, + "hostname" TEXT, + "environment" TEXT, + "project_id" UUID, + PRIMARY KEY ("id") , + UNIQUE ("submitter_id") + ); + CREATE TABLE IF NOT EXISTS "vectis"."security_event" ( + "id" UUID NOT NULL, + "submitter_id" TEXT NOT NULL, + "timestamp" TEXT NOT NULL, + "action" TEXT NOT NULL, + "severity" TEXT, + "finding_type" TEXT, + "src_ip" TEXT, + "user_id" TEXT, + "http_request" TEXT, + "http_verb" TEXT, + "http_status_code" BIGINT, + "http_user_agent" TEXT, + "user_country_name" TEXT, + "resource_id" TEXT, + "rule_id" TEXT, + "cloud_account_id" TEXT, + "event_source" TEXT, + "subject_id" UUID, + PRIMARY KEY ("id") , + UNIQUE ("submitter_id") + ); + CREATE TABLE IF NOT EXISTS "vectis"."audit_event" ( + "id" UUID NOT NULL, + "submitter_id" TEXT NOT NULL, + "timestamp" TEXT NOT NULL, + "user_id" TEXT, + "src_ip" TEXT, + "http_user_agent" TEXT, + "action_name" TEXT, + "resource" TEXT, + "result" TEXT, + "is_read_only" BOOLEAN, + "cloud_region" TEXT, + "event_type" TEXT, + "request_parameters" TEXT, + "response_elements" TEXT, + "cloud_account_id" TEXT, + "event_source" TEXT, + "subject_id" UUID, + PRIMARY KEY ("id") , + UNIQUE ("submitter_id") + ); + CREATE TABLE IF NOT EXISTS "vectis"."telemetry_event" ( + "id" UUID NOT NULL, + "submitter_id" TEXT NOT NULL, + "timestamp" TEXT NOT NULL, + "src_ip" TEXT, + "dest_ip" TEXT, + "src_port" BIGINT, + "dest_port" BIGINT, + "protocol" TEXT, + "bytes_in" BIGINT, + "bytes_out" BIGINT, + "http_request" TEXT, + "http_status_code" BIGINT, + "http_user_agent" TEXT, + "packets" BIGINT, + "duration" DOUBLE PRECISION, + "disposition" TEXT, + "log_level" TEXT, + "cloud_account_id" TEXT, + "event_source" TEXT, + "subject_id" UUID, + PRIMARY KEY ("id") , + UNIQUE ("submitter_id") + ); + CREATE TABLE IF NOT EXISTS "vectis"."threat_indicator" ( + "id" UUID NOT NULL, + "submitter_id" TEXT NOT NULL, + "name" TEXT, + "description" TEXT, + "indicator_types" JSONB, + "pattern" TEXT NOT NULL, + "pattern_type" TEXT NOT NULL, + "valid_from" TEXT NOT NULL, + "valid_until" TEXT, + PRIMARY KEY ("id") , + UNIQUE ("submitter_id") + ); + CREATE TABLE IF NOT EXISTS "vectis"."security_incident" ( + "id" UUID NOT NULL, + "submitter_id" TEXT NOT NULL, + "incident_name" TEXT NOT NULL, + "description" TEXT, + "severity" TEXT, + PRIMARY KEY ("id") , + UNIQUE ("submitter_id") + ); + CREATE TABLE IF NOT EXISTS "vectis"."security_incident_security_event" ( + "security_incident_id" UUID NOT NULL, + "security_event_id" UUID NOT NULL, + PRIMARY KEY ("security_incident_id", "security_event_id") + ); + CREATE TABLE IF NOT EXISTS "vectis"."security_incident_audit_event" ( + "security_incident_id" UUID NOT NULL, + "audit_event_id" UUID NOT NULL, + PRIMARY KEY ("security_incident_id", "audit_event_id") + ); + CREATE TABLE IF NOT EXISTS "vectis"."security_incident_telemetry_event" ( + "security_incident_id" UUID NOT NULL, + "telemetry_event_id" UUID NOT NULL, + PRIMARY KEY ("security_incident_id", "telemetry_event_id") + ); + CREATE TABLE IF NOT EXISTS "vectis"."security_incident_threat_indicator" ( + "security_incident_id" UUID NOT NULL, + "threat_indicator_id" UUID NOT NULL, + PRIMARY KEY ("security_incident_id", "threat_indicator_id") + ); + + 001_rls_and_masked_views.sql: | + -- Migration 001: Row-Level Security + Masked Views for Vectis Search + BEGIN; + ALTER TABLE vectis.security_event + ADD COLUMN IF NOT EXISTS auth_resource_path TEXT + DEFAULT '/programs/vectis/projects/siem'; + ALTER TABLE vectis.audit_event + ADD COLUMN IF NOT EXISTS auth_resource_path TEXT + DEFAULT '/programs/vectis/projects/siem'; + ALTER TABLE vectis.threat_indicator + ADD COLUMN IF NOT EXISTS auth_resource_path TEXT + DEFAULT '/programs/vectis/projects/siem'; + ALTER TABLE vectis.subject + ADD COLUMN IF NOT EXISTS auth_resource_path TEXT + DEFAULT '/programs/vectis/projects/siem'; + UPDATE vectis.security_event SET auth_resource_path = '/programs/vectis/projects/siem' WHERE auth_resource_path IS NULL; + UPDATE vectis.audit_event SET auth_resource_path = '/programs/vectis/projects/siem' WHERE auth_resource_path IS NULL; + UPDATE vectis.threat_indicator SET auth_resource_path = '/programs/vectis/projects/siem' WHERE auth_resource_path IS NULL; + UPDATE vectis.subject SET auth_resource_path = '/programs/vectis/projects/siem' WHERE auth_resource_path IS NULL; + ALTER TABLE vectis.security_event ENABLE ROW LEVEL SECURITY; + ALTER TABLE vectis.audit_event ENABLE ROW LEVEL SECURITY; + ALTER TABLE vectis.threat_indicator ENABLE ROW LEVEL SECURITY; + ALTER TABLE vectis.subject ENABLE ROW LEVEL SECURITY; + DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_policies WHERE tablename='security_event' AND policyname='siem_row_access') THEN + CREATE POLICY siem_row_access ON vectis.security_event FOR SELECT + USING (auth_resource_path = ANY(string_to_array(current_setting('app.allowed_paths', true), ','))); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_policies WHERE tablename='audit_event' AND policyname='siem_row_access') THEN + CREATE POLICY siem_row_access ON vectis.audit_event FOR SELECT + USING (auth_resource_path = ANY(string_to_array(current_setting('app.allowed_paths', true), ','))); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_policies WHERE tablename='threat_indicator' AND policyname='siem_row_access') THEN + CREATE POLICY siem_row_access ON vectis.threat_indicator FOR SELECT + USING (auth_resource_path = ANY(string_to_array(current_setting('app.allowed_paths', true), ','))); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_policies WHERE tablename='subject' AND policyname='siem_row_access') THEN + CREATE POLICY siem_row_access ON vectis.subject FOR SELECT + USING (auth_resource_path = ANY(string_to_array(current_setting('app.allowed_paths', true), ','))); + END IF; + END $$; + DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'vectis_search_user') THEN + CREATE ROLE vectis_search_user WITH LOGIN PASSWORD 'vectis_search_dev'; + END IF; + END $$; + GRANT USAGE ON SCHEMA vectis TO vectis_search_user; + GRANT SELECT ON ALL TABLES IN SCHEMA vectis TO vectis_search_user; + ALTER TABLE vectis.security_event FORCE ROW LEVEL SECURITY; + ALTER TABLE vectis.audit_event FORCE ROW LEVEL SECURITY; + ALTER TABLE vectis.threat_indicator FORCE ROW LEVEL SECURITY; + ALTER TABLE vectis.subject FORCE ROW LEVEL SECURITY; + CREATE SCHEMA IF NOT EXISTS vectis_search; + GRANT USAGE ON SCHEMA vectis_search TO vectis_search_user; + CREATE OR REPLACE VIEW vectis_search.security_event WITH (security_invoker = true) AS + SELECT id, submitter_id, "timestamp", action, severity, finding_type, src_ip, + CASE WHEN current_setting('app.can_unmask', true) = 'true' THEN user_id ELSE '***' END AS user_id, + http_request, http_verb, http_status_code, http_user_agent, user_country_name, + resource_id, rule_id, cloud_account_id, event_source, subject_id + FROM vectis.security_event; + CREATE OR REPLACE VIEW vectis_search.audit_event WITH (security_invoker = true) AS + SELECT id, submitter_id, "timestamp", action_name, event_source, result, cloud_region, + cloud_account_id, src_ip, + CASE WHEN current_setting('app.can_unmask', true) = 'true' THEN user_id ELSE '***' END AS user_id, + resource, event_type, http_user_agent + FROM vectis.audit_event; + CREATE OR REPLACE VIEW vectis_search.threat_indicator WITH (security_invoker = true) AS + SELECT id, submitter_id, name, description, pattern, pattern_type, valid_from + FROM vectis.threat_indicator; + CREATE OR REPLACE VIEW vectis_search.subject WITH (security_invoker = true) AS + SELECT id, submitter_id, asset_type, cloud_account_id, cloud_provider, environment + FROM vectis.subject; + GRANT SELECT ON ALL TABLES IN SCHEMA vectis_search TO vectis_search_user; + COMMIT; + + 002_drs_auth_resource_path.sql: | + -- Migration 002: Add auth_resource_path to drs_object (idempotent). + DO $$ BEGIN + IF EXISTS (SELECT 1 FROM information_schema.tables + WHERE table_schema = 'drs' AND table_name = 'drs_object') THEN + ALTER TABLE drs.drs_object ADD COLUMN IF NOT EXISTS auth_resource_path TEXT; + END IF; + END $$; + + 003_s3_monitor_grants.sql: | + -- Migration 003: s3-monitor role grants on vectis schema. + DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 's3_monitor_role') THEN + CREATE ROLE s3_monitor_role NOLOGIN; + END IF; + END $$; + + GRANT USAGE ON SCHEMA vectis TO s3_monitor_role; + GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA vectis TO s3_monitor_role; + GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA vectis TO s3_monitor_role; + + ALTER DEFAULT PRIVILEGES IN SCHEMA vectis + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO s3_monitor_role; + ALTER DEFAULT PRIVILEGES IN SCHEMA vectis + GRANT USAGE, SELECT ON SEQUENCES TO s3_monitor_role; + + DO $$ + BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 's3_monitor_user') THEN + GRANT s3_monitor_role TO s3_monitor_user; + END IF; + END $$; +{{- end }} diff --git a/helm/vectis-overlays/templates/migration-job.yaml b/helm/vectis-overlays/templates/migration-job.yaml new file mode 100644 index 000000000..1a177af4a --- /dev/null +++ b/helm/vectis-overlays/templates/migration-job.yaml @@ -0,0 +1,69 @@ +{{- if .Values.migrations.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: vectis-db-migration + namespace: {{ .Release.Namespace }} + labels: + app: vectis-db-migration +spec: + backoffLimit: 3 + ttlSecondsAfterFinished: 3600 + template: + metadata: + labels: + app: gen3job + spec: + restartPolicy: OnFailure + automountServiceAccountToken: false + volumes: + - name: sql + configMap: + name: vectis-db-migrations + containers: + - name: migrate + image: postgres:15-alpine + imagePullPolicy: IfNotPresent + volumeMounts: + - name: sql + mountPath: /sql + env: + - name: PGHOST + valueFrom: + secretKeyRef: + name: vectis-db-migration-creds + key: host + optional: false + - name: PGUSER + valueFrom: + secretKeyRef: + name: vectis-db-migration-creds + key: username + optional: false + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: vectis-db-migration-creds + key: password + optional: false + - name: PGDATABASE + value: {{ .Values.rds.database }} + - name: PGPORT + value: {{ .Values.rds.port | quote }} + - name: PGSSLMODE + value: require + command: ["/bin/sh"] + args: + - "-ec" + - | + echo "==> Running vectis DB migrations against $PGHOST" + echo "==> [000] Base schema (idempotent — CREATE IF NOT EXISTS)..." + psql -f /sql/000_create_schema.sql + echo "==> [001] RLS + masked views..." + psql -f /sql/001_rls_and_masked_views.sql + echo "==> [002] DRS auth_resource_path column..." + psql -f /sql/002_drs_auth_resource_path.sql + echo "==> [003] s3-monitor role grants..." + psql -f /sql/003_s3_monitor_grants.sql + echo "==> All migrations complete." +{{- end }} diff --git a/helm/vectis-overlays/templates/migration-secret.yaml b/helm/vectis-overlays/templates/migration-secret.yaml new file mode 100644 index 000000000..ff98ff6e5 --- /dev/null +++ b/helm/vectis-overlays/templates/migration-secret.yaml @@ -0,0 +1,31 @@ +{{- if .Values.migrations.enabled }} +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: vectis-db-migration-creds + namespace: {{ .Release.Namespace }} + annotations: + argocd.argoproj.io/sync-wave: "-1" +spec: + refreshInterval: 1h + secretStoreRef: + name: {{ .Release.Namespace }} + kind: ClusterSecretStore + target: + name: vectis-db-migration-creds + creationPolicy: Owner + template: + data: + host: "{{ .Values.rds.endpoint }}" + username: "{{ `{{.username}}` }}" + password: "{{ `{{.password}}` }}" + data: + - secretKey: username + remoteRef: + key: {{ .Values.rds.secretName }} + property: username + - secretKey: password + remoteRef: + key: {{ .Values.rds.secretName }} + property: password +{{- end }} diff --git a/helm/vectis-overlays/templates/s3-monitor-crossplane.yaml b/helm/vectis-overlays/templates/s3-monitor-crossplane.yaml new file mode 100644 index 000000000..6934c6502 --- /dev/null +++ b/helm/vectis-overlays/templates/s3-monitor-crossplane.yaml @@ -0,0 +1,75 @@ +{{- $global := .Values.global | default dict }} +{{- $crossplane := get $global "crossplane" | default dict }} +{{- $crossplaneEnabled := get $crossplane "enabled" | default false }} +{{- if and .Values.s3Monitor.enabled .Values.s3Monitor.serviceAccount.create $crossplaneEnabled }} +apiVersion: iam.aws.crossplane.io/v1beta1 +kind: Role +metadata: + name: "{{ get $global "environment" }}-{{ .Release.Namespace }}-{{ .Values.s3Monitor.serviceAccount.name }}" +spec: + providerConfigRef: + name: {{ get $crossplane "providerConfigName" | default "provider-aws" }} + forProvider: + name: "{{ get $global "environment" }}-{{ .Release.Namespace }}-{{ .Values.s3Monitor.serviceAccount.name }}" + description: "IRSA role for s3-monitor in {{ .Release.Namespace }}" + assumeRolePolicyDocument: | + { + "Version":"2012-10-17", + "Statement":[ + { + "Effect":"Allow", + "Principal":{"Federated":"arn:aws:iam::{{ get $crossplane "accountId" }}:oidc-provider/{{ get $crossplane "oidcProviderUrl" }}"}, + "Action":"sts:AssumeRoleWithWebIdentity", + "Condition":{ + "StringEquals":{ + "{{ get $crossplane "oidcProviderUrl" }}:sub":"system:serviceaccount:{{ .Release.Namespace }}:{{ .Values.s3Monitor.serviceAccount.name }}", + "{{ get $crossplane "oidcProviderUrl" }}:aud":"sts.amazonaws.com" + } + } + } + ] + } +--- +apiVersion: iam.aws.crossplane.io/v1beta1 +kind: Policy +metadata: + name: "{{ get $global "environment" }}-{{ .Release.Namespace }}-s3-monitor-role-policy" +spec: + providerConfigRef: + name: {{ get $crossplane "providerConfigName" | default "provider-aws" }} + forProvider: + name: "{{ get $global "environment" }}-{{ .Release.Namespace }}-s3-monitor-role-policy" + document: | + { + "Version":"2012-10-17", + "Statement":[ + { + "Effect":"Allow", + "Action":["s3:ListBucket"], + "Resource":["arn:aws:s3:::{{ .Values.s3Monitor.s3.bucket }}"] + }, + { + "Effect":"Allow", + "Action":["s3:GetObject"], + "Resource":["arn:aws:s3:::{{ .Values.s3Monitor.s3.bucket }}/*"] + }, + { + "Effect":"Allow", + "Action":["secretsmanager:DescribeSecret","secretsmanager:GetSecretValue"], + "Resource":["arn:aws:secretsmanager:*:{{ get $crossplane "accountId" }}:secret:{{ .Values.s3Monitor.db.secretName }}*"] + } + ] + } +--- +apiVersion: iam.aws.crossplane.io/v1beta1 +kind: RolePolicyAttachment +metadata: + name: "{{ .Values.s3Monitor.serviceAccount.name }}-{{ .Release.Namespace }}-managed-policy-attachment" +spec: + providerConfigRef: + name: {{ get $crossplane "providerConfigName" | default "provider-aws" }} + forProvider: + roleName: "{{ get $global "environment" }}-{{ .Release.Namespace }}-{{ .Values.s3Monitor.serviceAccount.name }}" + policyArnRef: + name: "{{ get $global "environment" }}-{{ .Release.Namespace }}-s3-monitor-role-policy" +{{- end }} diff --git a/helm/vectis-overlays/templates/s3-monitor-secret.yaml b/helm/vectis-overlays/templates/s3-monitor-secret.yaml new file mode 100644 index 000000000..b7a3ba5e7 --- /dev/null +++ b/helm/vectis-overlays/templates/s3-monitor-secret.yaml @@ -0,0 +1,38 @@ +{{- if .Values.s3Monitor.enabled }} +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: {{ .Values.s3Monitor.db.secretTargetName }} + namespace: {{ .Release.Namespace }} + annotations: + argocd.argoproj.io/sync-wave: "-1" +spec: + refreshInterval: 1h + secretStoreRef: + {{- if ne .Values.global.externalSecrets.clusterSecretStoreRef "" }} + name: {{ .Values.global.externalSecrets.clusterSecretStoreRef }} + kind: ClusterSecretStore + {{- else }} + name: {{include "common.SecretStore" .}} + kind: SecretStore + {{- end }} + target: + name: {{ .Values.s3Monitor.db.secretTargetName }} + creationPolicy: Owner + template: + data: + PGHOST: {{ .Values.s3Monitor.db.host | quote }} + PGPORT: {{ .Values.s3Monitor.db.port | quote }} + PGDATABASE: {{ .Values.s3Monitor.db.database | quote }} + PGUSER: "{{ `{{.username}}` }}" + PGPASSWORD: "{{ `{{.password}}` }}" + data: + - secretKey: username + remoteRef: + key: {{ .Values.s3Monitor.db.secretName }} + property: username + - secretKey: password + remoteRef: + key: {{ .Values.s3Monitor.db.secretName }} + property: password +{{- end }} diff --git a/helm/vectis-overlays/templates/search-auth-proxy-crossplane.yaml b/helm/vectis-overlays/templates/search-auth-proxy-crossplane.yaml new file mode 100644 index 000000000..5960f6a60 --- /dev/null +++ b/helm/vectis-overlays/templates/search-auth-proxy-crossplane.yaml @@ -0,0 +1,68 @@ +{{- $global := .Values.global | default dict }} +{{- $crossplane := get $global "crossplane" | default dict }} +{{- $crossplaneEnabled := get $crossplane "enabled" | default false }} +{{- if and .Values.searchAuthProxy.enabled $crossplaneEnabled }} +apiVersion: iam.aws.crossplane.io/v1beta1 +kind: Role +metadata: + name: "{{ get $global "environment" }}-{{ .Release.Namespace }}-{{ .Values.searchAuthProxy.serviceAccount.name }}" +spec: + providerConfigRef: + name: {{ get $crossplane "providerConfigName" | default "provider-aws" }} + forProvider: + name: "{{ get $global "environment" }}-{{ .Release.Namespace }}-{{ .Values.searchAuthProxy.serviceAccount.name }}" + description: "Role for search-auth-proxy service account for {{ get $global "environment" }}" + assumeRolePolicyDocument: | + { + "Version":"2012-10-17", + "Statement":[ + { + "Sid":"", + "Effect":"Allow", + "Principal":{"Federated":"arn:aws:iam::{{ get $crossplane "accountId" }}:oidc-provider/{{ get $crossplane "oidcProviderUrl" }}"}, + "Action":"sts:AssumeRoleWithWebIdentity", + "Condition":{ + "StringEquals":{ + "{{ get $crossplane "oidcProviderUrl" }}:sub":"system:serviceaccount:{{ .Release.Namespace }}:{{ .Values.searchAuthProxy.serviceAccount.name }}", + "{{ get $crossplane "oidcProviderUrl" }}:aud":"sts.amazonaws.com" + } + } + } + ] + } +--- +apiVersion: iam.aws.crossplane.io/v1beta1 +kind: Policy +metadata: + name: "{{ get $global "environment" }}-{{ .Release.Namespace }}-search-auth-proxy-role-policy" +spec: + providerConfigRef: + name: {{ get $crossplane "providerConfigName" | default "provider-aws" }} + forProvider: + name: "{{ get $global "environment" }}-{{ .Release.Namespace }}-search-auth-proxy-role-policy" + document: | + { + "Version":"2012-10-17", + "Statement":[ + { + "Effect":"Allow", + "Action":["execute-api:Invoke"], + "Resource":[ + "arn:aws:execute-api:{{ .Values.searchAuthProxy.apiGateway.region }}:{{ get $crossplane "accountId" }}:{{ .Values.searchAuthProxy.apiGateway.apiId }}/*/*/*" + ] + } + ] + } +--- +apiVersion: iam.aws.crossplane.io/v1beta1 +kind: RolePolicyAttachment +metadata: + name: "{{ .Values.searchAuthProxy.serviceAccount.name }}-{{ .Release.Namespace }}-managed-policy-attachment" +spec: + providerConfigRef: + name: {{ get $crossplane "providerConfigName" | default "provider-aws" }} + forProvider: + roleName: "{{ get $global "environment" }}-{{ .Release.Namespace }}-{{ .Values.searchAuthProxy.serviceAccount.name }}" + policyArnRef: + name: "{{ get $global "environment" }}-{{ .Release.Namespace }}-search-auth-proxy-role-policy" +{{- end }} diff --git a/helm/vectis-overlays/templates/search-auth-proxy-secret.yaml b/helm/vectis-overlays/templates/search-auth-proxy-secret.yaml new file mode 100644 index 000000000..815c9cacb --- /dev/null +++ b/helm/vectis-overlays/templates/search-auth-proxy-secret.yaml @@ -0,0 +1,36 @@ +{{- if .Values.searchAuthProxy.enabled }} +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: vectis-search-auth-proxy + namespace: {{ .Release.Namespace }} + annotations: + argocd.argoproj.io/sync-wave: "-1" +spec: + refreshInterval: 1h + secretStoreRef: + # name: {{ .Release.Namespace }} + # kind: ClusterSecretStore + {{- if ne .Values.global.externalSecrets.clusterSecretStoreRef "" }} + name: {{ .Values.global.externalSecrets.clusterSecretStoreRef }} + kind: ClusterSecretStore + {{- else }} + name: {{include "common.SecretStore" .}} + kind: SecretStore + {{- end }} + target: + name: vectis-search-auth-proxy + creationPolicy: Owner + template: + data: + postgres_dsn: "postgresql://{{ `{{.username}}` }}:{{ `{{.password}}` }}@{{ .Values.rds.endpoint }}:{{ .Values.rds.port }}/{{ .Values.rds.database }}" + data: + - secretKey: username + remoteRef: + key: {{ default .Values.rds.secretName .Values.searchAuthProxy.appDsnSecretName }} + property: username + - secretKey: password + remoteRef: + key: {{ default .Values.rds.secretName .Values.searchAuthProxy.appDsnSecretName }} + property: password +{{- end }} diff --git a/helm/vectis-overlays/templates/search-auth-proxy.yaml b/helm/vectis-overlays/templates/search-auth-proxy.yaml new file mode 100644 index 000000000..5bb97bcc0 --- /dev/null +++ b/helm/vectis-overlays/templates/search-auth-proxy.yaml @@ -0,0 +1,88 @@ +{{- if .Values.searchAuthProxy.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.searchAuthProxy.serviceName }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Values.searchAuthProxy.serviceName }} +spec: + replicas: {{ .Values.searchAuthProxy.replicaCount }} + selector: + matchLabels: + app: {{ .Values.searchAuthProxy.serviceName }} + template: + metadata: + labels: + app: {{ .Values.searchAuthProxy.serviceName }} + spec: + serviceAccountName: {{ .Values.searchAuthProxy.serviceAccount.name }} + {{- if .Values.searchAuthProxy.hostAliases }} + hostAliases: + {{- toYaml .Values.searchAuthProxy.hostAliases | nindent 8 }} + {{- end }} + containers: + - name: {{ .Values.searchAuthProxy.serviceName }} + image: "{{ .Values.searchAuthProxy.image.repository }}:{{ .Values.searchAuthProxy.image.tag }}" + imagePullPolicy: {{ .Values.searchAuthProxy.image.pullPolicy }} + ports: + - containerPort: 8000 + env: + - name: SEARCH_API_BASE + value: "https://{{ .Values.searchAuthProxy.apiGateway.apiId }}.execute-api.{{ .Values.searchAuthProxy.apiGateway.region }}.amazonaws.com/search" + - name: SIEM_SEARCH_BASE + value: {{ default (printf "https://%s.execute-api.%s.amazonaws.com/search" .Values.searchAuthProxy.apiGateway.apiId .Values.searchAuthProxy.apiGateway.region) .Values.searchAuthProxy.siemBackend.baseUrl | quote }} + - name: SEARCH_API_REQUIRE_SIGV4 + value: {{ .Values.searchAuthProxy.siemBackend.searchApiRequireSigv4 | quote }} + {{- if .Values.searchAuthProxy.env.SEARCH_API_TLS_VERIFY }} + - name: SEARCH_API_TLS_VERIFY + value: {{ .Values.searchAuthProxy.env.SEARCH_API_TLS_VERIFY | quote }} + {{- end }} + - name: SIEM_SEARCH_REQUIRE_SIGV4 + value: {{ .Values.searchAuthProxy.siemBackend.siemRequireSigv4 | quote }} + - name: SIEM_INDEXES + value: {{ .Values.searchAuthProxy.siemBackend.indexes | quote }} + - name: ARBORIST_URL + value: "http://arborist-service.{{ .Release.Namespace }}.svc.cluster.local" + - name: SEARCH_PROXY_DEFAULT_LIMIT + value: {{ .Values.searchAuthProxy.env.SEARCH_PROXY_DEFAULT_LIMIT | quote }} + - name: SEARCH_PROXY_MAX_LIMIT + value: {{ .Values.searchAuthProxy.env.SEARCH_PROXY_MAX_LIMIT | quote }} + {{- if .Values.searchAuthProxy.snapshotsBucket }} + - name: SNAPSHOTS_BUCKET + value: {{ .Values.searchAuthProxy.snapshotsBucket | quote }} + {{- end }} + {{- if .Values.searchAuthProxy.snapshotsKeyPrefix }} + - name: SNAPSHOTS_KEY_PREFIX + value: {{ .Values.searchAuthProxy.snapshotsKeyPrefix | quote }} + {{- end }} + readinessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 20 + resources: + {{- toYaml .Values.searchAuthProxy.resources | nindent 12 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.searchAuthProxy.serviceName }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Values.searchAuthProxy.serviceName }} +spec: + selector: + app: {{ .Values.searchAuthProxy.serviceName }} + ports: + - name: http + port: 8000 + targetPort: 8000 +{{- end }} diff --git a/helm/vectis-overlays/templates/service-accounts.yaml b/helm/vectis-overlays/templates/service-accounts.yaml new file mode 100644 index 000000000..b741494f2 --- /dev/null +++ b/helm/vectis-overlays/templates/service-accounts.yaml @@ -0,0 +1,65 @@ +{{- $global := .Values.global | default dict }} +{{- $crossplane := get $global "crossplane" | default dict }} +{{- $crossplaneEnabled := get $crossplane "enabled" | default false }} +{{- if and .Values.guppyCompat.enabled .Values.guppyCompat.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.guppyCompat.serviceAccount.name }} + namespace: {{ .Release.Namespace }} + {{- with .Values.guppyCompat.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +--- +{{- end }} +{{- if and .Values.siemService.enabled .Values.siemService.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.siemService.serviceAccount.name }} + namespace: {{ .Release.Namespace }} + {{- if $crossplaneEnabled }} + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::{{ get $crossplane "accountId" }}:role/{{ get $global "environment" }}-{{ .Release.Namespace }}-{{ .Values.siemService.serviceAccount.name }} + {{- else }} + {{- with .Values.siemService.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +--- +{{- end }} +{{- if and .Values.searchAuthProxy.enabled .Values.searchAuthProxy.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.searchAuthProxy.serviceAccount.name }} + namespace: {{ .Release.Namespace }} + {{- if $crossplaneEnabled }} + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::{{ get $crossplane "accountId" }}:role/{{ get $global "environment" }}-{{ .Release.Namespace }}-{{ .Values.searchAuthProxy.serviceAccount.name }} + {{- else }} + {{- with .Values.searchAuthProxy.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} +{{- if and .Values.s3Monitor.enabled .Values.s3Monitor.serviceAccount.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.s3Monitor.serviceAccount.name }} + namespace: {{ .Release.Namespace }} + {{- if $crossplaneEnabled }} + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::{{ get $crossplane "accountId" }}:role/{{ get $global "environment" }}-{{ .Release.Namespace }}-{{ .Values.s3Monitor.serviceAccount.name }} + {{- else }} + {{- with .Values.s3Monitor.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/vectis-overlays/templates/siem-service-crossplane.yaml b/helm/vectis-overlays/templates/siem-service-crossplane.yaml new file mode 100644 index 000000000..9329df34b --- /dev/null +++ b/helm/vectis-overlays/templates/siem-service-crossplane.yaml @@ -0,0 +1,69 @@ +{{- $global := .Values.global | default dict }} +{{- $crossplane := get $global "crossplane" | default dict }} +{{- $crossplaneEnabled := get $crossplane "enabled" | default false }} +{{- if and .Values.siemService.enabled .Values.siemService.serviceAccount.create $crossplaneEnabled }} +apiVersion: iam.aws.crossplane.io/v1beta1 +kind: Role +metadata: + name: "{{ get $global "environment" }}-{{ .Release.Namespace }}-siem-service" +spec: + providerConfigRef: + name: {{ get $crossplane "providerConfigName" | default "provider-aws" }} + forProvider: + name: "{{ get $global "environment" }}-{{ .Release.Namespace }}-siem-service" + description: "IRSA role for siem-service in {{ .Release.Namespace }} — S3 access to siem views config" + assumeRolePolicyDocument: | + { + "Version":"2012-10-17", + "Statement":[ + { + "Effect":"Allow", + "Principal":{"Federated":"arn:aws:iam::{{ get $crossplane "accountId" }}:oidc-provider/{{ get $crossplane "oidcProviderUrl" }}"}, + "Action":"sts:AssumeRoleWithWebIdentity", + "Condition":{ + "StringEquals":{ + "{{ get $crossplane "oidcProviderUrl" }}:sub":"system:serviceaccount:{{ .Release.Namespace }}:{{ .Values.siemService.serviceAccount.name }}", + "{{ get $crossplane "oidcProviderUrl" }}:aud":"sts.amazonaws.com" + } + } + } + ] + } +--- +apiVersion: iam.aws.crossplane.io/v1beta1 +kind: Policy +metadata: + name: "{{ get $global "environment" }}-{{ .Release.Namespace }}-siem-service-role-policy" +spec: + providerConfigRef: + name: {{ get $crossplane "providerConfigName" | default "provider-aws" }} + forProvider: + name: "{{ get $global "environment" }}-{{ .Release.Namespace }}-siem-service-role-policy" + document: | + { + "Version":"2012-10-17", + "Statement":[ + { + "Effect":"Allow", + "Action":["s3:GetObject","s3:PutObject"], + "Resource":[ + {{- $bucket := .Values.siemService.env.SIEM_VIEWS_S3_BUCKET | default "" }} + "arn:aws:s3:::{{ $bucket }}/siem/views.json", + "arn:aws:s3:::{{ $bucket }}/siem/default-views.json" + ] + } + ] + } +--- +apiVersion: iam.aws.crossplane.io/v1beta1 +kind: RolePolicyAttachment +metadata: + name: "siem-service-{{ .Release.Namespace }}-managed-policy-attachment" +spec: + providerConfigRef: + name: {{ get $crossplane "providerConfigName" | default "provider-aws" }} + forProvider: + roleName: "{{ get $global "environment" }}-{{ .Release.Namespace }}-siem-service" + policyArnRef: + name: "{{ get $global "environment" }}-{{ .Release.Namespace }}-siem-service-role-policy" +{{- end }} diff --git a/helm/vectis-overlays/templates/siem-service.yaml b/helm/vectis-overlays/templates/siem-service.yaml new file mode 100644 index 000000000..f92959d33 --- /dev/null +++ b/helm/vectis-overlays/templates/siem-service.yaml @@ -0,0 +1,73 @@ +{{- if .Values.siemService.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: siem-service + namespace: {{ .Release.Namespace }} + labels: + app: siem-service +spec: + replicas: {{ .Values.siemService.replicaCount }} + selector: + matchLabels: + app: siem-service + template: + metadata: + labels: + app: siem-service + spec: + serviceAccountName: {{ .Values.siemService.serviceAccount.name }} + containers: + - name: siem-service + image: "{{ .Values.siemService.image.repository }}:{{ .Values.siemService.image.tag }}" + imagePullPolicy: {{ .Values.siemService.image.pullPolicy }} + ports: + - containerPort: 8000 + env: + - name: ARBORIST_URL + value: "http://arborist-service.{{ .Release.Namespace }}.svc.cluster.local" + - name: SIEM_DEFAULT_LIMIT + value: {{ .Values.siemService.env.SIEM_DEFAULT_LIMIT | quote }} + - name: SIEM_MAX_LIMIT + value: {{ .Values.siemService.env.SIEM_MAX_LIMIT | quote }} + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: vectis-search-auth-proxy + key: postgres_dsn + - name: DB_SCHEMA + value: {{ .Values.siemService.env.DB_SCHEMA | default "vectis" | quote }} + - name: SIEM_BACKEND + value: {{ .Values.siemService.env.SIEM_BACKEND | default "postgres" | quote }} + - name: SIEM_INDEX_BACKENDS + value: {{ .Values.siemService.env.SIEM_INDEX_BACKENDS | default "" | quote }} + readinessProbe: + httpGet: + path: /siem/health + port: 8000 + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /siem/health + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 20 + resources: + {{- toYaml .Values.siemService.resources | nindent 12 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: siem-service + namespace: {{ .Release.Namespace }} + labels: + app: siem-service +spec: + selector: + app: siem-service + ports: + - name: http + port: 8000 + targetPort: 8000 +{{- end }} diff --git a/helm/vectis-overlays/values.yaml b/helm/vectis-overlays/values.yaml new file mode 100644 index 000000000..6cab6e769 --- /dev/null +++ b/helm/vectis-overlays/values.yaml @@ -0,0 +1,158 @@ +# vectis-overlays default values. +# All service-internal URLs use .Release.Namespace so the chart is env-agnostic. +# Override image tags per-environment in gitops values. +# Adding global value + + +global: + externalSecrets: + clusterSecretStoreRef: "" + +guppyCompat: + enabled: false + replicaCount: 1 + serviceAccount: + create: true + name: guppy-compat-service + annotations: {} + image: + repository: quay.io/cdis/gen3-vectis + tag: guppy-compat-v1 + pullPolicy: Always + resources: + requests: + cpu: "50m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "512Mi" + env: + GUPPY_COMPAT_MAX_LIMIT: "5000" + +siemService: + enabled: true + replicaCount: 1 + serviceAccount: + create: true + name: siem-service + annotations: {} + image: + repository: quay.io/cdis/gen3-vectis + tag: siem-service-v1 + pullPolicy: Always + resources: + requests: + cpu: "50m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "512Mi" + env: + DB_SCHEMA: "vectis" + SIEM_DEFAULT_LIMIT: "250" + SIEM_MAX_LIMIT: "2000" + SIEM_VIEWS_S3_BUCKET: "" + SIEM_VIEWS_S3_KEY: "siem/views.json" + SIEM_VIEWS_S3_ENABLED: "false" + +searchAuthProxy: + enabled: true + replicaCount: 1 + serviceName: search-auth-proxy + serviceAccount: + create: true + name: search-auth-proxy + annotations: {} + image: + repository: quay.io/cdis/gen3-vectis + tag: search-auth-proxy-v1 + pullPolicy: Always + resources: + requests: + cpu: "50m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "512Mi" + env: + SEARCH_PROXY_DEFAULT_LIMIT: "50" + SEARCH_PROXY_MAX_LIMIT: "1000" + SEARCH_API_TLS_VERIFY: "" + # S3 bucket for pre-baked /search/graphql snapshots, populated by the + # snapshot-publisher Lambda (CDK SnapshotsBucketName output). Leave empty + # to disable the /snapshots/* passthrough. + snapshotsBucket: "" + snapshotsKeyPrefix: "snapshots/" + apiGateway: + apiId: "" + region: "us-east-1" + # Override DNS for the execute-api hostname to bypass the VPC endpoint. + # Set per-environment to the public API Gateway IPs when the execute-api + # VPC endpoint causes 403s (HTTP APIs are not routable through private + # execute-api VPC endpoints — they require a Private REST API). + hostAliases: [] + # Dual-route backend configuration: + # - Discovery/Guppy traffic uses SEARCH_API_BASE (API Gateway/Lambda path) + # - SIEM index traffic can use SIEM_SEARCH_BASE (RDS/ES-backed path) + siemBackend: + baseUrl: "" + searchApiRequireSigv4: "true" + siemRequireSigv4: "auto" + indexes: "security_event,audit_event,threat_indicator" + postgresDsnSecretRef: + name: vectis-search-auth-proxy + key: postgres_dsn + # Optional separate AWS secret name for the search-role credentials. + # Falls back to rds.secretName for backward compatibility. + appDsnSecretName: "" + +s3Monitor: + enabled: false + schedule: "*/5 * * * *" + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 1 + serviceAccount: + create: true + name: s3-monitor-sa + annotations: {} + image: + repository: python + tag: "3.11-slim" + pullPolicy: IfNotPresent + initImage: + repository: busybox + tag: "1.36" + pullPolicy: IfNotPresent + resources: + requests: + cpu: "50m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "512Mi" + aws: + region: "us-east-1" + s3: + bucket: "" + prefix: "" + db: + schema: "vectis" + table: "s3_metadata" + secretName: "" + secretTargetName: "s3-monitor-db-creds" + host: "" + port: "5432" + database: "postgres" + +# CDK RDS secret in Secrets Manager — used to build the postgres DSN ExternalSecret. +# Set per-environment in gitops values. +rds: + secretName: "" + endpoint: "" + port: "5432" + database: postgres + +# Database migrations — one-time Job to bootstrap the vectis schema and RLS views. +# Set migrations.enabled: true in gitops values to trigger the Job, then flip back to false. +migrations: + enabled: false diff --git a/helm/workspace-proxy/Chart.yaml b/helm/workspace-proxy/Chart.yaml new file mode 100644 index 000000000..23c0d1f53 --- /dev/null +++ b/helm/workspace-proxy/Chart.yaml @@ -0,0 +1,14 @@ +apiVersion: v2 +name: workspace-proxy +description: > + Per-user workspace HTTP/WebSocket router for gen3 vectis. + Replaces Emissary/Ambassador. Reads Service annotations written by Hatchery + to resolve each user's workspace upstream, then proxies traffic from revproxy. +type: application +version: 0.0.0 +appVersion: "1.0" + +dependencies: + - name: common + version: "*" + repository: file://../common diff --git a/helm/workspace-proxy/README.md b/helm/workspace-proxy/README.md new file mode 100644 index 000000000..a712cca15 --- /dev/null +++ b/helm/workspace-proxy/README.md @@ -0,0 +1,34 @@ +# workspace-proxy + +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.0](https://img.shields.io/badge/AppVersion-1.0-informational?style=flat-square) + +Per-user workspace HTTP/WebSocket router for gen3 vectis. Replaces Emissary/Ambassador. Reads Service annotations written by Hatchery to resolve each user's workspace upstream, then proxies traffic from revproxy. + +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + +## Requirements + +| Repository | Name | +|------------|------| +| file://../common | common | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| deploymentNamespace | string | `""` | | +| image.pullPolicy | string | `"Always"` | | +| image.repository | string | `"quay.io/cdis/gen3-vectis"` | | +| image.tag | string | `"qa-goproxy"` | | +| jegKernelSpecPolicy | string | `""` | | +| kubernetesApiServerCIDRs | list | `[]` | | +| listenAddr | string | `":8080"` | | +| networkPolicy.enabled | bool | `true` | | +| replicaCount | int | `2` | | +| resources.limits.cpu | string | `"500m"` | | +| resources.limits.memory | string | `"256Mi"` | | +| resources.requests.cpu | string | `"50m"` | | +| resources.requests.memory | string | `"64Mi"` | | +| workspaceNamespace | string | `""` | | diff --git a/helm/workspace-proxy/templates/deployment.yaml b/helm/workspace-proxy/templates/deployment.yaml new file mode 100644 index 000000000..40bdaf1db --- /dev/null +++ b/helm/workspace-proxy/templates/deployment.yaml @@ -0,0 +1,63 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: workspace-proxy + namespace: {{ .Values.deploymentNamespace }} + labels: + app: workspace-proxy +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: workspace-proxy + template: + metadata: + labels: + app: workspace-proxy + spec: + serviceAccountName: workspace-proxy + automountServiceAccountToken: true + containers: + - name: workspace-proxy + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - --listen={{ .Values.listenAddr }} + - --workspace-namespace={{ .Values.workspaceNamespace }} + env: + - name: JEG_GATEWAY_URL + value: "http://jupyter-enterprise-gateway.{{ .Values.workspaceNamespace }}.svc.cluster.local:8888" + - name: WORKSPACE_NAMESPACE + value: "{{ .Values.workspaceNamespace }}" + - name: JEG_KERNEL_SPEC_POLICY + value: {{ .Values.jegKernelSpecPolicy | quote }} + ports: + - name: http + containerPort: 8080 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 5 + periodSeconds: 15 + readinessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 3 + periodSeconds: 10 + resources: + {{- toYaml .Values.resources | nindent 12 }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + capabilities: + drop: + - ALL + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault diff --git a/helm/workspace-proxy/templates/namespace.yaml b/helm/workspace-proxy/templates/namespace.yaml new file mode 100644 index 000000000..c2beca049 --- /dev/null +++ b/helm/workspace-proxy/templates/namespace.yaml @@ -0,0 +1,2 @@ +{{- /* Namespace owned by the hatchery chart (jupyter-namespace.yaml). + workspace-proxy deploys into it but does not create it. */ -}} diff --git a/helm/workspace-proxy/templates/netpol.yaml b/helm/workspace-proxy/templates/netpol.yaml new file mode 100644 index 000000000..5b3efcdac --- /dev/null +++ b/helm/workspace-proxy/templates/netpol.yaml @@ -0,0 +1,56 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: workspace-proxy + namespace: {{ .Values.deploymentNamespace }} + labels: + app: workspace-proxy +spec: + podSelector: + matchLabels: + app: workspace-proxy + policyTypes: + - Ingress + - Egress + + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Values.deploymentNamespace }} + podSelector: + matchLabels: + app: revproxy + ports: + - protocol: TCP + port: 8080 + + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Values.workspaceNamespace }} + ports: + - protocol: TCP + port: 80 + - protocol: TCP + port: 8888 + {{- range .Values.kubernetesApiServerCIDRs }} + - to: + - ipBlock: + cidr: {{ . | quote }} + ports: + - protocol: TCP + port: 443 + {{- end }} +{{- end }} diff --git a/helm/workspace-proxy/templates/rbac.yaml b/helm/workspace-proxy/templates/rbac.yaml new file mode 100644 index 000000000..c0826c679 --- /dev/null +++ b/helm/workspace-proxy/templates/rbac.yaml @@ -0,0 +1,29 @@ +# Role in the workspace namespace — workspace-proxy reads Services written by Hatchery +# (one per user session) to resolve proxy upstreams. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: workspace-proxy + namespace: {{ .Values.workspaceNamespace }} + labels: + app: workspace-proxy +rules: + - apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: workspace-proxy + namespace: {{ .Values.workspaceNamespace }} + labels: + app: workspace-proxy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: workspace-proxy +subjects: + - kind: ServiceAccount + name: workspace-proxy + namespace: {{ .Values.deploymentNamespace }} diff --git a/helm/workspace-proxy/templates/service.yaml b/helm/workspace-proxy/templates/service.yaml new file mode 100644 index 000000000..2f2b29ada --- /dev/null +++ b/helm/workspace-proxy/templates/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: workspace-proxy-service + namespace: {{ .Values.deploymentNamespace }} + labels: + app: workspace-proxy +spec: + type: ClusterIP + selector: + app: workspace-proxy + ports: + - name: http + port: 8080 + targetPort: http + protocol: TCP diff --git a/helm/workspace-proxy/templates/serviceaccount.yaml b/helm/workspace-proxy/templates/serviceaccount.yaml new file mode 100644 index 000000000..d9d0ff5b7 --- /dev/null +++ b/helm/workspace-proxy/templates/serviceaccount.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: workspace-proxy + namespace: {{ .Values.deploymentNamespace }} + labels: + app: workspace-proxy diff --git a/helm/workspace-proxy/values.yaml b/helm/workspace-proxy/values.yaml new file mode 100644 index 000000000..19c43318f --- /dev/null +++ b/helm/workspace-proxy/values.yaml @@ -0,0 +1,39 @@ +# workspace-proxy default values. +# Override via your environment's workspace-proxy-values.yaml + +# Namespace where the workspace-proxy pod runs (same as gen3 helm release). +# Must be set per-environment in gitops values. +deploymentNamespace: "" + +# Namespace where Hatchery creates user workspace pods and services. +# CRITICAL: must match hatchery user-namespace and jeg workspaceNamespace exactly. +workspaceNamespace: "" + +replicaCount: 2 + +image: + repository: quay.io/cdis/gen3-vectis + tag: qa-goproxy + pullPolicy: Always + +listenAddr: ":8080" + +# Optional JSON policy controlling which JEG kernelspecs are visible/launchable. +# Example: {"allowedSpecs":["python3"],"costPerHour":{"python3":0.0}} +jegKernelSpecPolicy: "" + +resources: + requests: + cpu: "50m" + memory: "64Mi" + limits: + cpu: "500m" + memory: "256Mi" + +# NetworkPolicy: restrict ingress to revproxy only. +networkPolicy: + enabled: true + +# Egress destinations for the in-cluster Kubernetes API service +# (kubernetes.default.svc -> service ClusterIP). Must be set per environment. +kubernetesApiServerCIDRs: [] diff --git a/helm/wts/Chart.yaml b/helm/wts/Chart.yaml index 6b471f60f..b4a2a0ade 100644 --- a/helm/wts/Chart.yaml +++ b/helm/wts/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.40 +version: 0.0.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -25,7 +25,7 @@ appVersion: "master" dependencies: - name: common - version: 0.1.36 + version: "*" repository: file://../common - name: postgresql version: 11.9.13 diff --git a/helm/wts/README.md b/helm/wts/README.md index 3b4497d61..ea69e4c59 100644 --- a/helm/wts/README.md +++ b/helm/wts/README.md @@ -1,15 +1,19 @@ # wts -![Version: 0.1.40](https://img.shields.io/badge/Version-0.1.40-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 workspace token service +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + ## Requirements -| Repository | Name | Version | -|------------|------|---------| -| file://../common | common | 0.1.36 | -| https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | +| Repository | Name | +|------------|------| +| file://../common | common | +| https://charts.bitnami.com/bitnami | postgresql | ## Values diff --git a/helm/wts/templates/deployment.yaml b/helm/wts/templates/deployment.yaml index 5a4503aa3..6c32c51b9 100644 --- a/helm/wts/templates/deployment.yaml +++ b/helm/wts/templates/deployment.yaml @@ -73,6 +73,8 @@ spec: serviceAccountName: workspace-token-service containers: - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} volumeMounts: @@ -147,6 +149,8 @@ spec: {{- toYaml .Values.resources | nindent 12 }} initContainers: - name: wts-db-migrate + securityContext: + {{- toYaml .Values.securityContext | nindent 10 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} volumeMounts: diff --git a/helm/zendesk-wrapper/.helmignore b/helm/zendesk-wrapper/.helmignore new file mode 100644 index 000000000..21846e965 --- /dev/null +++ b/helm/zendesk-wrapper/.helmignore @@ -0,0 +1,17 @@ +.DS_Store +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +*.swp +*.bak +*.tmp +*.orig +*~ +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/zendesk-wrapper/Chart.yaml b/helm/zendesk-wrapper/Chart.yaml new file mode 100644 index 000000000..9696ec2a6 --- /dev/null +++ b/helm/zendesk-wrapper/Chart.yaml @@ -0,0 +1,29 @@ +apiVersion: v2 +name: zendesk-wrapper +description: A Helm chart for gen3 Zendesk Wrapper Service + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.0.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "master" + +dependencies: + - name: common + version: "*" + repository: file://../common diff --git a/helm/zendesk-wrapper/README.md b/helm/zendesk-wrapper/README.md new file mode 100644 index 000000000..fc2e7f9b9 --- /dev/null +++ b/helm/zendesk-wrapper/README.md @@ -0,0 +1,69 @@ +# zendesk-wrapper + +![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) + +A Helm chart for gen3 Zendesk Wrapper Service + +Published versions of this chart are listed in the +[Helm repository](https://helm.gen3.org) (`helm search repo gen3`) and on the +[releases page](https://github.com/uc-cdis/gen3-helm/releases). + +## Requirements + +| Repository | Name | +|------------|------| +| file://../common | common | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].key | string | `"app"` | | +| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].operator | string | `"In"` | | +| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].values[0] | string | `"zendesk-wrapper"` | | +| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey | string | `"kubernetes.io/hostname"` | | +| affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].weight | int | `100` | | +| autoscaling | object | `{}` | | +| commonLabels | string | `nil` | | +| criticalService | string | `"false"` | | +| env | map | `[{"name":"GEN3_ZENDESK_URL","value":""},{"name":"GEN3_ZENDESK_BRAND_ID","value":""}]` | Environment variables for the Zendesk wrapper service | +| env[0] | string | `{"name":"GEN3_ZENDESK_URL","value":""}` | Zendesk instance URL (e.g., https://gen3support.zendesk.com) | +| env[1] | string | `{"name":"GEN3_ZENDESK_BRAND_ID","value":""}` | Zendesk brand ID (e.g., 123456, can be found in Zendesk admin panel) | +| externalSecrets | map | `{"name":"zendesk-wrapper-secret"}` | Secret environment variables (referenced from Kubernetes secrets) | +| externalSecrets.name | string | `"zendesk-wrapper-secret"` | Name of the Kubernetes secret containing the Zendesk API token | +| global.autoscaling.averageCPUValue | string | `"500m"` | | +| global.autoscaling.averageMemoryValue | string | `"500Mi"` | | +| global.autoscaling.enabled | bool | `false` | | +| global.autoscaling.maxReplicas | int | `10` | | +| global.autoscaling.minReplicas | int | `1` | | +| global.environment | string | `"default"` | | +| global.externalSecrets | map | `{"deploy":false,"separateSecretStore":false}` | External Secrets settings. | +| global.externalSecrets.deploy | bool | `false` | Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any zendesk wrapper secrets you have deployed. | +| global.externalSecrets.separateSecretStore | string | `false` | Will deploy a separate External Secret Store for this service. | +| global.minAvailable | int | `1` | | +| global.netPolicy.dbSubnet | string | `""` | | +| global.netPolicy.enabled | bool | `false` | | +| global.pdb | bool | `false` | | +| global.topologySpread.enabled | bool | `false` | | +| global.topologySpread.maxSkew | int | `1` | | +| global.topologySpread.topologyKey | string | `"topology.kubernetes.io/zone"` | | +| image.pullPolicy | string | `"Always"` | | +| image.repository | string | `"quay.io/cdis/zendesk-wrapper-service"` | | +| image.tag | string | `""` | | +| metricsEnabled | string | `nil` | | +| netPolicy.egressApps[0] | string | `"zendesk-wrapper"` | | +| netPolicy.ingressApps[0] | string | `"zendesk-wrapper"` | | +| partOf | string | `"Core-Service"` | | +| podAnnotations."gen3.io/network-ingress" | string | `"zendesk-wrapper"` | | +| release | string | `"production"` | | +| replicaCount | int | `1` | | +| resources.limits.memory | string | `"128Mi"` | | +| revisionHistoryLimit | int | `2` | | +| selectorLabels | string | `nil` | | +| service.httpPort | int | `80` | Port on which the service is exposed | +| service.httpsPort | int | `443` | Secure port on which the service is exposed | +| service.targetPort | int | `80` | Port on which the service is exposed for Zendesk wrapper API | +| service.type | string | `"ClusterIP"` | | +| strategy.rollingUpdate.maxSurge | int | `1` | | +| strategy.rollingUpdate.maxUnavailable | int | `0` | | +| strategy.type | string | `"RollingUpdate"` | | diff --git a/helm/zendesk-wrapper/templates/NOTES.txt b/helm/zendesk-wrapper/templates/NOTES.txt new file mode 100644 index 000000000..c1e7e1aef --- /dev/null +++ b/helm/zendesk-wrapper/templates/NOTES.txt @@ -0,0 +1 @@ +{{ .Chart.Name }} has been deployed successfully. diff --git a/helm/zendesk-wrapper/templates/_helpers.tpl b/helm/zendesk-wrapper/templates/_helpers.tpl new file mode 100644 index 000000000..b0e3dafcf --- /dev/null +++ b/helm/zendesk-wrapper/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "zendesk-wrapper.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "zendesk-wrapper.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "zendesk-wrapper.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "zendesk-wrapper.labels" -}} +{{- if .Values.commonLabels }} + {{- with .Values.commonLabels }} + {{- toYaml . }} + {{- end }} +{{- else }} + {{- (include "common.commonLabels" .)}} +{{- end }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "zendesk-wrapper.selectorLabels" -}} +{{- if .Values.selectorLabels }} + {{- with .Values.selectorLabels }} + {{- toYaml . }} + {{- end }} +{{- else }} + {{- (include "common.selectorLabels" .)}} +{{- end }} +{{- end }} + +{{/* + Zendesk Wrapper Secrets Manager Name +*/}} +{{- define "zendesk-wrapper-secret" -}} +{{- default "zendesk-wrapper-secret" .Values.externalSecrets.name }} +{{- end }} diff --git a/helm/zendesk-wrapper/templates/deployment.yaml b/helm/zendesk-wrapper/templates/deployment.yaml new file mode 100644 index 000000000..42bd23a40 --- /dev/null +++ b/helm/zendesk-wrapper/templates/deployment.yaml @@ -0,0 +1,77 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: zendesk-wrapper-deployment + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "zendesk-wrapper.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "zendesk-wrapper.selectorLabels" . | nindent 6 }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- with .Values.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "zendesk-wrapper.selectorLabels" . | nindent 8 }} + {{- include "common.extraLabels" . | nindent 8 }} + netnolimit: 'yes' + public: 'yes' + spec: + {{- if .Values.global.topologySpread.enabled }} + {{- include "common.TopologySpread" . | nindent 6 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + readinessProbe: + httpGet: + path: /_status/ + port: http + initialDelaySeconds: 30 + periodSeconds: 60 + timeoutSeconds: 30 + livenessProbe: + httpGet: + path: /_status/ + port: http + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 30 + failureThreshold: 6 + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - /bin/bash + - /src/start.sh + ports: + - name: http + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + resources: + {{- toYaml .Values.resources | nindent 12 }} + env: + {{- toYaml .Values.env | nindent 12 }} + - name: ZENDESK_OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: zendesk-wrapper-secret + key: clientSecret + - name: ZENDESK_OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + name: zendesk-wrapper-secret + key: clientID diff --git a/helm/zendesk-wrapper/templates/external-secret.yaml b/helm/zendesk-wrapper/templates/external-secret.yaml new file mode 100644 index 000000000..0dc01be23 --- /dev/null +++ b/helm/zendesk-wrapper/templates/external-secret.yaml @@ -0,0 +1,24 @@ +--- +{{- if .Values.global.externalSecrets.deploy }} +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: zendesk-wrapper-secret +spec: + refreshInterval: 5m + secretStoreRef: + name: {{include "common.SecretStore" .}} + kind: SecretStore + target: + name: zendesk-wrapper-secret + creationPolicy: Owner + data: + - secretKey: clientID + remoteRef: + key: {{include "zendesk-wrapper-secret" .}} + property: clientID + - secretKey: clientSecret + remoteRef: + key: {{include "zendesk-wrapper-secret" .}} + property: clientSecret +{{- end }} diff --git a/helm/zendesk-wrapper/templates/service.yaml b/helm/zendesk-wrapper/templates/service.yaml new file mode 100644 index 000000000..7e2fa975e --- /dev/null +++ b/helm/zendesk-wrapper/templates/service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + name: "zendesk-wrapper-service" + labels: + {{- include "zendesk-wrapper.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - protocol: TCP + port: {{ .Values.service.httpPort }} + targetPort: {{ .Values.service.targetPort }} + name: http + nodePort: null + - protocol: TCP + port: {{ .Values.service.httpsPort }} + targetPort: {{ .Values.service.targetPort }} + name: https + nodePort: null + selector: + {{- include "zendesk-wrapper.selectorLabels" . | nindent 4 }} diff --git a/helm/zendesk-wrapper/values.yaml b/helm/zendesk-wrapper/values.yaml new file mode 100644 index 000000000..96b77440a --- /dev/null +++ b/helm/zendesk-wrapper/values.yaml @@ -0,0 +1,98 @@ +# Default values for zendesk-wrapper. + +global: + environment: default + pdb: false + minAvailable: 1 + netPolicy: + enabled: false + dbSubnet: "" + # -- (map) External Secrets settings. + externalSecrets: + # -- (bool) Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any zendesk wrapper secrets you have deployed. + deploy: false + # -- (string) Will deploy a separate External Secret Store for this service. + separateSecretStore: false + autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 10 + averageCPUValue: 500m + averageMemoryValue: 500Mi + topologySpread: + enabled: false + topologyKey: "topology.kubernetes.io/zone" + maxSkew: 1 + +autoscaling: {} + +metricsEnabled: + +podAnnotations: {"gen3.io/network-ingress": "zendesk-wrapper"} + +replicaCount: 1 + +revisionHistoryLimit: 2 + +strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + +affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - zendesk-wrapper + topologyKey: "kubernetes.io/hostname" + +image: + repository: quay.io/cdis/zendesk-wrapper-service + pullPolicy: Always + tag: "" + +resources: + limits: + memory: 128Mi + +service: + type: ClusterIP + # -- (int) Port on which the service is exposed + httpPort: 80 + # -- (int) Secure port on which the service is exposed + httpsPort: 443 + # -- (int) Port on which the service is exposed for Zendesk wrapper API + targetPort: 80 + +netPolicy: + ingressApps: + - zendesk-wrapper + egressApps: + - zendesk-wrapper + +# -- (map) Environment variables for the Zendesk wrapper service +env: + # -- (string) Zendesk instance URL (e.g., https://gen3support.zendesk.com) + - name: GEN3_ZENDESK_URL + value: "" + # -- (string) Zendesk brand ID (e.g., 123456, can be found in Zendesk admin panel) + - name: GEN3_ZENDESK_BRAND_ID + value: "" + +# -- (map) Secret environment variables (referenced from Kubernetes secrets) +externalSecrets: + # -- (string) Name of the Kubernetes secret containing the Zendesk API token + name: "zendesk-wrapper-secret" + +release: "production" +criticalService: "false" +partOf: "Core-Service" +selectorLabels: +commonLabels: diff --git a/vectis/Notes.md b/vectis/Notes.md new file mode 100644 index 000000000..c511fc6c2 --- /dev/null +++ b/vectis/Notes.md @@ -0,0 +1,66 @@ +# Note on running vectis locally + +Need to support linux-amd platform as these do not have images for arm64. + +need to enable the following: +* Rosetta for macbook +* Kind config below + +To run a hatchery pod: need to run: `kubectl label node kind-multi-node-control-plane role=jupyter` + +# create api credential with scope "credentials": +``` +fence-create token-create --scopes openid,user,fence,data,credentials,google_service_account,google_credentials --type access_token --exp 10800 --username craigrbarnes@uchicago.edu +``` + +## Kind config +```yaml +# kind config to handle running kind with linux-amd64 nodes + +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: kind-multi-node +networking: + ipFamily: ipv4 + apiServerAddress: 127.0.0.1 +nodes: + - role: control-plane + extraMounts: + - hostPath: ./coredns-custom + containerPath: /etc/coredns/custom + kubeadmConfigPatches: + - | + kind: ClusterConfiguration + apiVersion: kubeadm.k8s.io/v1beta3 + dns: + type: CoreDNS + coreDNS: + extraArgs: + conf: /etc/coredns/custom/Corefile + - | + kind: InitConfiguration + apiVersion: kubeadm.k8s.io/v1beta3 + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + extraPortMappings: + - containerPort: 80 + hostPort: 80 + protocol: TCP + - containerPort: 443 + hostPort: 443 + protocol: TCP +containerdConfigPatches: + - |- + [plugins."io.containerd.grpc.v1.cri"] + disable_apparmor = true + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + DisableNewKeyring = true + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."runc-amd64"] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."runc-amd64".options] + SystemdCgroup = true + DisableNewKeyring = true +``` \ No newline at end of file