diff --git a/README.md b/README.md index 77f4226..ce6a9f0 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,39 @@ runcrate jobs cancel own code, 124 on timeout, 130 when cancelled. Ctrl-C detaches and leaves the job running; `-d` / `--detach` prints the job ID and returns. +## Schedules + +A schedule starts an instance or a job later: once with `--at`, or on a recurring +5-field `--cron`. A command after `--` makes it a job (same flags as `runcrate run`); +without one it starts an instance (same flags as `runcrate instances create`, so +`--name` is required). `--max-runtime` is required for both, so nothing a schedule +starts keeps running unattended. + +```bash +runcrate schedule create --cron '0 2 * * *' --tz Europe/London --name nightly-train \ + --gpu H100 --volume --max-runtime 2h -- python train.py +runcrate schedule create --at '2026-10-21 09:00' --name morning-box \ + --gpu A4000 --max-runtime 8h --idle-stop 30m +runcrate schedule list [-q] [--json] # when, next run (your local time), last result +runcrate schedule get # details, last run, last error +runcrate schedule pause # resume turns it back on +runcrate schedule run-now # start it now; the next run stays as it is +runcrate schedule delete [--yes] # what it already started keeps running +``` + +- `--at '2026-10-21 02:00'` is the wall time in `--tz`; a time with an offset + (`2026-10-21T02:00:00+01:00`) is used as is. `--tz` takes an IANA name and + defaults to this machine's timezone, for `--cron` too. +- Across daylight saving: a time the clocks skip moves forward by the gap (02:30 + becomes 03:30), and a time that happens twice runs the first time. +- Recurring runs are at least 15 minutes apart. `--overlap skip` (the default) + skips a run while the previous one is still going; `--overlap allow` starts it. +- Runs start within about 2 minutes of their time, as you, in the current + environment. While the environment's monthly budget is reached, runs are skipped. +- A schedule set up with an API key or an agent sign-in launches with it; when + that key is revoked or the sign-in ends, the schedule pauses (last result + `paused: key revoked`). + ## Updating ```bash diff --git a/internal/api/schedules.go b/internal/api/schedules.go new file mode 100644 index 0000000..9c1fb60 --- /dev/null +++ b/internal/api/schedules.go @@ -0,0 +1,141 @@ +package api + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/url" +) + +// Schedule starts an instance or a job later: once at RunAt, or on a 5-field +// Cron in Timezone. +type Schedule struct { + ID string `json:"id"` + Name string `json:"name"` + Kind string `json:"kind"` // "instance" or "job" + EnvironmentID string `json:"environment_id"` + CreatedBy string `json:"created_by"` + // Spec is the create body of the kind. Env values are never returned, only + // their names under env_keys. + Spec map[string]any `json:"spec"` + MaxRuntimeMinutes int `json:"max_runtime_minutes"` + RunAt *string `json:"run_at"` + Cron *string `json:"cron"` + Timezone string `json:"timezone"` + Overlap string `json:"overlap"` + Enabled bool `json:"enabled"` + NextRunAt *string `json:"next_run_at"` + LastRunAt *string `json:"last_run_at"` + // LastStatus is launched, failed, skipped_budget, skipped_overlap, + // no_access or credential_revoked. + LastStatus *string `json:"last_status"` + LastError *string `json:"last_error"` + // RunAs is the user each run launches as; RunAsCredential is how the + // schedule was set up: user, api_key or agent. + RunAs *string `json:"run_as"` + RunAsCredential string `json:"run_as_credential"` + LastInstanceID *string `json:"last_instance_id"` + LastJobID *string `json:"last_job_id"` + RunCount int `json:"run_count"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +// CreateScheduleRequest is the body of POST /api/v1/schedules. Spec is a +// CreateInstanceRequest or a CreateJobRequest; set exactly one of RunAt and Cron. +type CreateScheduleRequest struct { + Kind string `json:"kind"` + Spec any `json:"spec"` + RunAt string `json:"run_at,omitempty"` + Cron string `json:"cron,omitempty"` + Timezone string `json:"timezone,omitempty"` + Overlap string `json:"overlap,omitempty"` + Name string `json:"name,omitempty"` +} + +// UpdateScheduleRequest is the body of PATCH /api/v1/schedules/{id}. Nil +// fields are left as they are. +type UpdateScheduleRequest struct { + Enabled *bool `json:"enabled,omitempty"` +} + +// ScheduleRun is what run-now started: Instance or Job, by the schedule's kind. +type ScheduleRun struct { + Schedule Schedule `json:"schedule"` + Instance *Instance `json:"instance,omitempty"` + Job *Job `json:"job,omitempty"` +} + +// DeletedSchedule is the response of DELETE /api/v1/schedules/{id}. +type DeletedSchedule struct { + ID string `json:"id"` + Deleted bool `json:"deleted"` +} + +// CreateSchedule creates a schedule in the current environment. +func (c *Client) CreateSchedule(req CreateScheduleRequest) (*Schedule, error) { + return callJSON[Schedule](c, "POST", "/api/v1/schedules", req) +} + +// ListSchedules returns the environment's schedules, newest first. +func (c *Client) ListSchedules() ([]Schedule, error) { + list, err := callJSON[[]Schedule](c, "GET", "/api/v1/schedules", nil) + if err != nil { + return nil, err + } + return *list, nil +} + +// GetSchedule returns one schedule. +func (c *Client) GetSchedule(id string) (*Schedule, error) { + return callJSON[Schedule](c, "GET", schedulePath(id), nil) +} + +// UpdateSchedule changes a schedule; enabled false pauses it, true resumes it. +func (c *Client) UpdateSchedule(id string, req UpdateScheduleRequest) (*Schedule, error) { + return callJSON[Schedule](c, "PATCH", schedulePath(id), req) +} + +// DeleteSchedule removes a schedule. What it already started keeps running. +func (c *Client) DeleteSchedule(id string) (*DeletedSchedule, error) { + return callJSON[DeletedSchedule](c, "DELETE", schedulePath(id), nil) +} + +// RunScheduleNow launches the schedule's instance or job immediately, whether +// it is enabled or not, without moving its next run. +func (c *Client) RunScheduleNow(id string) (*ScheduleRun, error) { + return callJSON[ScheduleRun](c, "POST", schedulePath(id)+"/run-now", nil) +} + +func schedulePath(id string) string { + return "/api/v1/schedules/" + url.PathEscape(id) +} + +// callJSON sends body (nil for none) as JSON and decodes the { data: T } reply. +func callJSON[T any](c *Client, method, path string, body any) (*T, error) { + var reader io.Reader + if body != nil { + raw, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("encoding request: %w", err) + } + reader = bytes.NewReader(raw) + } + + resp, err := c.do(method, path, reader) + if err != nil { + return nil, fmt.Errorf("network error: %w", err) + } + defer resp.Body.Close() + + if err := checkHTTPStatus(resp); err != nil { + return nil, err + } + + var result apiEnvelope[T] + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("parsing response: %w", err) + } + return &result.Data, nil +} diff --git a/internal/commands/helpers.go b/internal/commands/helpers.go index d537cf1..1e623d2 100644 --- a/internal/commands/helpers.go +++ b/internal/commands/helpers.go @@ -3,6 +3,7 @@ package commands import ( "encoding/json" "fmt" + "io" "os" "os/exec" "runtime" @@ -41,7 +42,12 @@ func loadClient() (*api.Client, error) { // newTable creates a tabwriter for formatted table output. func newTable() *tabwriter.Writer { - return tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + return newTableTo(os.Stdout) +} + +// newTableTo is newTable writing to w. +func newTableTo(w io.Writer) *tabwriter.Writer { + return tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) } // printJSON outputs data as indented JSON. diff --git a/internal/commands/instances.go b/internal/commands/instances.go index ac2e2f3..a4dbd78 100644 --- a/internal/commands/instances.go +++ b/internal/commands/instances.go @@ -237,6 +237,62 @@ var instancesTypesCmd = &cobra.Command{ // ── create ────────────────────────────────────────────────────────────────── +// addInstanceOnlyFlags registers the create flags that only an instance takes, +// shared by instances create and schedule create. +func addInstanceOnlyFlags(cmd *cobra.Command) { + cmd.Flags().String("type-id", "", "Instance type ID from 'runcrate instances types'") + cmd.Flags().String("idle-stop", "", + "Stop the instance after this long with GPU use under 5% and nobody connected (SSH, web terminal or editor), 5m to 24h (e.g. 30m), GPU instances only. Recommended for unattended work") +} + +// buildInstanceRequest turns the create flags (name, gpu, gpu-count, region, +// template, type-id, volume, max-runtime, idle-stop) into the create body. +func buildInstanceRequest(cmd *cobra.Command) (api.CreateInstanceRequest, error) { + var req api.CreateInstanceRequest + name, _ := cmd.Flags().GetString("name") + gpuRaw, _ := cmd.Flags().GetString("gpu") + gpuCountRaw, _ := cmd.Flags().GetInt("gpu-count") + region, _ := cmd.Flags().GetString("region") + template, _ := cmd.Flags().GetString("template") + typeID, _ := cmd.Flags().GetString("type-id") + volumeID, _ := cmd.Flags().GetString("volume") + maxRuntimeRaw, _ := cmd.Flags().GetString("max-runtime") + idleStopRaw, _ := cmd.Flags().GetString("idle-stop") + + maxRuntime, err := parseMinutesFlag("max-runtime", maxRuntimeRaw, maxRuntimeMaxMinutes) + if err != nil { + return req, err + } + idleStop, err := parseMinutesFlag("idle-stop", idleStopRaw, idleStopMaxMinutes) + if err != nil { + return req, err + } + + gpuType, gpuCount, err := parseGPUFlag(gpuRaw, gpuCountRaw, cmd.Flags().Changed("gpu-count")) + if err != nil { + return req, err + } + + if name == "" { + return req, fmt.Errorf("--name is required") + } + if gpuType == "" && typeID == "" { + return req, fmt.Errorf("provide --gpu for auto-deploy or --type-id from 'runcrate instances types'") + } + + return api.CreateInstanceRequest{ + Name: name, + GPUType: gpuType, + GPUCount: gpuCount, + Region: region, + Template: template, + InstanceTypeID: typeID, + MaxRuntimeMinutes: maxRuntime, + IdleShutdownMinutes: idleStop, + StorageID: volumeID, + }, nil +} + var instancesCreateCmd = &cobra.Command{ Use: "create", Short: "Create and deploy a new instance", @@ -247,36 +303,11 @@ does not keep billing after the job ends. Auto stop terminates the instance, so anything you need on a storage volume.`, Example: " runcrate instances create --name train --gpu H100 --max-runtime 4h --idle-stop 30m", RunE: func(cmd *cobra.Command, args []string) error { - name, _ := cmd.Flags().GetString("name") - gpuRaw, _ := cmd.Flags().GetString("gpu") - gpuCountRaw, _ := cmd.Flags().GetInt("gpu-count") - region, _ := cmd.Flags().GetString("region") - template, _ := cmd.Flags().GetString("template") - typeID, _ := cmd.Flags().GetString("type-id") - volumeID, _ := cmd.Flags().GetString("volume") - maxRuntimeRaw, _ := cmd.Flags().GetString("max-runtime") - idleStopRaw, _ := cmd.Flags().GetString("idle-stop") - - maxRuntime, err := parseMinutesFlag("max-runtime", maxRuntimeRaw, maxRuntimeMaxMinutes) - if err != nil { - return err - } - idleStop, err := parseMinutesFlag("idle-stop", idleStopRaw, idleStopMaxMinutes) + req, err := buildInstanceRequest(cmd) if err != nil { return err } - - gpuType, gpuCount, err := parseGPUFlag(gpuRaw, gpuCountRaw, cmd.Flags().Changed("gpu-count")) - if err != nil { - return err - } - - if name == "" { - return fmt.Errorf("--name is required") - } - if gpuType == "" && typeID == "" { - return fmt.Errorf("provide --gpu for auto-deploy or --type-id from 'runcrate instances types'") - } + volumeID := req.StorageID client, err := loadClient() if err != nil { @@ -286,23 +317,11 @@ anything you need on a storage volume.`, // Check for duplicate active instance name existing, _ := client.ListInstances("") for _, inst := range existing { - if strings.EqualFold(inst.Name, name) && isActiveInstance(inst.Status) { - return fmt.Errorf("instance %q already exists (ID: %s, status: %s). Use a different name or delete it first", name, inst.ID, inst.Status) + if strings.EqualFold(inst.Name, req.Name) && isActiveInstance(inst.Status) { + return fmt.Errorf("instance %q already exists (ID: %s, status: %s). Use a different name or delete it first", req.Name, inst.ID, inst.Status) } } - req := api.CreateInstanceRequest{ - Name: name, - GPUType: gpuType, - GPUCount: gpuCount, - Region: region, - Template: template, - InstanceTypeID: typeID, - MaxRuntimeMinutes: maxRuntime, - IdleShutdownMinutes: idleStop, - StorageID: volumeID, - } - instance, err := client.CreateInstance(req) if api.IsBudgetReached(err) { return errors.New(budgetReachedMessage) @@ -588,12 +607,10 @@ func init() { instancesCreateCmd.Flags().Int("gpu-count", 1, "Number of GPUs (alternative to colon syntax)") instancesCreateCmd.Flags().String("region", "", "Preferred region") instancesCreateCmd.Flags().String("template", "", "OS template") - instancesCreateCmd.Flags().String("type-id", "", "Instance type ID from 'runcrate instances types'") instancesCreateCmd.Flags().String("volume", "", "Storage volume ID to attach, from 'runcrate volumes list'; its files are at /workspace") instancesCreateCmd.Flags().String("max-runtime", "", "Stop the instance automatically after this long, 5m to 168h (e.g. 4h). Recommended for unattended work") - instancesCreateCmd.Flags().String("idle-stop", "", - "Stop the instance after this long with GPU use under 5% and nobody connected (SSH, web terminal or editor), 5m to 24h (e.g. 30m), GPU instances only. Recommended for unattended work") + addInstanceOnlyFlags(instancesCreateCmd) // auto-stop instancesAutoStopCmd.Flags().String("max-runtime", "", diff --git a/internal/commands/run.go b/internal/commands/run.go index 84a8f53..4f323dc 100644 --- a/internal/commands/run.go +++ b/internal/commands/run.go @@ -99,7 +99,15 @@ func parseEnvFlags(vals []string, lookup func(string) (string, bool)) (map[strin return env, nil } -// buildJobRequest turns run's flags and arguments into the create body. +// addJobOnlyFlags registers the flags that only a job takes, shared by run and +// schedule create. +func addJobOnlyFlags(cmd *cobra.Command) { + cmd.Flags().String("image", "", "Container image to run the command in") + cmd.Flags().StringArray("env", nil, "Environment variable KEY=VALUE, or KEY to copy it from your shell (repeatable)") +} + +// buildJobRequest turns the job flags (gpu, gpu-count, max-runtime, volume, +// image, env, template, name, region) and the command into the create body. func buildJobRequest(cmd *cobra.Command, args []string, lookupEnv func(string) (string, bool)) (api.CreateJobRequest, error) { var req api.CreateJobRequest @@ -168,8 +176,7 @@ running.`, cmd.Flags().String("volume", "", "Volume ID to mount as /workspace") cmd.Flags().String("max-runtime", formatMinutes(defaultMaxRuntimeMinutes), "Stop the job after this long: 90m, 2h, or minutes (1m to 24h)") cmd.Flags().String("template", "", "OS template (default ubuntu-devbox)") - cmd.Flags().String("image", "", "Container image to run the command in") - cmd.Flags().StringArray("env", nil, "Environment variable KEY=VALUE, or KEY to copy it from your shell (repeatable)") + addJobOnlyFlags(cmd) cmd.Flags().String("name", "", "Job name (default job-)") cmd.Flags().String("region", "", "Preferred region") cmd.Flags().BoolP("detach", "d", false, "Print the job ID and return without following its output") diff --git a/internal/commands/schedule.go b/internal/commands/schedule.go new file mode 100644 index 0000000..eebf9f1 --- /dev/null +++ b/internal/commands/schedule.go @@ -0,0 +1,662 @@ +package commands + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + // Zone data built in, so --tz works where the OS has none (Windows). + _ "time/tzdata" + + "github.com/runcrate/cli/internal/api" + "github.com/spf13/cobra" +) + +// ── when ──────────────────────────────────────────────────────────────────── + +// Wall-time layouts for --at, read in --tz. +var atWallLayouts = []string{ + "2006-01-02 15:04", + "2006-01-02 15:04:05", + "2006-01-02T15:04", + "2006-01-02T15:04:05", +} + +// Layouts for --at with an explicit offset or Z, used as is. +var atOffsetLayouts = []string{ + time.RFC3339, + "2006-01-02T15:04Z07:00", + "2006-01-02 15:04Z07:00", + "2006-01-02 15:04:05Z07:00", +} + +const displayTimeLayout = "2006-01-02 15:04 MST" + +// parseAt reads --at. A time with an offset or Z is that instant; a wall time +// such as "2026-10-21 02:00" is read in loc. notice is set when the wall time +// does not exist in loc (clocks go forward) and was moved past the gap. +func parseAt(value string, loc *time.Location, now time.Time) (t time.Time, notice string, err error) { + value = strings.TrimSpace(value) + for _, layout := range atOffsetLayouts { + if t, err = time.Parse(layout, value); err == nil { + return checkFuture(value, t, now) + } + } + for _, layout := range atWallLayouts { + naive, perr := time.ParseInLocation(layout, value, time.UTC) + if perr != nil { + continue + } + t, shifted := wallTime(naive, loc) + if shifted { + notice = fmt.Sprintf("%s does not exist in %s (the clocks go forward), so it runs at %s.", + naive.Format("2006-01-02 15:04"), loc, t.In(loc).Format(displayTimeLayout)) + } + t, _, err = checkFuture(value, t, now) + return t, notice, err + } + return time.Time{}, "", fmt.Errorf("invalid --at %q: use a time like '2026-10-21 02:00' (in --tz) or '2026-10-21T02:00:00+01:00'", value) +} + +func checkFuture(value string, t, now time.Time) (time.Time, string, error) { + if !t.After(now) { + return time.Time{}, "", fmt.Errorf("--at %s is in the past", value) + } + return t, "", nil +} + +// wallTime is the instant when loc's clocks read naive's wall clock (naive is +// read as UTC). A time that happens twice (clocks go back) is the first one; a +// time that never happens (clocks go forward) keeps the offset from before the +// change, so it lands the length of the gap later (02:30 becomes 03:30), as the +// server does for cron. shifted reports that case. +func wallTime(naive time.Time, loc *time.Location) (t time.Time, shifted bool) { + _, before := naive.Add(-24 * time.Hour).In(loc).Zone() + _, after := naive.Add(24 * time.Hour).In(loc).Zone() + var first time.Time + for _, offset := range []int{before, after} { + c := naive.Add(-time.Duration(offset) * time.Second) + if !sameWallClock(c.In(loc), naive) { + continue + } + if first.IsZero() || c.Before(first) { + first = c + } + } + if !first.IsZero() { + return first, false + } + return naive.Add(-time.Duration(before) * time.Second), true +} + +func sameWallClock(a, b time.Time) bool { + ay, am, ad := a.Date() + by, bm, bd := b.Date() + return ay == by && am == bm && ad == bd && + a.Hour() == b.Hour() && a.Minute() == b.Minute() && a.Second() == b.Second() +} + +// localTimezone is this machine's IANA zone: $TZ when it names one, else the +// /etc/localtime link. +func localTimezone() (string, error) { + if tz, ok := os.LookupEnv("TZ"); ok { + tz = strings.TrimPrefix(tz, ":") + if tz == "" { + return "UTC", nil + } + if !filepath.IsAbs(tz) { + if _, err := time.LoadLocation(tz); err == nil { + return tz, nil + } + } + } + if target, err := os.Readlink("/etc/localtime"); err == nil { + if _, name, ok := strings.Cut(target, "zoneinfo/"); ok { + if _, err := time.LoadLocation(name); err == nil { + return name, nil + } + } + } + return "", errors.New("could not tell this machine's timezone; pass --tz, e.g. --tz Europe/London") +} + +// resolveTimezone loads --tz, or this machine's zone when it is empty. +func resolveTimezone(name string) (*time.Location, error) { + if name == "" { + local, err := localTimezone() + if err != nil { + return nil, err + } + name = local + } + loc, err := time.LoadLocation(name) + if err != nil || name == "Local" { + return nil, fmt.Errorf("unknown --tz %q: use an IANA name such as Europe/London or America/New_York", name) + } + return loc, nil +} + +// ── create ────────────────────────────────────────────────────────────────── + +// Flags that only one kind takes. +var ( + instanceOnlyScheduleFlags = []string{"type-id", "idle-stop"} + jobOnlyScheduleFlags = []string{"image", "env"} +) + +// buildScheduleRequest turns schedule create's flags and the job command after +// -- into the create body. A command after -- makes it a job schedule, else an +// instance schedule. notice is a note for stderr (a DST-shifted --at). +func buildScheduleRequest(cmd *cobra.Command, args []string, lookupEnv func(string) (string, bool), now time.Time) (req api.CreateScheduleRequest, notice string, err error) { + flags := cmd.Flags() + at, _ := flags.GetString("at") + cron, _ := flags.GetString("cron") + tz, _ := flags.GetString("tz") + overlap, _ := flags.GetString("overlap") + + // When. + switch { + case at != "" && cron != "": + return req, "", errors.New("pass only one of --at (one run) and --cron (recurring)") + case at == "" && cron == "": + return req, "", errors.New("pass --at '2026-10-21 02:00' for one run or --cron '0 2 * * *' for a recurring one") + } + loc, err := resolveTimezone(tz) + if err != nil { + return req, "", err + } + req.Timezone = loc.String() + if at != "" { + if flags.Changed("overlap") { + return req, "", errors.New("--overlap only applies to --cron schedules") + } + t, n, err := parseAt(at, loc, now) + if err != nil { + return req, "", err + } + req.RunAt = t.UTC().Format(time.RFC3339) + notice = n + } else { + fields := strings.Fields(cron) + if len(fields) != 5 { + return req, "", fmt.Errorf("invalid --cron %q: use 5 fields, minute hour day-of-month month day-of-week, e.g. '0 2 * * *'", cron) + } + req.Cron = strings.Join(fields, " ") + if overlap != "skip" && overlap != "allow" { + return req, "", fmt.Errorf("invalid --overlap %q: use skip or allow", overlap) + } + req.Overlap = overlap + } + + // What. + dash := cmd.ArgsLenAtDash() + if dash > 0 || (dash < 0 && len(args) > 0) { + return req, "", fmt.Errorf("unexpected argument %q: put a job's command after --, e.g. -- python train.py", args[0]) + } + isJob := dash == 0 + if isJob && len(args) == 0 { + return req, "", errors.New("give the job's command after --, e.g. -- python train.py") + } + + maxRuntime, _ := flags.GetString("max-runtime") + if strings.TrimSpace(maxRuntime) == "" { + return req, "", errors.New("--max-runtime is required for a schedule, so nothing it starts keeps running (and billing) unattended, e.g. --max-runtime 2h (instances 5m to 168h, jobs 1m to 24h)") + } + + req.Name, _ = flags.GetString("name") + if isJob { + if err := refuseFlags(flags.Changed, instanceOnlyScheduleFlags, "instance schedules (no command after --)"); err != nil { + return req, "", err + } + spec, err := buildJobRequest(cmd, args, lookupEnv) + if err != nil { + return req, "", err + } + req.Kind, req.Spec = "job", spec + return req, notice, nil + } + + if err := refuseFlags(flags.Changed, jobOnlyScheduleFlags, "job schedules (a command after --)"); err != nil { + return req, "", err + } + spec, err := buildInstanceRequest(cmd) + if err != nil { + return req, "", err + } + req.Kind, req.Spec = "instance", spec + return req, notice, nil +} + +func refuseFlags(changed func(string) bool, names []string, onlyFor string) error { + for _, name := range names { + if changed(name) { + return fmt.Errorf("--%s only applies to %s", name, onlyFor) + } + } + return nil +} + +func newScheduleCreateCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "create (--at