Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .github/file-size-budget.txt
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ free/github-runner/third_party/sdk/identity.go 338
free/idme/internal/handlers.go 536
free/invitations/third_party/sdk/devkit/cmd/new-plugin/main.go 510
free/invitations/third_party/sdk/identity.go 338
free/job-queue/go/api/handlers.go 356
free/link-preview/third_party/sdk/devkit/cmd/new-plugin/main.go 510
free/link-preview/third_party/sdk/identity.go 338
free/linkedin/internal/handlers.go 346
Expand Down
3 changes: 3 additions & 0 deletions .github/gitleaks.toml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,9 @@ regexes = [
'''^PKCS8-wrapped$''',
# free/ci/test/security/redteam/c_pairing_test.go (P7-TRUST-18): NodeKeyID fixture of the pairing prompt test, a counting sequence (history commits carry it; later ones use node-key-1)
'''^0123456789abcdef$''',
# free/ci/internal/roles/signin_test.go (D-1276): the fake token prefix the rate-limit test guesses with
# (a counting sequence, the whole matched value); present in branch history that the CI fetch scans
'''^guess-0123456789$''',
'''wJalrXUtnFEMI''',
'''bPxRfiCYEXAMPLEKEY''',

Expand Down
6 changes: 3 additions & 3 deletions free/ci/internal/exec/container_facts_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func TestExit137IsVerifiedAgainstTheRuntime(t *testing.T) {
t.Run(name, func(t *testing.T) {
fakeRuntime(t, "", "137")
fakeEvents(t, tc.events...)
res := (&Executor{}).Run(context.Background(), plannedSpec(t, "plan-137-"+name, hardPlanner{}))
res := (&Executor{Isolation: BaselineIsolator{}}).Run(context.Background(), plannedSpec(t, "plan-137-"+name, hardPlanner{}))
if res.Reason != tc.reason {
t.Fatalf("reason %q want %q (%+v)", res.Reason, tc.reason, res)
}
Expand All @@ -48,15 +48,15 @@ func TestExit125IsVerifiedAgainstTheRuntime(t *testing.T) {
t.Run("jobs own exit", func(t *testing.T) {
fakeRuntime(t, "", "125")
fakeEvents(t, "create", "start", "die")
res := (&Executor{}).Run(context.Background(), plannedSpec(t, "plan-125-job", hardPlanner{}))
res := (&Executor{Isolation: BaselineIsolator{}}).Run(context.Background(), plannedSpec(t, "plan-125-job", hardPlanner{}))
if res.Err != nil || res.ExitCode != 125 || res.Attempt.FailureClass != "code" {
t.Fatalf("a job that ran and exited 125 is a code failure, not a runtime one: %+v", res)
}
})
t.Run("runtime failed", func(t *testing.T) {
fakeRuntime(t, "", "125")
fakeEvents(t, "create", "destroy") // created, never started
res := (&Executor{}).Run(context.Background(), plannedSpec(t, "plan-125-rt", hardPlanner{}))
res := (&Executor{Isolation: BaselineIsolator{}}).Run(context.Background(), plannedSpec(t, "plan-125-rt", hardPlanner{}))
var e *Error
if !errors.As(res.Err, &e) || e.Code != "E613" {
t.Fatalf("a container that never started is an E613 runtime failure: %+v", res)
Expand Down
141 changes: 0 additions & 141 deletions free/job-queue/go/api/handlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"encoding/json"
"fmt"
"log/slog"
"math"
"net/http"
"strings"
"sync"
Expand Down Expand Up @@ -78,146 +77,6 @@ func (s *Server) routes() {
s.mux.HandleFunc("/metrics", s.handleMetrics)
}

// StartWorkers launches one goroutine per queue × concurrency worker.
func (s *Server) StartWorkers(ctx context.Context) error {
for _, q := range s.cfg.Queues {
for i := 0; i < s.cfg.Concurrency; i++ {
go s.worker(ctx, q)
}
s.log.Info("workers started", "queue", q, "concurrency", s.cfg.Concurrency)
}
return nil
}

// worker blocks on BRPOPLPUSH, processes each job, and updates np_jobqueue_jobs status.
func (s *Server) worker(ctx context.Context, queue string) {
qKey := "queue:" + queue
processingKey := "queue:" + queue + ":processing"
for {
select {
case <-ctx.Done():
return
default:
}

// BRPOPLPUSH: atomically move job from tail of qKey to head of processingKey.
result, err := s.rdb.BRPopLPush(ctx, qKey, processingKey, 5*time.Second).Result()
if err == redis.Nil {
continue
}
if err != nil {
if ctx.Err() != nil {
return
}
s.log.Error("BRPOPLPUSH error", "queue", queue, "err", err)
time.Sleep(time.Second)
continue
}

s.processJob(ctx, queue, processingKey, result)
}
}

type jobPayload struct {
ID string `json:"id"`
JobType string `json:"job_type"`
Payload json.RawMessage `json:"payload"`
AttemptCount int `json:"attempt_count"`
}

func (s *Server) processJob(ctx context.Context, queue, processingKey, raw string) {
var job jobPayload
if err := json.Unmarshal([]byte(raw), &job); err != nil {
s.log.Error("invalid job payload", "raw", raw, "err", err)
s.rdb.LRem(ctx, processingKey, 1, raw)
return
}

s.log.Info("processing job", "id", job.ID, "type", job.JobType, "queue", queue, "attempt", job.AttemptCount+1)
s.updateJobStatus(ctx, job.ID, "active", "")

// Dispatch to registered handlers (stub: log only — real handlers registered via SDK).
err := s.dispatch(ctx, job.JobType, job.Payload)

if err != nil {
job.AttemptCount++
if job.AttemptCount >= s.cfg.MaxAttempts {
s.log.Warn("job exhausted retries, moving to DLQ", "id", job.ID, "attempts", job.AttemptCount)
s.updateJobStatus(ctx, job.ID, "dead", err.Error())
s.moveToDLQ(ctx, job.ID, err.Error())
} else {
delay := retryDelay(job.AttemptCount)
s.log.Info("job failed, scheduling retry", "id", job.ID, "delay", delay, "attempt", job.AttemptCount)
s.updateJobStatus(ctx, job.ID, "pending", err.Error())
// Preserve the incremented attempt count in the Redis execution payload.
retryRaw, _ := json.Marshal(job)
// Re-enqueue after delay
go func(raw string, delay time.Duration) {
time.Sleep(delay)
s.rdb.LPush(ctx, "queue:"+queue, raw)
}(string(retryRaw), delay)
}
} else {
s.updateJobStatus(ctx, job.ID, "completed", "")
}

// Remove from processing set regardless of outcome.
s.rdb.LRem(ctx, processingKey, 1, raw)
}

// dispatch calls the registered handler for jobType.
// Returns an error for unknown job types so jobs fail fast and route through
// the retry/DLQ path instead of being silently marked completed.
// Wire real handlers here or use the SDK RegisterHandler bridge once available.
func (s *Server) dispatch(_ context.Context, jobType string, _ json.RawMessage) error {
s.log.Warn("no handler registered for job type; failing job so it routes to retry/DLQ",
"job_type", jobType)
return fmt.Errorf("no handler registered for job_type %q", jobType)
}

// retryDelay returns exponential backoff: 2^attempt seconds, capped at 1h.
func retryDelay(attempt int) time.Duration {
secs := math.Pow(2, float64(attempt))
if secs > 3600 {
secs = 3600
}
return time.Duration(secs) * time.Second
}

func (s *Server) updateJobStatus(ctx context.Context, id, status, errMsg string) {
if s.db == nil {
return
}
now := time.Now().UTC()
var q string
var args []interface{}
switch status {
case "active":
q = `UPDATE public.np_jobqueue_jobs SET status=$1, started_at=$2, attempt_count=attempt_count+1 WHERE id=$3 AND source_account_id='primary'`
args = []interface{}{status, now, id}
case "completed":
q = `UPDATE public.np_jobqueue_jobs SET status=$1, completed_at=$2 WHERE id=$3 AND source_account_id='primary'`
args = []interface{}{status, now, id}
default:
q = `UPDATE public.np_jobqueue_jobs SET status=$1, error=$2 WHERE id=$3 AND source_account_id='primary'`
args = []interface{}{status, nullStr(errMsg), id}
}
if _, err := s.db.Exec(ctx, q, args...); err != nil {
s.log.Warn("failed to update job status in Postgres", "id", id, "err", err)
}
}

func (s *Server) moveToDLQ(ctx context.Context, jobID, finalError string) {
if s.db == nil {
return
}
_, err := s.db.Exec(ctx,
`INSERT INTO public.np_job_dlq (job_id, final_error) SELECT id, $2 FROM public.np_jobqueue_jobs WHERE id=$1 AND source_account_id='primary'`, jobID, finalError)
if err != nil {
s.log.Warn("failed to insert into np_job_dlq", "job_id", jobID, "err", err)
}
}

// HTTP handlers (handleHealth lives in health.go)

func (s *Server) handleEnqueue(w http.ResponseWriter, r *http.Request) {
Expand Down
151 changes: 151 additions & 0 deletions free/job-queue/go/api/worker.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
package api

import (
"context"
"encoding/json"
"fmt"
"math"
"time"

"github.com/redis/go-redis/v9"
)

// StartWorkers launches one goroutine per queue × concurrency worker.
func (s *Server) StartWorkers(ctx context.Context) error {
for _, q := range s.cfg.Queues {
for i := 0; i < s.cfg.Concurrency; i++ {
go s.worker(ctx, q)
}
s.log.Info("workers started", "queue", q, "concurrency", s.cfg.Concurrency)
}
return nil
}

// worker blocks on BRPOPLPUSH, processes each job, and updates np_jobqueue_jobs status.
func (s *Server) worker(ctx context.Context, queue string) {
qKey := "queue:" + queue
processingKey := "queue:" + queue + ":processing"
for {
select {
case <-ctx.Done():
return
default:
}

// BRPOPLPUSH: atomically move job from tail of qKey to head of processingKey.
result, err := s.rdb.BRPopLPush(ctx, qKey, processingKey, 5*time.Second).Result()
if err == redis.Nil {
continue
}
if err != nil {
if ctx.Err() != nil {
return
}
s.log.Error("BRPOPLPUSH error", "queue", queue, "err", err)
time.Sleep(time.Second)
continue
}

s.processJob(ctx, queue, processingKey, result)
}
}

type jobPayload struct {
ID string `json:"id"`
JobType string `json:"job_type"`
Payload json.RawMessage `json:"payload"`
AttemptCount int `json:"attempt_count"`
}

func (s *Server) processJob(ctx context.Context, queue, processingKey, raw string) {
var job jobPayload
if err := json.Unmarshal([]byte(raw), &job); err != nil {
s.log.Error("invalid job payload", "raw", raw, "err", err)
s.rdb.LRem(ctx, processingKey, 1, raw)
return
}

s.log.Info("processing job", "id", job.ID, "type", job.JobType, "queue", queue, "attempt", job.AttemptCount+1)
s.updateJobStatus(ctx, job.ID, "active", "")

// Dispatch to registered handlers (stub: log only — real handlers registered via SDK).
err := s.dispatch(ctx, job.JobType, job.Payload)

if err != nil {
job.AttemptCount++
if job.AttemptCount >= s.cfg.MaxAttempts {
s.log.Warn("job exhausted retries, moving to DLQ", "id", job.ID, "attempts", job.AttemptCount)
s.updateJobStatus(ctx, job.ID, "dead", err.Error())
s.moveToDLQ(ctx, job.ID, err.Error())
} else {
delay := retryDelay(job.AttemptCount)
s.log.Info("job failed, scheduling retry", "id", job.ID, "delay", delay, "attempt", job.AttemptCount)
s.updateJobStatus(ctx, job.ID, "pending", err.Error())
// Preserve the incremented attempt count in the Redis execution payload.
retryRaw, _ := json.Marshal(job)
// Re-enqueue after delay
go func(raw string, delay time.Duration) {
time.Sleep(delay)
s.rdb.LPush(ctx, "queue:"+queue, raw)
}(string(retryRaw), delay)
}
} else {
s.updateJobStatus(ctx, job.ID, "completed", "")
}

// Remove from processing set regardless of outcome.
s.rdb.LRem(ctx, processingKey, 1, raw)
}

// dispatch calls the registered handler for jobType.
// Returns an error for unknown job types so jobs fail fast and route through
// the retry/DLQ path instead of being silently marked completed.
// Wire real handlers here or use the SDK RegisterHandler bridge once available.
func (s *Server) dispatch(_ context.Context, jobType string, _ json.RawMessage) error {
s.log.Warn("no handler registered for job type; failing job so it routes to retry/DLQ",
"job_type", jobType)
return fmt.Errorf("no handler registered for job_type %q", jobType)
}

// retryDelay returns exponential backoff: 2^attempt seconds, capped at 1h.
func retryDelay(attempt int) time.Duration {
secs := math.Pow(2, float64(attempt))
if secs > 3600 {
secs = 3600
}
return time.Duration(secs) * time.Second
}

func (s *Server) updateJobStatus(ctx context.Context, id, status, errMsg string) {
if s.db == nil {
return
}
now := time.Now().UTC()
var q string
var args []interface{}
switch status {
case "active":
q = `UPDATE public.np_jobqueue_jobs SET status=$1, started_at=$2, attempt_count=attempt_count+1 WHERE id=$3 AND source_account_id='primary'`
args = []interface{}{status, now, id}
case "completed":
q = `UPDATE public.np_jobqueue_jobs SET status=$1, completed_at=$2 WHERE id=$3 AND source_account_id='primary'`
args = []interface{}{status, now, id}
default:
q = `UPDATE public.np_jobqueue_jobs SET status=$1, error=$2 WHERE id=$3 AND source_account_id='primary'`
args = []interface{}{status, nullStr(errMsg), id}
}
if _, err := s.db.Exec(ctx, q, args...); err != nil {
s.log.Warn("failed to update job status in Postgres", "id", id, "err", err)
}
}

func (s *Server) moveToDLQ(ctx context.Context, jobID, finalError string) {
if s.db == nil {
return
}
_, err := s.db.Exec(ctx,
`INSERT INTO public.np_job_dlq (job_id, final_error) SELECT id, $2 FROM public.np_jobqueue_jobs WHERE id=$1 AND source_account_id='primary'`, jobID, finalError)
if err != nil {
s.log.Warn("failed to insert into np_job_dlq", "job_id", jobID, "err", err)
}
}
2 changes: 1 addition & 1 deletion registry.json
Original file line number Diff line number Diff line change
Expand Up @@ -7976,7 +7976,7 @@
"minNselfVersion": "1.0.0",
"min_nself_version": "1.0.0",
"name": "workflows",
"port": 3733,
"port": 3712,
"releaseTag": "v1.2.13",
"release_signature": {
"alg": "ed25519",
Expand Down
Loading