diff --git a/docs/EVENTS.md b/docs/EVENTS.md index 0dbeffe..6656c9c 100644 --- a/docs/EVENTS.md +++ b/docs/EVENTS.md @@ -65,7 +65,7 @@ events and share delivery logging and deduplication when a delivery ID is given. ## Actions runners -Install the GitHub App on the repository with Administration write permission +Install the GitHub App on the repository with Administration write and Actions read permissions and subscribe its webhook to workflow jobs. Keep its private key readable only by the agent. The delivery's installation ID selects the installation token. The token is restricted to the delivery's repository. @@ -77,8 +77,11 @@ The runner name is unique. JIT labels come from the subscription and the work fo or the delivery log. The run receives JIT configuration only as secret env `JIT`; its command is `sh -c 'exec ./run.sh --jitconfig "$JIT"'` in `/home/runner`, with `RUNNER_ALLOW_RUNASROOT=1` and egress enabled. -Completed and cancelled job events require no cleanup action: the runner exits -with the job. +When GitHub assigns or finishes a job, FlatRun cancels pending runner requests +for that job on the same webhook endpoint and repository. Before starting a +runner, FlatRun checks that GitHub still lists the job as queued. This also +covers missed or out-of-order deliveries. A failed lookup marks the local run +as failed. Once a runner starts, GitHub controls its assigned job. Image defaults to `ghcr.io/actions/actions-runner`, timeout to 600 seconds (maximum 3600), and runner group to 1. Set runtime to `firecracker` or `docker`. diff --git a/internal/api/event_delivery.go b/internal/api/event_delivery.go index 4229cb1..a6de3b8 100644 --- a/internal/api/event_delivery.go +++ b/internal/api/event_delivery.go @@ -15,6 +15,7 @@ import ( "github.com/flatrun/agent/internal/auth" "github.com/flatrun/agent/internal/events" + "github.com/flatrun/agent/internal/runs" "github.com/flatrun/agent/internal/webhook" "github.com/flatrun/agent/pkg/config" "github.com/gin-gonic/gin" @@ -84,6 +85,16 @@ func (s *Server) receiveEvent(c *gin.Context) { for name, result := range record.Subscriptions { outcomes[name] = result } + if s.runsManager != nil && (event.Type == "github.workflow_job.completed" || event.Type == "github.workflow_job.in_progress") { + count, err := s.runsManager.CancelPending(runnerJobKey(event)) + if err != nil { + state.finish(record, "failed", outcomes) + return + } + if count > 0 { + outcome = "succeeded" + } + } for _, subscription := range s.config.Events.Subscriptions { if len(record.Subscriptions) > 0 && record.Subscriptions[subscription.Name] == "" { continue @@ -116,6 +127,10 @@ func (s *Server) receiveEvent(c *gin.Context) { c.JSON(http.StatusAccepted, gin.H{"status": "accepted"}) } +func runnerJobKey(e events.Event) string { + return strings.Join([]string{e.Source, strings.ToLower(e.Attribute("subject")), e.Attribute("job_id")}, "\x00") +} + func (s *Server) actOnEvent(subscription config.EventSubscription, e events.Event, parsed *webhook.Event) error { switch subscription.Action { case "run", "github_runner": @@ -136,8 +151,7 @@ func (s *Server) actOnEvent(subscription config.EventSubscription, e events.Even spec.Env["FLATRUN_EVENT_TYPE"] = e.Type spec.Env["FLATRUN_EVENT_SUBJECT"] = e.Attribute("subject") - secrets := map[string]string{} - defer clear(secrets) + var preparation runs.Preparation if subscription.Action == "github_runner" { ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) defer cancel() @@ -149,11 +163,16 @@ func (s *Server) actOnEvent(subscription config.EventSubscription, e events.Even log.Printf("Event runner delivery failed: runtime %s unavailable: %v", spec.Runtime, err) return fmt.Errorf("runner runtime %s unavailable: %w", spec.Runtime, err) } - jit, err := app.JIT(ctx, e, subscription.GitHubRunner.RunnerGroupID, subscription.Labels) - if err != nil { - return err - } - secrets["JIT"] = jit + preparation = runs.Preparation{Key: runnerJobKey(e), Apply: func(ctx context.Context, secrets map[string]string) (bool, error) { + ctx, cancel := context.WithTimeout(ctx, 45*time.Second) + defer cancel() + jit, err := app.JIT(ctx, e, subscription.GitHubRunner.RunnerGroupID, subscription.Labels) + if err != nil || jit == "" { + return false, err + } + secrets["JIT"] = jit + return true, nil + }} spec.Command = []string{"sh", "-c", `exec ./run.sh --jitconfig "$JIT"`} spec.WorkDir = subscription.GitHubRunner.WorkDir if spec.WorkDir == "" { @@ -171,7 +190,7 @@ func (s *Server) actOnEvent(subscription config.EventSubscription, e events.Even } } } - _, err := s.runsManager.Submit(0, "events", spec, secrets) + _, err := s.runsManager.SubmitPrepared(0, "events", spec, nil, preparation) return err case "deploy", "preview": deployment, err := s.manager.GetDeployment(subscription.Deployment) diff --git a/internal/api/event_delivery_test.go b/internal/api/event_delivery_test.go index 69faa45..e9b21e3 100644 --- a/internal/api/event_delivery_test.go +++ b/internal/api/event_delivery_test.go @@ -81,12 +81,39 @@ func TestEventHTTPRunner(t *testing.T) { release := make(chan struct{}) started := make(chan struct{}, 1) var apiCalls atomic.Int32 + var completed atomic.Bool + var blockLookup atomic.Bool + lookupStarted := make(chan struct{}) + resume := make(chan struct{}) + defer close(resume) github := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { apiCalls.Add(1) if r.URL.Path == "/app/installations/3456996/access_tokens" { - started <- struct{}{} + select { + case started <- struct{}{}: + default: + } <-release } + if r.URL.Path == "/repos/Codertocat/Hello-World/actions/jobs/289782451" { + if r.Method != http.MethodGet || r.Header.Get("Authorization") != "Bearer ghs_16C7e42F292c6912E7710c838347Ae178B4a" { + t.Error("invalid job lookup") + } + if blockLookup.Load() { + close(lookupStarted) + select { + case <-r.Context().Done(): + return + case <-resume: + } + } + if completed.Load() { + _, _ = io.WriteString(w, `{"status":"completed"}`) + } else { + _, _ = io.WriteString(w, `{"status":"queued"}`) + } + return + } if r.Method != "POST" || r.Header.Get("Accept") != "application/vnd.github+json" { t.Error("invalid API request") } @@ -222,7 +249,7 @@ func TestEventHTTPRunner(t *testing.T) { if status := deliver("in_progress.payload.json", "in_progress.payload.json", "test-secret"); status != 200 { t.Fatalf("ignored duplicate: %d", status) } - if apiCalls.Load() != 2 { + if apiCalls.Load() != 3 { t.Fatalf("API calls: %d", apiCalls.Load()) } deadline := time.Now().Add(5 * time.Second) @@ -259,6 +286,61 @@ func TestEventHTTPRunner(t *testing.T) { t.Fatal("extra run") default: } + waitCancelled := func(want int) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + records, err := manager.Store.List(nil, 10) + if err != nil { + t.Fatal(err) + } + count := 0 + for _, r := range records { + if r.Status == runs.StatusCancelled { + count++ + } + } + if count == want { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("expected %d cancelled runner requests", want) + } + completed.Store(true) + if deliver("late-queued", "queued.payload.json", "test-secret") != 202 { + t.Fatal("late delivery rejected") + } + s.deliveries.wg.Wait() + waitCancelled(1) + if apiCalls.Load() != 5 { + t.Fatal("finished job requested a runner") + } + + blockLookup.Store(true) + if deliver("preparing", "queued.payload.json", "test-secret") != 202 { + t.Fatal("queue failed") + } + select { + case <-lookupStarted: + case <-time.After(5 * time.Second): + t.Fatal("lookup did not start") + } + if deliver("waiting", "queued.payload.json", "test-secret") != 202 { + t.Fatal("queue failed") + } + s.deliveries.wg.Wait() + if deliver("finished", "completed.success.with-organization.payload.json", "test-secret") != 202 { + t.Fatal("completion rejected") + } + s.deliveries.wg.Wait() + waitCancelled(3) + select { + case <-rt.specs: + t.Fatal("started an obsolete runner") + default: + } + } func TestEventReadHTTPAuthorization(t *testing.T) { diff --git a/internal/runs/manager.go b/internal/runs/manager.go index e2a2b4d..ee1ad81 100644 --- a/internal/runs/manager.go +++ b/internal/runs/manager.go @@ -26,7 +26,13 @@ var knownRuntimeTypes = []runtimes.Type{runtimes.TypeDocker, runtimes.TypeFirecr var ErrClosed = errors.New("runs manager is closed") +type Preparation struct { + Key string + Apply func(context.Context, map[string]string) (bool, error) +} + type pending struct { + preparation Preparation reservation uint64 run Run secrets map[string]string @@ -125,6 +131,10 @@ func NewManagerWithMemoryBudget(root string, runtime runtimes.Runtime, workers i } func (m *Manager) Submit(owner int64, username string, spec Spec, secrets map[string]string) (*Run, error) { + return m.SubmitPrepared(owner, username, spec, secrets, Preparation{}) +} + +func (m *Manager) SubmitPrepared(owner int64, username string, spec Spec, secrets map[string]string, preparation Preparation) (*Run, error) { if err := spec.Validate(); err != nil { return nil, err } @@ -172,29 +182,53 @@ func (m *Manager) Submit(owner int64, username string, spec Spec, secrets map[st return nil, err } ctx, cancel := context.WithCancel(context.Background()) - queued := &pending{run: run, secrets: secretValues, ctx: ctx, cancel: cancel} + queued := &pending{preparation: preparation, run: run, secrets: secretValues, ctx: ctx, cancel: cancel} m.active[run.ID] = queued m.queue = append(m.queue, queued) m.ready.Signal() return &run, nil } +func (m *Manager) CancelPending(key string) (int, error) { + m.mu.Lock() + defer m.mu.Unlock() + if key == "" { + return 0, nil + } + count := 0 + for _, job := range m.active { + if job.preparation.Key == key { + if err := m.cancelLocked(job); err != nil { + return count, err + } + count++ + } + } + return count, nil +} + func (m *Manager) Cancel(id string) error { m.mu.Lock() defer m.mu.Unlock() - queued := m.active[id] - if queued == nil { - return nil + if job := m.active[id]; job != nil { + return m.cancelLocked(job) } - queued.cancel() + return nil +} + +func (m *Manager) cancelLocked(job *pending) error { + job.cancel() m.ready.Broadcast() - if queued.run.Status == StatusQueued { - clear(queued.secrets) - queued.secrets = nil - queued.run.Status = StatusCancelled + if index := slices.Index(m.queue, job); index >= 0 { + job.run.Status = StatusCancelled now := time.Now().UTC() - queued.run.FinishedAt = &now - return m.Store.Save(&queued.run) + job.run.FinishedAt = &now + if err := m.Store.Save(&job.run); err != nil { + return err + } + clear(job.secrets) + m.queue = slices.Delete(m.queue, index, index+1) + delete(m.active, job.run.ID) } return nil } @@ -311,6 +345,23 @@ func (m *Manager) execute(queued *pending) { }() defer m.finishRun(&run) + if prepare := queued.preparation.Apply; prepare != nil { + needed, err := prepare(ctx, queued.secrets) + m.mu.Lock() + queued.preparation.Key = "" + cancelled := queued.ctx.Err() != nil + m.mu.Unlock() + switch { + case cancelled || (err == nil && !needed): + run.Status = StatusCancelled + return + case err != nil: + run.Status = StatusFailed + run.Error = err.Error() + return + } + } + artifactDirectory := filepath.Join(m.root, run.ID, "artifacts") if err := os.MkdirAll(artifactDirectory, 0700); err != nil { run.Status = StatusFailed diff --git a/internal/webhook/jit.go b/internal/webhook/jit.go index 41d64f6..5549e65 100644 --- a/internal/webhook/jit.go +++ b/internal/webhook/jit.go @@ -10,6 +10,7 @@ import ( "net/http" "net/url" "os" + "strconv" "strings" "time" @@ -46,7 +47,7 @@ func (a GitHubApp) JIT(ctx context.Context, e events.Event, group int64, labels var installation struct { Token string `json:"token"` } - if err := a.post(ctx, fmt.Sprintf("/app/installations/%d/access_tokens", installationID), token, map[string]any{"repositories": []string{strings.Split(e.Attribute("subject"), "/")[1]}}, &installation); err != nil { + if err := a.request(ctx, http.MethodPost, fmt.Sprintf("/app/installations/%d/access_tokens", installationID), token, map[string]any{"repositories": []string{strings.Split(e.Attribute("subject"), "/")[1]}}, &installation, http.StatusCreated); err != nil { return "", err } if installation.Token == "" { @@ -56,7 +57,25 @@ func (a GitHubApp) JIT(ctx context.Context, e events.Event, group int64, labels Encoded string `json:"encoded_jit_config"` } parts := strings.Split(e.Attribute("subject"), "/") - err = a.post(ctx, "/repos/"+url.PathEscape(parts[0])+"/"+url.PathEscape(parts[1])+"/actions/runners/generate-jitconfig", installation.Token, map[string]any{"name": "flatrun-" + uuid.NewString(), "runner_group_id": group, "labels": labels, "work_folder": "_work"}, &config) + jobID, err := strconv.ParseInt(e.Attribute("job_id"), 10, 64) + if err != nil || jobID <= 0 { + return "", errors.New("invalid workflow job ID") + } + repo := "/repos/" + url.PathEscape(parts[0]) + "/" + url.PathEscape(parts[1]) + "/actions" + var job struct { + Status string `json:"status"` + } + if err := a.request(ctx, http.MethodGet, repo+"/jobs/"+strconv.FormatInt(jobID, 10), installation.Token, nil, &job, http.StatusOK); err != nil { + return "", err + } + switch job.Status { + case "in_progress", "completed": + return "", nil + case "queued": + default: + return "", fmt.Errorf("unexpected workflow job status %q", job.Status) + } + err = a.request(ctx, http.MethodPost, repo+"/runners/generate-jitconfig", installation.Token, map[string]any{"name": "flatrun-" + uuid.NewString(), "runner_group_id": group, "labels": labels, "work_folder": "_work"}, &config, http.StatusCreated) if err != nil { return "", err } @@ -66,16 +85,20 @@ func (a GitHubApp) JIT(ctx context.Context, e events.Event, group int64, labels return config.Encoded, nil } -func (a GitHubApp) post(ctx context.Context, path, token string, body, output any) error { - raw, err := json.Marshal(body) - if err != nil { - return errors.New("invalid GitHub request") +func (a GitHubApp) request(ctx context.Context, method, path, token string, body, output any, expected int) error { + var reader io.Reader + if body != nil { + raw, err := json.Marshal(body) + if err != nil { + return errors.New("invalid GitHub request") + } + reader = bytes.NewReader(raw) } base := a.BaseURL if base == "" { base = "https://api.github.com" } - request, err := http.NewRequestWithContext(ctx, http.MethodPost, base+path, bytes.NewReader(raw)) + request, err := http.NewRequestWithContext(ctx, method, base+path, reader) if err != nil { return errors.New("invalid GitHub API URL") } @@ -92,7 +115,7 @@ func (a GitHubApp) post(ctx context.Context, path, token string, body, output an return errors.New("GitHub API request failed") } defer response.Body.Close() - if response.StatusCode != http.StatusCreated { + if response.StatusCode != expected { return fmt.Errorf("GitHub API returned %d", response.StatusCode) } if json.NewDecoder(io.LimitReader(response.Body, 2<<20)).Decode(output) != nil {