Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions docs/EVENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ events and share delivery logging and deduplication when a delivery ID is given.

## Actions runners

Install the GitHub App on the repository with Administration write permission
Install the GitHub App on the repository with Administration write and Actions read permissions
and subscribe its webhook to workflow jobs. Keep its private key readable only
by the agent. The delivery's installation ID selects the installation token.
The token is restricted to the delivery's repository.
Expand All @@ -77,8 +77,11 @@ The runner name is unique. JIT labels come from the subscription and the work fo
or the delivery log. The run receives JIT configuration only as secret env
`JIT`; its command is `sh -c 'exec ./run.sh --jitconfig "$JIT"'` in
`/home/runner`, with `RUNNER_ALLOW_RUNASROOT=1` and egress enabled.
Completed and cancelled job events require no cleanup action: the runner exits
with the job.
When GitHub assigns or finishes a job, FlatRun cancels pending runner requests
for that job on the same webhook endpoint and repository. Before starting a
runner, FlatRun checks that GitHub still lists the job as queued. This also
covers missed or out-of-order deliveries. A failed lookup marks the local run
as failed. Once a runner starts, GitHub controls its assigned job.

Image defaults to `ghcr.io/actions/actions-runner`, timeout to 600 seconds
(maximum 3600), and runner group to 1. Set runtime to `firecracker` or `docker`.
Expand Down
35 changes: 27 additions & 8 deletions internal/api/event_delivery.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (

"github.com/flatrun/agent/internal/auth"
"github.com/flatrun/agent/internal/events"
"github.com/flatrun/agent/internal/runs"
"github.com/flatrun/agent/internal/webhook"
"github.com/flatrun/agent/pkg/config"
"github.com/gin-gonic/gin"
Expand Down Expand Up @@ -84,6 +85,16 @@ func (s *Server) receiveEvent(c *gin.Context) {
for name, result := range record.Subscriptions {
outcomes[name] = result
}
if s.runsManager != nil && (event.Type == "github.workflow_job.completed" || event.Type == "github.workflow_job.in_progress") {
count, err := s.runsManager.CancelPending(runnerJobKey(event))
if err != nil {
state.finish(record, "failed", outcomes)
return
}
if count > 0 {
outcome = "succeeded"
}
}
for _, subscription := range s.config.Events.Subscriptions {
if len(record.Subscriptions) > 0 && record.Subscriptions[subscription.Name] == "" {
continue
Expand Down Expand Up @@ -116,6 +127,10 @@ func (s *Server) receiveEvent(c *gin.Context) {
c.JSON(http.StatusAccepted, gin.H{"status": "accepted"})
}

func runnerJobKey(e events.Event) string {
return strings.Join([]string{e.Source, strings.ToLower(e.Attribute("subject")), e.Attribute("job_id")}, "\x00")
}

func (s *Server) actOnEvent(subscription config.EventSubscription, e events.Event, parsed *webhook.Event) error {
switch subscription.Action {
case "run", "github_runner":
Expand All @@ -136,8 +151,7 @@ func (s *Server) actOnEvent(subscription config.EventSubscription, e events.Even
spec.Env["FLATRUN_EVENT_TYPE"] = e.Type
spec.Env["FLATRUN_EVENT_SUBJECT"] = e.Attribute("subject")

secrets := map[string]string{}
defer clear(secrets)
var preparation runs.Preparation
if subscription.Action == "github_runner" {
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
Expand All @@ -149,11 +163,16 @@ func (s *Server) actOnEvent(subscription config.EventSubscription, e events.Even
log.Printf("Event runner delivery failed: runtime %s unavailable: %v", spec.Runtime, err)
return fmt.Errorf("runner runtime %s unavailable: %w", spec.Runtime, err)
}
jit, err := app.JIT(ctx, e, subscription.GitHubRunner.RunnerGroupID, subscription.Labels)
if err != nil {
return err
}
secrets["JIT"] = jit
preparation = runs.Preparation{Key: runnerJobKey(e), Apply: func(ctx context.Context, secrets map[string]string) (bool, error) {
ctx, cancel := context.WithTimeout(ctx, 45*time.Second)
defer cancel()
jit, err := app.JIT(ctx, e, subscription.GitHubRunner.RunnerGroupID, subscription.Labels)
if err != nil || jit == "" {
return false, err
}
secrets["JIT"] = jit
return true, nil
}}
spec.Command = []string{"sh", "-c", `exec ./run.sh --jitconfig "$JIT"`}
spec.WorkDir = subscription.GitHubRunner.WorkDir
if spec.WorkDir == "" {
Expand All @@ -171,7 +190,7 @@ func (s *Server) actOnEvent(subscription config.EventSubscription, e events.Even
}
}
}
_, err := s.runsManager.Submit(0, "events", spec, secrets)
_, err := s.runsManager.SubmitPrepared(0, "events", spec, nil, preparation)
return err
case "deploy", "preview":
deployment, err := s.manager.GetDeployment(subscription.Deployment)
Expand Down
86 changes: 84 additions & 2 deletions internal/api/event_delivery_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,12 +81,39 @@ func TestEventHTTPRunner(t *testing.T) {
release := make(chan struct{})
started := make(chan struct{}, 1)
var apiCalls atomic.Int32
var completed atomic.Bool
var blockLookup atomic.Bool
lookupStarted := make(chan struct{})
resume := make(chan struct{})
defer close(resume)
github := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
apiCalls.Add(1)
if r.URL.Path == "/app/installations/3456996/access_tokens" {
started <- struct{}{}
select {
case started <- struct{}{}:
default:
}
<-release
}
if r.URL.Path == "/repos/Codertocat/Hello-World/actions/jobs/289782451" {
if r.Method != http.MethodGet || r.Header.Get("Authorization") != "Bearer ghs_16C7e42F292c6912E7710c838347Ae178B4a" {
t.Error("invalid job lookup")
}
if blockLookup.Load() {
close(lookupStarted)
select {
case <-r.Context().Done():
return
case <-resume:
}
}
if completed.Load() {
_, _ = io.WriteString(w, `{"status":"completed"}`)
} else {
_, _ = io.WriteString(w, `{"status":"queued"}`)
}
return
}
if r.Method != "POST" || r.Header.Get("Accept") != "application/vnd.github+json" {
t.Error("invalid API request")
}
Expand Down Expand Up @@ -222,7 +249,7 @@ func TestEventHTTPRunner(t *testing.T) {
if status := deliver("in_progress.payload.json", "in_progress.payload.json", "test-secret"); status != 200 {
t.Fatalf("ignored duplicate: %d", status)
}
if apiCalls.Load() != 2 {
if apiCalls.Load() != 3 {
t.Fatalf("API calls: %d", apiCalls.Load())
}
deadline := time.Now().Add(5 * time.Second)
Expand Down Expand Up @@ -259,6 +286,61 @@ func TestEventHTTPRunner(t *testing.T) {
t.Fatal("extra run")
default:
}
waitCancelled := func(want int) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
records, err := manager.Store.List(nil, 10)
if err != nil {
t.Fatal(err)
}
count := 0
for _, r := range records {
if r.Status == runs.StatusCancelled {
count++
}
}
if count == want {
return
}
time.Sleep(10 * time.Millisecond)
}
t.Fatalf("expected %d cancelled runner requests", want)
}
completed.Store(true)
if deliver("late-queued", "queued.payload.json", "test-secret") != 202 {
t.Fatal("late delivery rejected")
}
s.deliveries.wg.Wait()
waitCancelled(1)
if apiCalls.Load() != 5 {
t.Fatal("finished job requested a runner")
}

blockLookup.Store(true)
if deliver("preparing", "queued.payload.json", "test-secret") != 202 {
t.Fatal("queue failed")
}
select {
case <-lookupStarted:
case <-time.After(5 * time.Second):
t.Fatal("lookup did not start")
}
if deliver("waiting", "queued.payload.json", "test-secret") != 202 {
t.Fatal("queue failed")
}
s.deliveries.wg.Wait()
if deliver("finished", "completed.success.with-organization.payload.json", "test-secret") != 202 {
t.Fatal("completion rejected")
}
s.deliveries.wg.Wait()
waitCancelled(3)
select {
case <-rt.specs:
t.Fatal("started an obsolete runner")
default:
}

}

func TestEventReadHTTPAuthorization(t *testing.T) {
Expand Down
73 changes: 62 additions & 11 deletions internal/runs/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,13 @@ var knownRuntimeTypes = []runtimes.Type{runtimes.TypeDocker, runtimes.TypeFirecr

var ErrClosed = errors.New("runs manager is closed")

type Preparation struct {
Key string
Apply func(context.Context, map[string]string) (bool, error)
}

type pending struct {
preparation Preparation
reservation uint64
run Run
secrets map[string]string
Expand Down Expand Up @@ -125,6 +131,10 @@ func NewManagerWithMemoryBudget(root string, runtime runtimes.Runtime, workers i
}

func (m *Manager) Submit(owner int64, username string, spec Spec, secrets map[string]string) (*Run, error) {
return m.SubmitPrepared(owner, username, spec, secrets, Preparation{})
}

func (m *Manager) SubmitPrepared(owner int64, username string, spec Spec, secrets map[string]string, preparation Preparation) (*Run, error) {
if err := spec.Validate(); err != nil {
return nil, err
}
Expand Down Expand Up @@ -172,29 +182,53 @@ func (m *Manager) Submit(owner int64, username string, spec Spec, secrets map[st
return nil, err
}
ctx, cancel := context.WithCancel(context.Background())
queued := &pending{run: run, secrets: secretValues, ctx: ctx, cancel: cancel}
queued := &pending{preparation: preparation, run: run, secrets: secretValues, ctx: ctx, cancel: cancel}
m.active[run.ID] = queued
m.queue = append(m.queue, queued)
m.ready.Signal()
return &run, nil
}

func (m *Manager) CancelPending(key string) (int, error) {
m.mu.Lock()
defer m.mu.Unlock()
if key == "" {
return 0, nil
}
count := 0
for _, job := range m.active {
if job.preparation.Key == key {
if err := m.cancelLocked(job); err != nil {
return count, err
}
count++
}
}
return count, nil
}

func (m *Manager) Cancel(id string) error {
m.mu.Lock()
defer m.mu.Unlock()
queued := m.active[id]
if queued == nil {
return nil
if job := m.active[id]; job != nil {
return m.cancelLocked(job)
}
queued.cancel()
return nil
}

func (m *Manager) cancelLocked(job *pending) error {
job.cancel()
m.ready.Broadcast()
if queued.run.Status == StatusQueued {
clear(queued.secrets)
queued.secrets = nil
queued.run.Status = StatusCancelled
if index := slices.Index(m.queue, job); index >= 0 {
job.run.Status = StatusCancelled
now := time.Now().UTC()
queued.run.FinishedAt = &now
return m.Store.Save(&queued.run)
job.run.FinishedAt = &now
if err := m.Store.Save(&job.run); err != nil {
return err
}
clear(job.secrets)
m.queue = slices.Delete(m.queue, index, index+1)
delete(m.active, job.run.ID)
}
return nil
}
Expand Down Expand Up @@ -311,6 +345,23 @@ func (m *Manager) execute(queued *pending) {
}()
defer m.finishRun(&run)

if prepare := queued.preparation.Apply; prepare != nil {
needed, err := prepare(ctx, queued.secrets)
m.mu.Lock()
queued.preparation.Key = ""

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Clearing the key when JIT preparation returns makes this run invisible to cancellation while the worker is still setting up the runtime. If an in_progress or completed webhook arrives in that gap, the stale runner request can still start and consume a worker; keep it cancellable until startup or perform a final status check immediately before launch.

cancelled := queued.ctx.Err() != nil
m.mu.Unlock()
switch {
case cancelled || (err == nil && !needed):
run.Status = StatusCancelled
return
case err != nil:
run.Status = StatusFailed
run.Error = err.Error()
return
}
}

artifactDirectory := filepath.Join(m.root, run.ID, "artifacts")
if err := os.MkdirAll(artifactDirectory, 0700); err != nil {
run.Status = StatusFailed
Expand Down
Loading