Skip to content

feat(ids): use queue-scoped decimal resource IDs #770

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
behinddwalls wants to merge 4 commits into from
Open

feat(ids): use queue-scoped decimal resource IDs #770

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5b79c54
feat(ids): use queue-scoped decimal resource IDs
behinddwalls Oct 2, 2026
2192f75
fix(tests): scope resource ID fixtures and queries by queue
behinddwalls Oct 2, 2026
857b445
refactor(ids): simplify shared helpers and counter scope
behinddwalls Oct 2, 2026
12fca80
refactor(counter): keep domain as the sequence name
behinddwalls Oct 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,10 @@ local-init-runway-queue-schema: ## Apply queue schema only (mysql-queue) for Run

local-init-stovepipe-schemas: ## Apply storage (mysql-app) and queue (mysql-queue) schemas for Stovepipe compose stacks
@echo "Applying storage schema to mysql-app..."
@for file in platform/extension/counter/mysql/schema/*.sql; do \
echo " - Applying $$(basename $$file)..."; \
docker exec -i $(STOVEPIPE_LOCAL_PROJECT)-mysql-app-1 mysql -uroot -proot submitqueue < $$file 2>&1 | grep -v "Using a password" || true; \
done
@for file in stovepipe/extension/storage/mysql/schema/*.sql; do \
echo " - Applying $$(basename $$file)..."; \
docker exec -i $(STOVEPIPE_LOCAL_PROJECT)-mysql-app-1 mysql -uroot -proot submitqueue < $$file 2>&1 | grep -v "Using a password" || true; \
Expand Down
4 changes: 2 additions & 2 deletions api/base/hook/hook_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func TestHookEventRoundTrip(t *testing.T) {
Source: "runway",
Type: "merge.completed",
TimestampMs: 1722800012345,
Payload: mustStruct(t, map[string]any{"request_id": "queue-a/42"}),
Payload: mustStruct(t, map[string]any{"request_id": "42"}),
},
"envelope only": {
Id: "stovepipe/commit.green/git-abc/1",
Expand Down Expand Up @@ -136,7 +136,7 @@ func TestEventIDIsDerived(t *testing.T) {
"unversioned": NewUnversionedEventID("submitqueue", "batch.failed", "batch-778", "msg-1", 0),
"second ordinal": NewUnversionedEventID("submitqueue", "batch.failed", "batch-778", "msg-1", 1),
"other cause": NewUnversionedEventID("submitqueue", "batch.failed", "batch-778", "msg-2", 0),
"slashed subject": NewEventID("stovepipe", "commit.green", "request/monorepo/main/42", 4),
"slashed subject": NewEventID("stovepipe", "commit.green", "opaque/subject/42", 4),
}

seen := map[string]string{}
Expand Down
20 changes: 10 additions & 10 deletions api/runway/messagequeue/merge_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,16 @@ import (

func TestMergeRequestRoundTrip(t *testing.T) {
req := &MergeRequest{
Id: "queue-a/42",
Id: "42",
QueueName: "queue-a",
Steps: []*MergeStep{
{
StepId: "queue-a/1",
StepId: "1",
Change: &changepb.Change{Uris: []string{"github://github.example.com/uber/repo/pull/1/0123456789abcdef0123456789abcdef01234567"}},
Strategy: strategypb.Strategy_REBASE,
},
{
StepId: "queue-a/2",
StepId: "2",
Change: &changepb.Change{Uris: []string{"github://github.example.com/uber/repo/pull/2/89abcdef0123456789abcdef0123456789abcdef"}},
Strategy: strategypb.Strategy_MERGE,
},
Expand All @@ -58,20 +58,20 @@ func TestMergeResultRoundTrip(t *testing.T) {
// failure. Both shapes share the one MergeResult contract.
cases := map[string]*MergeResult{
"merged with produced revisions": {
Id: "queue-a/42",
Id: "42",
Outcome: protopb.Outcome_SUCCEEDED,
Steps: []*StepResult{
{StepId: "queue-a/1", Outputs: []*StepOutput{{Id: "0123456789abcdef0123456789abcdef01234567"}}},
{StepId: "1", Outputs: []*StepOutput{{Id: "0123456789abcdef0123456789abcdef01234567"}}},
},
},
"failed with per-step reason": {
Id: "queue-a/42",
Id: "42",
Outcome: protopb.Outcome_FAILED,
Reason: "conflict in foo.go",
Steps: []*StepResult{{StepId: "queue-a/2", Reason: "conflict in foo.go"}},
Steps: []*StepResult{{StepId: "2", Reason: "conflict in foo.go"}},
},
"minimal": {
Id: "queue-a/42",
Id: "42",
Outcome: protopb.Outcome_SUCCEEDED,
},
}
Expand All @@ -93,9 +93,9 @@ func TestMergeResultRoundTrip(t *testing.T) {
// enum values on the wire.
func TestWireFormat(t *testing.T) {
data, err := Marshal(&MergeRequest{
Id: "queue-a/42",
Id: "42",
QueueName: "queue-a",
Steps: []*MergeStep{{StepId: "queue-a/1", Strategy: strategypb.Strategy_SQUASH_REBASE}},
Steps: []*MergeStep{{StepId: "1", Strategy: strategypb.Strategy_SQUASH_REBASE}},
})
require.NoError(t, err)

Expand Down
13 changes: 6 additions & 7 deletions api/stovepipe/proto/stovepipe.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,23 +42,22 @@ message PingResponse {
// IngestRequest is the request for the Ingest method. The poller reports that a queue
// (a named repo+ref) has a new commit to validate.
message IngestRequest {
// Logical queue name to validate (e.g. "monorepo/main"). It namespaces the minted
// request ID and is the stable handle for the repo+ref being validated.
// Logical queue name to validate (e.g. "monorepo/main"). Together with the minted
// request ID it identifies the request and is the stable handle for the repo+ref.
string queue = 1;
}

// IngestResponse is the response for the Ingest method.
message IngestResponse {
// The minted request ID, namespaced by queue. Format: "request/<queue>/<counter>"
// (e.g. "request/monorepo/main/42").
// Canonical decimal request ID, unique within the selected queue.
string id = 1;
}

// GetRequestHistoryByIDRequest selects one request history by the ID returned from Ingest.
message GetRequestHistoryByIDRequest {
// Logical queue containing the request.
string queue = 1;
// Globally unique request identifier returned from Ingest.
// Canonical decimal request identifier returned from Ingest.
string request_id = 2;
}

Expand Down Expand Up @@ -89,7 +88,7 @@ message HistoryEvent {

// RequestHistory identifies one request and contains its ordered events.
message RequestHistory {
// Globally unique request identifier.
// Queue-scoped decimal request identifier.
string request_id = 1;
// Events ordered by occurrence time and stable event identity.
repeated HistoryEvent events = 2;
Expand Down Expand Up @@ -133,7 +132,7 @@ message ProjectValidation {

// GetProjectStatusByURIResponse contains the selected validation's current projection.
message GetProjectStatusByURIResponse {
// Globally unique identifier of the validation request.
// Queue-scoped decimal identifier of the validation request.
string request_id = 1;
// Logical queue containing the validation request.
string queue = 2;
Expand Down
13 changes: 6 additions & 7 deletions api/stovepipe/protopb/stovepipe.pb.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 6 additions & 6 deletions api/submitqueue/gateway/proto/gateway.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,13 @@ message LandRequest {

// LandResponse defines the response to a land request.
message LandResponse {
// Globally unique identifier for the land request. Used to track the land request lifecycle.
// Canonical decimal identifier for the land request, unique within its queue.
string sqid = 1;
}

// CancelRequest defines a request to cancel an in-flight land request. If the request is part of a batch, the entire batch is cancelled.
message CancelRequest {
// Globally unique identifier of the land request to cancel, returned by a prior Land call.
// Canonical decimal identifier of the land request to cancel, returned by a prior Land call.
string sqid = 1;
// Optional human-readable reason for the cancellation. Recorded for observability.
string reason = 2;
Expand All @@ -89,7 +89,7 @@ message CancelResponse {

// RequestSummary is the gateway-owned materialized current view of one received request.
message RequestSummary {
// Globally unique request identifier. This is request_id in internal storage.
// Queue-scoped decimal request identifier. This is request_id in internal storage.
string sqid = 1;
// Queue supplied when the request was received.
string queue = 2;
Expand All @@ -107,7 +107,7 @@ message RequestSummary {

// GetRequestSummaryByIDRequest selects one request by the sqid returned from Land.
message GetRequestSummaryByIDRequest {
// Globally unique identifier for the request.
// Canonical decimal identifier for the request, resolved within queue.
string sqid = 1;
// Name of the queue processing the request. Required. A sqid is only resolvable within its own queue,
// so naming a queue the request does not belong to is reported as not found.
Expand Down Expand Up @@ -175,7 +175,7 @@ message ListQueuesResponse {

// GetRequestHistoryByIDRequest selects one retained request history by sqid.
message GetRequestHistoryByIDRequest {
// Globally unique identifier for the request.
// Canonical decimal identifier for the request, resolved within queue.
string sqid = 1;
// Name of the queue processing the request. Required. A sqid is only resolvable within its own queue,
// so naming a queue the request does not belong to is reported as not found.
Expand Down Expand Up @@ -217,7 +217,7 @@ message GetRequestHistoryByChangeURIRequest {

// RequestHistory groups retained events for one request.
message RequestHistory {
// Globally unique request identifier.
// Queue-scoped decimal request identifier.
string sqid = 1;
// Retained events ordered chronologically.
repeated HistoryEvent events = 2;
Expand Down
12 changes: 6 additions & 6 deletions api/submitqueue/gateway/protopb/gateway.pb.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 12 additions & 12 deletions doc/howto/QUICKSTART.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,9 @@ make demo-requests
```
Creating 3 synthetic change(s) via fake changes (no repository) — independent, 5 at a time, each enqueued as soon as it is created

REQUEST CHANGES ELAPSED STAGE
──────────── ────────────────── ─────── ─────────────────────────────────────────────
demo-queue/1 demo/0814-135021/1 13s accepted → started → validating → validated →
REQUEST CHANGES ELAPSED STAGE
─────── ────────────────── ─────── ─────────────────────────────────────────────
1 demo/0814-135021/1 13s accepted → started → validating → validated →
batching → batched → speculating → speculated →
landing → landed
```
Expand Down Expand Up @@ -112,7 +112,7 @@ Landing one change by hand instead:
```bash
make land QUEUE=demo-queue \
URI='git://demo.example.com/demo/refs%2Fheads%2Fmy-change/1111111111111111111111111111111111111111'
make land-status QUEUE=demo-queue SQID=demo-queue/1
make land-status QUEUE=demo-queue SQID=1
```

A change URI is `git://{remote}/{repo}/{ref}/{commit_sha}`. Two parts are checked before anything else happens: the **commit SHA must be 40 lowercase hex characters**, and the **ref must be fully qualified and percent-encoded** — `refs%2Fheads%2Fmy-change`, not `my-change`. Encoding is what keeps a branch name containing slashes inside a single path segment.
Expand Down Expand Up @@ -227,9 +227,9 @@ git -C /tmp/sq-sandbox/sandbox.git log --oneline main
```

```
b517508 squash: demo-queue/5 (sandbox@refs/heads/demo/0814-134848/2)
25c86c5 squash: demo-queue/4 (sandbox@refs/heads/demo/0814-134848/1)
9f72dcf squash: demo-queue/3 (sandbox@refs/heads/demo/0814-134848/3)
b517508 squash: 5 (sandbox@refs/heads/demo/0814-134848/2)
25c86c5 squash: 4 (sandbox@refs/heads/demo/0814-134848/1)
9f72dcf squash: 3 (sandbox@refs/heads/demo/0814-134848/3)
b5d86d6 seed the sandbox
```

Expand Down Expand Up @@ -299,11 +299,11 @@ make demo-requests
It opens real pull requests, enqueues each as it is created, and watches them land — having picked up from the running stack that this one is GitHub. A three-change run against a scratch repo:

```
REQUEST CHANGES ELAPSED STAGE
──────────── ──────────────────────────────────────────────── ─────── ──────────────────────────────
demo-queue/1 https://github.com/behinddwalls/sq-demo/pull/522 21s accepted → … → landed
demo-queue/2 https://github.com/behinddwalls/sq-demo/pull/523 22s accepted → … → landed
demo-queue/3 https://github.com/behinddwalls/sq-demo/pull/524 25s accepted → … → landed
REQUEST CHANGES ELAPSED STAGE
─────── ──────────────────────────────────────────────── ─────── ──────────────────────────────
1 https://github.com/behinddwalls/sq-demo/pull/522 21s accepted → … → landed
2 https://github.com/behinddwalls/sq-demo/pull/523 22s accepted → … → landed
3 https://github.com/behinddwalls/sq-demo/pull/524 25s accepted → … → landed
```

Their commits land on `main`, but the pull requests themselves stay open: nothing here calls GitHub's API to close them, and `SQUASH_REBASE` rewrites the commits, so a pull request's original head never becomes reachable from `main` for GitHub to notice on its own. Closing them requires a separately driven automation, which nothing in this stack provides.
Expand Down
Loading
Loading