diff --git a/Makefile b/Makefile index 22d48888d..45d64dc4c 100644 --- a/Makefile +++ b/Makefile @@ -416,6 +416,10 @@ local-init-runway-queue-schema: ## Apply queue schema only (mysql-queue) for Run local-init-stovepipe-schemas: ## Apply storage (mysql-app) and queue (mysql-queue) schemas for Stovepipe compose stacks @echo "Applying storage schema to mysql-app..." + @for file in platform/extension/counter/mysql/schema/*.sql; do \ + echo " - Applying $$(basename $$file)..."; \ + docker exec -i $(STOVEPIPE_LOCAL_PROJECT)-mysql-app-1 mysql -uroot -proot submitqueue < $$file 2>&1 | grep -v "Using a password" || true; \ + done @for file in stovepipe/extension/storage/mysql/schema/*.sql; do \ echo " - Applying $$(basename $$file)..."; \ docker exec -i $(STOVEPIPE_LOCAL_PROJECT)-mysql-app-1 mysql -uroot -proot submitqueue < $$file 2>&1 | grep -v "Using a password" || true; \ diff --git a/api/base/hook/hook_test.go b/api/base/hook/hook_test.go index f3a3b58c3..2a5f3c28b 100644 --- a/api/base/hook/hook_test.go +++ b/api/base/hook/hook_test.go @@ -51,7 +51,7 @@ func TestHookEventRoundTrip(t *testing.T) { Source: "runway", Type: "merge.completed", TimestampMs: 1722800012345, - Payload: mustStruct(t, map[string]any{"request_id": "queue-a/42"}), + Payload: mustStruct(t, map[string]any{"request_id": "42"}), }, "envelope only": { Id: "stovepipe/commit.green/git-abc/1", @@ -136,7 +136,7 @@ func TestEventIDIsDerived(t *testing.T) { "unversioned": NewUnversionedEventID("submitqueue", "batch.failed", "batch-778", "msg-1", 0), "second ordinal": NewUnversionedEventID("submitqueue", "batch.failed", "batch-778", "msg-1", 1), "other cause": NewUnversionedEventID("submitqueue", "batch.failed", "batch-778", "msg-2", 0), - "slashed subject": NewEventID("stovepipe", "commit.green", "request/monorepo/main/42", 4), + "slashed subject": NewEventID("stovepipe", "commit.green", "opaque/subject/42", 4), } seen := map[string]string{} diff --git a/api/runway/messagequeue/merge_test.go b/api/runway/messagequeue/merge_test.go index 754489db1..5e535eb28 100644 --- a/api/runway/messagequeue/merge_test.go +++ b/api/runway/messagequeue/merge_test.go @@ -28,16 +28,16 @@ import ( func TestMergeRequestRoundTrip(t *testing.T) { req := &MergeRequest{ - Id: "queue-a/42", + Id: "42", QueueName: "queue-a", Steps: []*MergeStep{ { - StepId: "queue-a/1", + StepId: "1", Change: &changepb.Change{Uris: []string{"github://github.example.com/uber/repo/pull/1/0123456789abcdef0123456789abcdef01234567"}}, Strategy: strategypb.Strategy_REBASE, }, { - StepId: "queue-a/2", + StepId: "2", Change: &changepb.Change{Uris: []string{"github://github.example.com/uber/repo/pull/2/89abcdef0123456789abcdef0123456789abcdef"}}, Strategy: strategypb.Strategy_MERGE, }, @@ -58,20 +58,20 @@ func TestMergeResultRoundTrip(t *testing.T) { // failure. Both shapes share the one MergeResult contract. cases := map[string]*MergeResult{ "merged with produced revisions": { - Id: "queue-a/42", + Id: "42", Outcome: protopb.Outcome_SUCCEEDED, Steps: []*StepResult{ - {StepId: "queue-a/1", Outputs: []*StepOutput{{Id: "0123456789abcdef0123456789abcdef01234567"}}}, + {StepId: "1", Outputs: []*StepOutput{{Id: "0123456789abcdef0123456789abcdef01234567"}}}, }, }, "failed with per-step reason": { - Id: "queue-a/42", + Id: "42", Outcome: protopb.Outcome_FAILED, Reason: "conflict in foo.go", - Steps: []*StepResult{{StepId: "queue-a/2", Reason: "conflict in foo.go"}}, + Steps: []*StepResult{{StepId: "2", Reason: "conflict in foo.go"}}, }, "minimal": { - Id: "queue-a/42", + Id: "42", Outcome: protopb.Outcome_SUCCEEDED, }, } @@ -93,9 +93,9 @@ func TestMergeResultRoundTrip(t *testing.T) { // enum values on the wire. func TestWireFormat(t *testing.T) { data, err := Marshal(&MergeRequest{ - Id: "queue-a/42", + Id: "42", QueueName: "queue-a", - Steps: []*MergeStep{{StepId: "queue-a/1", Strategy: strategypb.Strategy_SQUASH_REBASE}}, + Steps: []*MergeStep{{StepId: "1", Strategy: strategypb.Strategy_SQUASH_REBASE}}, }) require.NoError(t, err) diff --git a/api/stovepipe/proto/stovepipe.proto b/api/stovepipe/proto/stovepipe.proto index 6c6ca2b73..24b657d79 100644 --- a/api/stovepipe/proto/stovepipe.proto +++ b/api/stovepipe/proto/stovepipe.proto @@ -42,15 +42,14 @@ message PingResponse { // IngestRequest is the request for the Ingest method. The poller reports that a queue // (a named repo+ref) has a new commit to validate. message IngestRequest { - // Logical queue name to validate (e.g. "monorepo/main"). It namespaces the minted - // request ID and is the stable handle for the repo+ref being validated. + // Logical queue name to validate (e.g. "monorepo/main"). Together with the minted + // request ID it identifies the request and is the stable handle for the repo+ref. string queue = 1; } // IngestResponse is the response for the Ingest method. message IngestResponse { - // The minted request ID, namespaced by queue. Format: "request//" - // (e.g. "request/monorepo/main/42"). + // Canonical decimal request ID, unique within the selected queue. string id = 1; } @@ -58,7 +57,7 @@ message IngestResponse { message GetRequestHistoryByIDRequest { // Logical queue containing the request. string queue = 1; - // Globally unique request identifier returned from Ingest. + // Canonical decimal request identifier returned from Ingest. string request_id = 2; } @@ -89,7 +88,7 @@ message HistoryEvent { // RequestHistory identifies one request and contains its ordered events. message RequestHistory { - // Globally unique request identifier. + // Queue-scoped decimal request identifier. string request_id = 1; // Events ordered by occurrence time and stable event identity. repeated HistoryEvent events = 2; @@ -133,7 +132,7 @@ message ProjectValidation { // GetProjectStatusByURIResponse contains the selected validation's current projection. message GetProjectStatusByURIResponse { - // Globally unique identifier of the validation request. + // Queue-scoped decimal identifier of the validation request. string request_id = 1; // Logical queue containing the validation request. string queue = 2; diff --git a/api/stovepipe/protopb/stovepipe.pb.go b/api/stovepipe/protopb/stovepipe.pb.go index 5ff9013ce..48dc97754 100644 --- a/api/stovepipe/protopb/stovepipe.pb.go +++ b/api/stovepipe/protopb/stovepipe.pb.go @@ -159,8 +159,8 @@ func (x *PingResponse) GetHostname() string { // (a named repo+ref) has a new commit to validate. type IngestRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - // Logical queue name to validate (e.g. "monorepo/main"). It namespaces the minted - // request ID and is the stable handle for the repo+ref being validated. + // Logical queue name to validate (e.g. "monorepo/main"). Together with the minted + // request ID it identifies the request and is the stable handle for the repo+ref. Queue string `protobuf:"bytes,1,opt,name=queue,proto3" json:"queue,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -206,8 +206,7 @@ func (x *IngestRequest) GetQueue() string { // IngestResponse is the response for the Ingest method. type IngestResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - // The minted request ID, namespaced by queue. Format: "request//" - // (e.g. "request/monorepo/main/42"). + // Canonical decimal request ID, unique within the selected queue. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -255,7 +254,7 @@ type GetRequestHistoryByIDRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // Logical queue containing the request. Queue string `protobuf:"bytes,1,opt,name=queue,proto3" json:"queue,omitempty"` - // Globally unique request identifier returned from Ingest. + // Canonical decimal request identifier returned from Ingest. RequestId string `protobuf:"bytes,2,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -477,7 +476,7 @@ func (*HistoryEvent_Event) isHistoryEvent_Occurrence() {} // RequestHistory identifies one request and contains its ordered events. type RequestHistory struct { state protoimpl.MessageState `protogen:"open.v1"` - // Globally unique request identifier. + // Queue-scoped decimal request identifier. RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` // Events ordered by occurrence time and stable event identity. Events []*HistoryEvent `protobuf:"bytes,2,rep,name=events,proto3" json:"events,omitempty"` @@ -784,7 +783,7 @@ func (*ProjectValidation_BreakageDegree) isProjectValidation_Result() {} // GetProjectStatusByURIResponse contains the selected validation's current projection. type GetProjectStatusByURIResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - // Globally unique identifier of the validation request. + // Queue-scoped decimal identifier of the validation request. RequestId string `protobuf:"bytes,1,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` // Logical queue containing the validation request. Queue string `protobuf:"bytes,2,opt,name=queue,proto3" json:"queue,omitempty"` diff --git a/api/submitqueue/gateway/proto/gateway.proto b/api/submitqueue/gateway/proto/gateway.proto index 3d46b38e3..86a29f68b 100644 --- a/api/submitqueue/gateway/proto/gateway.proto +++ b/api/submitqueue/gateway/proto/gateway.proto @@ -64,13 +64,13 @@ message LandRequest { // LandResponse defines the response to a land request. message LandResponse { - // Globally unique identifier for the land request. Used to track the land request lifecycle. + // Canonical decimal identifier for the land request, unique within its queue. string sqid = 1; } // CancelRequest defines a request to cancel an in-flight land request. If the request is part of a batch, the entire batch is cancelled. message CancelRequest { - // Globally unique identifier of the land request to cancel, returned by a prior Land call. + // Canonical decimal identifier of the land request to cancel, returned by a prior Land call. string sqid = 1; // Optional human-readable reason for the cancellation. Recorded for observability. string reason = 2; @@ -89,7 +89,7 @@ message CancelResponse { // RequestSummary is the gateway-owned materialized current view of one received request. message RequestSummary { - // Globally unique request identifier. This is request_id in internal storage. + // Queue-scoped decimal request identifier. This is request_id in internal storage. string sqid = 1; // Queue supplied when the request was received. string queue = 2; @@ -107,7 +107,7 @@ message RequestSummary { // GetRequestSummaryByIDRequest selects one request by the sqid returned from Land. message GetRequestSummaryByIDRequest { - // Globally unique identifier for the request. + // Canonical decimal identifier for the request, resolved within queue. string sqid = 1; // Name of the queue processing the request. Required. A sqid is only resolvable within its own queue, // so naming a queue the request does not belong to is reported as not found. @@ -175,7 +175,7 @@ message ListQueuesResponse { // GetRequestHistoryByIDRequest selects one retained request history by sqid. message GetRequestHistoryByIDRequest { - // Globally unique identifier for the request. + // Canonical decimal identifier for the request, resolved within queue. string sqid = 1; // Name of the queue processing the request. Required. A sqid is only resolvable within its own queue, // so naming a queue the request does not belong to is reported as not found. @@ -217,7 +217,7 @@ message GetRequestHistoryByChangeURIRequest { // RequestHistory groups retained events for one request. message RequestHistory { - // Globally unique request identifier. + // Queue-scoped decimal request identifier. string sqid = 1; // Retained events ordered chronologically. repeated HistoryEvent events = 2; diff --git a/api/submitqueue/gateway/protopb/gateway.pb.go b/api/submitqueue/gateway/protopb/gateway.pb.go index 0a83dec4b..e18c3ccae 100644 --- a/api/submitqueue/gateway/protopb/gateway.pb.go +++ b/api/submitqueue/gateway/protopb/gateway.pb.go @@ -230,7 +230,7 @@ func (x *LandRequest) GetStrategy() protopb1.Strategy { // LandResponse defines the response to a land request. type LandResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - // Globally unique identifier for the land request. Used to track the land request lifecycle. + // Canonical decimal identifier for the land request, unique within its queue. Sqid string `protobuf:"bytes,1,opt,name=sqid,proto3" json:"sqid,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -276,7 +276,7 @@ func (x *LandResponse) GetSqid() string { // CancelRequest defines a request to cancel an in-flight land request. If the request is part of a batch, the entire batch is cancelled. type CancelRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - // Globally unique identifier of the land request to cancel, returned by a prior Land call. + // Canonical decimal identifier of the land request to cancel, returned by a prior Land call. Sqid string `protobuf:"bytes,1,opt,name=sqid,proto3" json:"sqid,omitempty"` // Optional human-readable reason for the cancellation. Recorded for observability. Reason string `protobuf:"bytes,2,opt,name=reason,proto3" json:"reason,omitempty"` @@ -382,7 +382,7 @@ func (*CancelResponse) Descriptor() ([]byte, []int) { // RequestSummary is the gateway-owned materialized current view of one received request. type RequestSummary struct { state protoimpl.MessageState `protogen:"open.v1"` - // Globally unique request identifier. This is request_id in internal storage. + // Queue-scoped decimal request identifier. This is request_id in internal storage. Sqid string `protobuf:"bytes,1,opt,name=sqid,proto3" json:"sqid,omitempty"` // Queue supplied when the request was received. Queue string `protobuf:"bytes,2,opt,name=queue,proto3" json:"queue,omitempty"` @@ -482,7 +482,7 @@ func (x *RequestSummary) GetMetadata() map[string]string { // GetRequestSummaryByIDRequest selects one request by the sqid returned from Land. type GetRequestSummaryByIDRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - // Globally unique identifier for the request. + // Canonical decimal identifier for the request, resolved within queue. Sqid string `protobuf:"bytes,1,opt,name=sqid,proto3" json:"sqid,omitempty"` // Name of the queue processing the request. Required. A sqid is only resolvable within its own queue, // so naming a queue the request does not belong to is reported as not found. @@ -952,7 +952,7 @@ func (x *ListQueuesResponse) GetQueues() []*Queue { // GetRequestHistoryByIDRequest selects one retained request history by sqid. type GetRequestHistoryByIDRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - // Globally unique identifier for the request. + // Canonical decimal identifier for the request, resolved within queue. Sqid string `protobuf:"bytes,1,opt,name=sqid,proto3" json:"sqid,omitempty"` // Name of the queue processing the request. Required. A sqid is only resolvable within its own queue, // so naming a queue the request does not belong to is reported as not found. @@ -1203,7 +1203,7 @@ func (x *GetRequestHistoryByChangeURIRequest) GetQueue() string { // RequestHistory groups retained events for one request. type RequestHistory struct { state protoimpl.MessageState `protogen:"open.v1"` - // Globally unique request identifier. + // Queue-scoped decimal request identifier. Sqid string `protobuf:"bytes,1,opt,name=sqid,proto3" json:"sqid,omitempty"` // Retained events ordered chronologically. Events []*HistoryEvent `protobuf:"bytes,2,rep,name=events,proto3" json:"events,omitempty"` diff --git a/doc/howto/QUICKSTART.md b/doc/howto/QUICKSTART.md index ffe0d6bff..d8892b71b 100644 --- a/doc/howto/QUICKSTART.md +++ b/doc/howto/QUICKSTART.md @@ -45,9 +45,9 @@ make demo-requests ``` Creating 3 synthetic change(s) via fake changes (no repository) — independent, 5 at a time, each enqueued as soon as it is created - REQUEST CHANGES ELAPSED STAGE - ──────────── ────────────────── ─────── ───────────────────────────────────────────── - demo-queue/1 demo/0814-135021/1 13s accepted → started → validating → validated → + REQUEST CHANGES ELAPSED STAGE + ─────── ────────────────── ─────── ───────────────────────────────────────────── + 1 demo/0814-135021/1 13s accepted → started → validating → validated → batching → batched → speculating → speculated → landing → landed ``` @@ -112,7 +112,7 @@ Landing one change by hand instead: ```bash make land QUEUE=demo-queue \ URI='git://demo.example.com/demo/refs%2Fheads%2Fmy-change/1111111111111111111111111111111111111111' -make land-status QUEUE=demo-queue SQID=demo-queue/1 +make land-status QUEUE=demo-queue SQID=1 ``` A change URI is `git://{remote}/{repo}/{ref}/{commit_sha}`. Two parts are checked before anything else happens: the **commit SHA must be 40 lowercase hex characters**, and the **ref must be fully qualified and percent-encoded** — `refs%2Fheads%2Fmy-change`, not `my-change`. Encoding is what keeps a branch name containing slashes inside a single path segment. @@ -227,9 +227,9 @@ git -C /tmp/sq-sandbox/sandbox.git log --oneline main ``` ``` -b517508 squash: demo-queue/5 (sandbox@refs/heads/demo/0814-134848/2) -25c86c5 squash: demo-queue/4 (sandbox@refs/heads/demo/0814-134848/1) -9f72dcf squash: demo-queue/3 (sandbox@refs/heads/demo/0814-134848/3) +b517508 squash: 5 (sandbox@refs/heads/demo/0814-134848/2) +25c86c5 squash: 4 (sandbox@refs/heads/demo/0814-134848/1) +9f72dcf squash: 3 (sandbox@refs/heads/demo/0814-134848/3) b5d86d6 seed the sandbox ``` @@ -299,11 +299,11 @@ make demo-requests It opens real pull requests, enqueues each as it is created, and watches them land — having picked up from the running stack that this one is GitHub. A three-change run against a scratch repo: ``` - REQUEST CHANGES ELAPSED STAGE - ──────────── ──────────────────────────────────────────────── ─────── ────────────────────────────── - demo-queue/1 https://github.com/behinddwalls/sq-demo/pull/522 21s accepted → … → landed - demo-queue/2 https://github.com/behinddwalls/sq-demo/pull/523 22s accepted → … → landed - demo-queue/3 https://github.com/behinddwalls/sq-demo/pull/524 25s accepted → … → landed + REQUEST CHANGES ELAPSED STAGE + ─────── ──────────────────────────────────────────────── ─────── ────────────────────────────── + 1 https://github.com/behinddwalls/sq-demo/pull/522 21s accepted → … → landed + 2 https://github.com/behinddwalls/sq-demo/pull/523 22s accepted → … → landed + 3 https://github.com/behinddwalls/sq-demo/pull/524 25s accepted → … → landed ``` Their commits land on `main`, but the pull requests themselves stay open: nothing here calls GitHub's API to close them, and `SQUASH_REBASE` rewrites the commits, so a pull request's original head never becomes reachable from `main` for GitHub to notice on its own. Closing them requires a separately driven automation, which nothing in this stack provides. diff --git a/doc/rfc/scoped-resource-ids.md b/doc/rfc/scoped-resource-ids.md index e408459ee..635e53ea2 100644 --- a/doc/rfc/scoped-resource-ids.md +++ b/doc/rfc/scoped-resource-ids.md @@ -2,26 +2,27 @@ ## Decision -A generated resource ID is the canonical decimal string for a positive value returned by a durable counter scoped to `(owner domain, queue, resource kind)`. +A generated resource ID is the canonical decimal string for a positive value returned by a durable counter scoped to `(queue, domain)` within an application's storage. -| Resource | Current ID | Proposed ID | Complete identity | +The counter domain names the sequence (`request` or `batch`), not the application. SubmitQueue and Stovepipe use separate storage backends; there is no additional application-domain key or schema change. + +| Resource | Current ID | Proposed ID | Identity within the application | |---|---:|---:|---| -| SubmitQueue request | `demo-queue/42` | `"42"` | `(submitqueue, demo-queue, request, "42")` | -| SubmitQueue batch | `demo-queue/batch/7` | `"7"` | `(submitqueue, demo-queue, batch, "7")` | -| Stovepipe request | `request/monorepo/main/42` | `"42"` | `(stovepipe, monorepo/main, request, "42")` | +| SubmitQueue request | `demo-queue/42` | `"42"` | `(demo-queue, request, "42")` | +| SubmitQueue batch | `demo-queue/batch/7` | `"7"` | `(demo-queue, batch, "7")` | +| Stovepipe request | `request/monorepo/main/42` | `"42"` | `(monorepo/main, request, "42")` | -The decimal ID is unique only within its scope. The same value may appear in another queue, resource kind, or domain. APIs and messages therefore carry the queue separately; their typed field or message type supplies the resource kind. +The decimal ID is unique only within its scope. The same value may appear in another queue, counter domain, or application. APIs and messages therefore carry the queue separately; their typed field or message type supplies the counter domain. Do not embed scope into the ID. Forms such as `demo-queue/42`, `demo-queue/batch/7`, `request.42`, and ARN-like resource names are not stored or accepted as IDs. ## Counter -The counter backend persists one high-water mark per `(owner domain, queue, resource kind)`. For example: +The counter backend persists one high-water mark per `(queue, domain)`. MySQL keeps its existing schema and primary key. For example: ```text -(submitqueue, demo-queue, request) -> 42 -(submitqueue, demo-queue, batch) -> 7 -(stovepipe, demo-queue, request) -> 11 +(demo-queue, request) -> 42 +(demo-queue, batch) -> 7 ``` Controllers allocate an ID before creating the resource; stores accept the caller-supplied ID and never generate one. @@ -60,7 +61,7 @@ Both columns use the same queue prefix for comparison. The before batch/change r ## Rejected alternatives -- **Queue or kind prefixes:** duplicate explicit context, lengthen keys, require parsing, and introduce URL separators. +- **Queue or domain prefixes:** duplicate explicit context, lengthen keys, require parsing, and introduce URL separators. - **ARN-like names:** solve global lookup, which current APIs neither provide nor require. - **UUIDs or a global counter:** provide global uniqueness at the cost of unnecessary encoding or coordination. - **Integer resource fields:** couple the persisted and wire contracts to the current counter representation without adding identity semantics. diff --git a/doc/rfc/stovepipe/steps/process.md b/doc/rfc/stovepipe/steps/process.md index 8f6962733..5ca099e6f 100644 --- a/doc/rfc/stovepipe/steps/process.md +++ b/doc/rfc/stovepipe/steps/process.md @@ -94,7 +94,7 @@ While a validation runs, ingest keeps admitting newer heads (distinct Requests, "Latest" is the **monotonic ingest order**, not VCS history: -- Ingest mints ids from a per-Queue counter (`request//`). The counter suffix is the order key (higher = later ingest = newer head); there is no separate `Sequence` field on `Request`. +- Ingest mints IDs from a durable per-queue counter and stores each value as its canonical decimal string. The numeric value is the order key (higher = later ingest = newer head); there is no separate `Sequence` field on `Request`. - Ingest also CASes `Queue.latest_request_id` to the accepted request's id when it is newer (via `CompareRequestID`) — the **out-of-band latest pointer** (a request id, distinct from `last_green_uri`) that makes coalescing a single-row comparison. Why not `SourceControl.History`: a history walk is expensive, and after a rewrite the superseded URIs may be off-ref entirely — exactly where history order is meaningless. Ingest order authoritatively says which head we learned of last. @@ -202,7 +202,7 @@ Per-queue knobs such as `max_concurrent` live outside this row — see [Per-Queu | Field | Role | |---|---| -| `ID` | Globally unique id (`request//`); the counter suffix is the ingest-order key for coalescing | +| `ID` | Canonical decimal string, unique within `Queue`; its numeric value is the ingest-order key for coalescing | | `BuildStrategy` | `incremental_since_green` \| `full`; immutable once set by `process` | | `BaseURI` | Last-green URI used as the incremental base; empty for full builds | diff --git a/doc/rfc/submitqueue/history-api.md b/doc/rfc/submitqueue/history-api.md index b9155fb6c..547557ee2 100644 --- a/doc/rfc/submitqueue/history-api.md +++ b/doc/rfc/submitqueue/history-api.md @@ -16,7 +16,7 @@ The gateway exposes two read-only RPCs because an `sqid` selects one event list ```proto message GetRequestHistoryByIDRequest { - // Globally unique identifier for a request, as returned by Land. + // Canonical decimal identifier for a request, as returned by Land. string sqid = 1; // Queue processing the request. Required: a sqid is only resolvable within its own queue. string queue = 2; @@ -46,7 +46,7 @@ message GetRequestHistoryByChangeURIRequest { } message RequestHistory { - // Globally unique identifier for the request associated with these events. + // Queue-scoped decimal identifier for the request associated with these events. string sqid = 1; // Retained request-log events ordered by timestamp_ms ascending with a stable tie-breaker. repeated HistoryEvent events = 2; diff --git a/platform/base/failure/failure_test.go b/platform/base/failure/failure_test.go index 1fce929e5..f6468ecf7 100644 --- a/platform/base/failure/failure_test.go +++ b/platform/base/failure/failure_test.go @@ -38,8 +38,8 @@ func TestRoundTrip(t *testing.T) { }, { name: "several subjects keep their order", - in: New("two at fault", Subject{Type: "batch", ID: "q/batch/2"}, Subject{Type: "batch", ID: "q/batch/1"}), - want: Failure{Subjects: []Subject{{Type: "batch", ID: "q/batch/2"}, {Type: "batch", ID: "q/batch/1"}}}, + in: New("two at fault", Subject{Type: "batch", ID: "2"}, Subject{Type: "batch", ID: "1"}), + want: Failure{Subjects: []Subject{{Type: "batch", ID: "2"}, {Type: "batch", ID: "1"}}}, }, { name: "nested detail survives", @@ -65,14 +65,14 @@ func TestRoundTrip(t *testing.T) { // legible column, and so decoding never has to tell an encoded failure apart // from a message that happens to look like one. func TestEncodeOmitsMessage(t *testing.T) { - encoded, err := Encode(New("boom", Subject{Type: "batch", ID: "q/batch/1"})) + encoded, err := Encode(New("boom", Subject{Type: "batch", ID: "1"})) require.NoError(t, err) assert.NotContains(t, string(encoded), "boom") got, err := Decode(encoded) require.NoError(t, err) assert.Empty(t, got.Message) - assert.Equal(t, []Subject{{Type: "batch", ID: "q/batch/1"}}, got.Subjects) + assert.Equal(t, []Subject{{Type: "batch", ID: "1"}}, got.Subjects) } // Nothing structured means nothing to store, which is what lets a caller treat @@ -108,12 +108,12 @@ func TestDetailNumbersDecodeAsFloat64(t *testing.T) { func TestIDsOfType(t *testing.T) { f := New("mixed", - Subject{Type: "batch", ID: "q/batch/1"}, + Subject{Type: "batch", ID: "1"}, Subject{Type: "queue", ID: "q"}, - Subject{Type: "batch", ID: "q/batch/2"}, + Subject{Type: "batch", ID: "2"}, ) - assert.Equal(t, []string{"q/batch/1", "q/batch/2"}, f.IDsOfType("batch")) + assert.Equal(t, []string{"1", "2"}, f.IDsOfType("batch")) assert.Equal(t, []string{"q"}, f.IDsOfType("queue")) assert.Empty(t, f.IDsOfType("request")) assert.Empty(t, Failure{}.IDsOfType("batch")) diff --git a/platform/base/id/BUILD.bazel b/platform/base/id/BUILD.bazel new file mode 100644 index 000000000..4e3d219be --- /dev/null +++ b/platform/base/id/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["id.go"], + importpath = "github.com/uber/submitqueue/platform/base/id", + visibility = ["//visibility:public"], +) + +go_test( + name = "go_default_test", + srcs = ["id_test.go"], + embed = [":go_default_library"], + deps = [ + "@com_github_stretchr_testify//assert:go_default_library", + "@com_github_stretchr_testify//require:go_default_library", + ], +) diff --git a/platform/base/id/id.go b/platform/base/id/id.go new file mode 100644 index 000000000..71ed9d28b --- /dev/null +++ b/platform/base/id/id.go @@ -0,0 +1,76 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package id formats, validates, and compares counter-generated resource IDs. +package id + +import ( + "fmt" + "strconv" +) + +// FromCounter returns the canonical decimal resource ID for value. +func FromCounter(value int64) (string, error) { + if value <= 0 { + return "", fmt.Errorf("counter value must be positive") + } + return strconv.FormatInt(value, 10), nil +} + +// Validate verifies that id is the canonical decimal representation of a positive int64. +func Validate(id string) error { + value, err := strconv.ParseInt(id, 10, 64) + if err != nil { + return fmt.Errorf("resource ID must be a positive decimal string: %w", err) + } + canonical, err := FromCounter(value) + if err != nil { + return err + } + if id != canonical { + return fmt.Errorf("resource ID must use canonical decimal form") + } + return nil +} + +// Compare compares two canonical resource IDs numerically. +func Compare(a, b string) (int, error) { + aValue, err := parseResourceID(a) + if err != nil { + return 0, fmt.Errorf("invalid first resource ID %q: %w", a, err) + } + bValue, err := parseResourceID(b) + if err != nil { + return 0, fmt.Errorf("invalid second resource ID %q: %w", b, err) + } + switch { + case aValue < bValue: + return -1, nil + case aValue > bValue: + return 1, nil + default: + return 0, nil + } +} + +func parseResourceID(id string) (int64, error) { + if err := Validate(id); err != nil { + return 0, err + } + value, err := strconv.ParseInt(id, 10, 64) + if err != nil { + return 0, err + } + return value, nil +} diff --git a/platform/base/id/id_test.go b/platform/base/id/id_test.go new file mode 100644 index 000000000..a6b0b08d2 --- /dev/null +++ b/platform/base/id/id_test.go @@ -0,0 +1,106 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package id + +import ( + "math" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFromCounter(t *testing.T) { + tests := []struct { + name string + value int64 + want string + wantErr bool + }{ + {name: "first", value: 1, want: "1"}, + {name: "maximum", value: math.MaxInt64, want: "9223372036854775807"}, + {name: "zero", wantErr: true}, + {name: "negative", value: -1, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := FromCounter(tt.value) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestValidate(t *testing.T) { + tests := []struct { + name string + id string + wantErr bool + }{ + {name: "valid", id: "42"}, + {name: "maximum", id: "9223372036854775807"}, + {name: "empty", wantErr: true}, + {name: "zero", id: "0", wantErr: true}, + {name: "negative", id: "-1", wantErr: true}, + {name: "leading zero", id: "042", wantErr: true}, + {name: "prefix", id: "request.42", wantErr: true}, + {name: "slash", id: "demo-queue/42", wantErr: true}, + {name: "overflow", id: "9223372036854775808", wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := Validate(tt.id) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + }) + } +} + +func TestCompare(t *testing.T) { + tests := []struct { + name string + a string + b string + want int + wantErr bool + }{ + {name: "older", a: "9", b: "10", want: -1}, + {name: "equal", a: "42", b: "42"}, + {name: "newer", a: "10", b: "9", want: 1}, + {name: "invalid first", a: "queue/9", b: "10", wantErr: true}, + {name: "invalid second", a: "9", b: "batch.10", wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := Compare(tt.a, tt.b) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/platform/errs/failure_test.go b/platform/errs/failure_test.go index f4d39ae1f..790d10660 100644 --- a/platform/errs/failure_test.go +++ b/platform/errs/failure_test.go @@ -45,11 +45,11 @@ func (driverClassifier) Classify(err error) Verdict { } func TestAttributionReadsBackSubjects(t *testing.T) { - err := Attribute(errors.New("boom"), failure.Subject{Type: "batch", ID: "q/batch/1"}) + err := Attribute(errors.New("boom"), failure.Subject{Type: "batch", ID: "1"}) got := Attribution(err) assert.Equal(t, "boom", got.Message) - assert.Equal(t, []failure.Subject{{Type: "batch", ID: "q/batch/1"}}, got.Subjects) + assert.Equal(t, []failure.Subject{{Type: "batch", ID: "1"}}, got.Subjects) } // An unattributed error still yields a usable failure — the message alone. @@ -74,12 +74,12 @@ func TestAttributeNilIsNil(t *testing.T) { // Attribution survives further wrapping, which is the normal shape: a leaf // attributes the entity it knows about, outer frames add context with %w. func TestAttributionThroughOuterWrap(t *testing.T) { - inner := Attribute(errors.New("boom"), failure.Subject{Type: "batch", ID: "q/batch/1"}) + inner := Attribute(errors.New("boom"), failure.Subject{Type: "batch", ID: "1"}) outer := fmt.Errorf("run failed: %w", inner) got := Attribution(outer) assert.Equal(t, "run failed: boom", got.Message) - assert.Equal(t, []failure.Subject{{Type: "batch", ID: "q/batch/1"}}, got.Subjects) + assert.Equal(t, []failure.Subject{{Type: "batch", ID: "1"}}, got.Subjects) } func TestAttributionMergesLayers(t *testing.T) { @@ -105,7 +105,7 @@ func TestAttributionMergesLayers(t *testing.T) { // being retryable. func TestAttributionDoesNotHideTheCauseFromClassifiers(t *testing.T) { cause := driverError{code: 1213} - attributed := Attribute(fmt.Errorf("write failed: %w", cause), failure.Subject{Type: "batch", ID: "q/batch/1"}) + attributed := Attribute(fmt.Errorf("write failed: %w", cause), failure.Subject{Type: "batch", ID: "1"}) var de driverError require.True(t, errors.As(attributed, &de), "errors.As must reach the cause through the wrapper") @@ -115,7 +115,7 @@ func TestAttributionDoesNotHideTheCauseFromClassifiers(t *testing.T) { assert.True(t, IsRetryable(processed), "classification must survive attribution") // And the attribution survives classification wrapping it in turn. - assert.Equal(t, []failure.Subject{{Type: "batch", ID: "q/batch/1"}}, Attribution(processed).Subjects) + assert.Equal(t, []failure.Subject{{Type: "batch", ID: "1"}}, Attribution(processed).Subjects) } // Attribution carries no verdict of its own, so an attributed error with diff --git a/platform/extension/consumergate/file/README.md b/platform/extension/consumergate/file/README.md index 78147c6d5..a787c1559 100644 --- a/platform/extension/consumergate/file/README.md +++ b/platform/extension/consumergate/file/README.md @@ -10,7 +10,7 @@ Stores gate state as plain files under an explicitly configured root directory f {dir}/parked/{consumer_group}/{topic}/{urlenc(id)}.json # one parked delivery record ``` -Partition keys and message IDs may contain `/` (request IDs like `queue/1`), so they are URL-encoded in file names. Gate files carry human-readable JSON metadata (`reason`, `created_by`, `created_at_ms`); parked records carry the payload, attempt, and `parked_at_ms` while a delivery is blocked. The record is refreshed on each re-check of a still-closed gate and removed on the admit path once the gate opens, so the parked tree contains only currently blocked deliveries and does not retain payloads after release. All writes go through temp-file-plus-rename so readers never see partial JSON. +Partition keys and message IDs may contain `/` (for example, a derived event ID), so they are URL-encoded in file names. Gate files carry human-readable JSON metadata (`reason`, `created_by`, `created_at_ms`); parked records carry the payload, attempt, and `parked_at_ms` while a delivery is blocked. The record is refreshed on each re-check of a still-closed gate and removed on the admit path once the gate opens, so the parked tree contains only currently blocked deliveries and does not retain payloads after release. All writes go through temp-file-plus-rename so readers never see partial JSON. Gate state is not cached and the store never polls. Each delivery attempt reads its applicable gate files; a blocked delivery's re-check cadence is the consumer's postpone delay, so gate state is re-read on every redelivery. diff --git a/platform/extension/consumergate/file/store.go b/platform/extension/consumergate/file/store.go index 464b3e7cc..8659e2e80 100644 --- a/platform/extension/consumergate/file/store.go +++ b/platform/extension/consumergate/file/store.go @@ -21,7 +21,7 @@ // parked/{consumer_group}/{topic}/t-{tenant}/p-{partition}/{urlenc(id)}.json // // Consumer groups and topics are filesystem-safe by the repo's naming rules; -// partition keys and message IDs may contain "/" (request IDs like "queue/1"), +// partition keys and message IDs may contain "/" (for example, derived event IDs), // so they are URL-encoded in file names. Gate files hold human-readable JSON // metadata so an operator finding a paused controller can tell why. All writes // go through temp-file-plus-rename so readers never see partial JSON. diff --git a/platform/extension/counter/README.md b/platform/extension/counter/README.md index 8e2d48979..5a9c9b865 100644 --- a/platform/extension/counter/README.md +++ b/platform/extension/counter/README.md @@ -1,6 +1,8 @@ # Counter -Vendor-agnostic interface for atomic sequential number generation, scoped per queue. +Vendor-agnostic interface for atomic sequential number generation, scoped by queue and domain. + +The domain names a counter sequence (`request` or `batch`), not an application. SubmitQueue and Stovepipe use separate storage backends; the existing MySQL schema and `(queue, domain)` key remain unchanged. ## Interface @@ -10,12 +12,12 @@ Resolves the Counter bound to one queue. The host wiring decides which backend s ### Counter -Generates unique, sequential values scoped to a domain string within the bound queue. +Generates unique, sequential values scoped to a domain within the bound queue. - **domain**: A string key naming a sequence within the queue (max 255 characters). Each `(queue, domain)` pair maintains its own independent sequence. - **Next**: Atomically increments and returns the next value. The first call for a new domain returns 1. Safe for concurrent use; values are unique but ordering is not guaranteed. -The domain is a sequence *name*, not a queue-qualified key — callers pass `"request"` or `"batch"`, never `"request/my-queue"`. Callers that embed the queue in a minted identifier build that string themselves, independently of the domain, so the two cannot drift into each other. +The domain is not an ID prefix. Callers pass `"request"` or `"batch"`; the returned number is formatted as a decimal string without embedding any scope. ## Usage diff --git a/platform/extension/counter/counter.go b/platform/extension/counter/counter.go index e272d2878..5245633ce 100644 --- a/platform/extension/counter/counter.go +++ b/platform/extension/counter/counter.go @@ -37,14 +37,15 @@ type Factory interface { For(config Config) (Counter, error) } -// Counter provides atomic sequential number generation for a given domain -// within the queue the instance is bound to. +// Counter provides atomic sequential number generation for a domain within +// the queue the instance is bound to. // Each call to Next returns the next value in the sequence for the specified domain. -// The value is guaranteed to be unique within the (queue, domain) pair throughout the system and persisted accordingly. +// The value is unique within the (queue, domain) pair. +// A domain names a sequence, such as "request" or "batch", not an application. type Counter interface { // Next atomically increments the counter for the given domain and returns the new value. // The first call for a new domain returns 1. - // The implementation should support at least 255 length domains. + // The implementation should support domains up to 255 bytes. // The function is safe to be called concurrently and will give unique results, but the order of the values is not guaranteed. Next(ctx context.Context, domain string) (int64, error) } diff --git a/platform/extension/counter/mysql/BUILD.bazel b/platform/extension/counter/mysql/BUILD.bazel index ae6e8b161..943cce213 100644 --- a/platform/extension/counter/mysql/BUILD.bazel +++ b/platform/extension/counter/mysql/BUILD.bazel @@ -1,4 +1,4 @@ -load("@rules_go//go:def.bzl", "go_library") +load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -11,3 +11,15 @@ go_library( "@com_github_uber_go_tally//:go_default_library", ], ) + +go_test( + name = "go_default_test", + srcs = ["counter_test.go"], + embed = [":go_default_library"], + deps = [ + "@com_github_data_dog_go_sqlmock//:go_default_library", + "@com_github_stretchr_testify//assert:go_default_library", + "@com_github_stretchr_testify//require:go_default_library", + "@com_github_uber_go_tally//:go_default_library", + ], +) diff --git a/platform/extension/counter/mysql/counter.go b/platform/extension/counter/mysql/counter.go index ec34e3c7b..8286ac278 100644 --- a/platform/extension/counter/mysql/counter.go +++ b/platform/extension/counter/mysql/counter.go @@ -37,7 +37,7 @@ func NewCounter(db *sql.DB, scope tally.Scope, queue string) counter.Counter { return &mysqlCounter{db: db, scope: scope, queue: queue} } -// Next atomically increments the counter for the given domain within the bound queue +// Next atomically increments the counter for the given domain within the bound scope // and returns the new value. // Uses MySQL's LAST_INSERT_ID() to set the value atomically and read the incremented value. func (c *mysqlCounter) Next(ctx context.Context, domain string) (ret int64, retErr error) { @@ -55,6 +55,9 @@ func (c *mysqlCounter) Next(ctx context.Context, domain string) (ret int64, retE if err != nil { return 0, fmt.Errorf("failed to get counter value for queue=%s domain=%s: %w", c.queue, domain, err) } + if value <= 0 { + return 0, fmt.Errorf("counter returned non-positive value for queue=%s domain=%s", c.queue, domain) + } return value, nil } diff --git a/platform/extension/counter/mysql/counter_test.go b/platform/extension/counter/mysql/counter_test.go new file mode 100644 index 000000000..7d8acc26c --- /dev/null +++ b/platform/extension/counter/mysql/counter_test.go @@ -0,0 +1,67 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mysql + +import ( + "context" + "fmt" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/uber-go/tally" +) + +func TestCounterNext(t *testing.T) { + const query = "INSERT INTO counter (queue, domain, value) VALUES (?, ?, LAST_INSERT_ID(1)) ON DUPLICATE KEY UPDATE value = LAST_INSERT_ID(value + 1)" + + tests := []struct { + name string + result int64 + execErr error + wantErr bool + }{ + {name: "returns allocated value", result: 42}, + {name: "rejects zero", wantErr: true}, + {name: "propagates increment failure", execErr: fmt.Errorf("database unavailable"), wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + require.NoError(t, err) + + expectation := mock.ExpectExec(query).WithArgs("demo-queue", "request") + if tt.execErr != nil { + expectation.WillReturnError(tt.execErr) + } else { + expectation.WillReturnResult(sqlmock.NewResult(tt.result, 1)) + } + mock.ExpectClose() + + counter := NewCounter(db, tally.NoopScope, "demo-queue") + got, err := counter.Next(context.Background(), "request") + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + assert.Equal(t, tt.result, got) + } + require.NoError(t, db.Close()) + require.NoError(t, mock.ExpectationsWereMet()) + }) + } +} diff --git a/platform/extension/messagequeue/mysql/subscriber_test.go b/platform/extension/messagequeue/mysql/subscriber_test.go index cff0337bb..62824c1fa 100644 --- a/platform/extension/messagequeue/mysql/subscriber_test.go +++ b/platform/extension/messagequeue/mysql/subscriber_test.go @@ -619,7 +619,7 @@ func TestSQLDelivery_NackDeadLettersWhenBudgetSpent(t *testing.T) { dlqConfig := extqueue.DLQConfig{Enabled: true, TopicSuffix: "_dlq"} d := newDeliveryForTest(sub, tt.attempt, dlqConfig, tt.retry) - f := failure.New("boom", failure.Subject{Type: "batch", ID: "q/batch/1"}) + f := failure.New("boom", failure.Subject{Type: "batch", ID: "1"}) if tt.wantDLQ { mockMsgStore.EXPECT().MoveToDLQ( diff --git a/runway/controller/dlq/dlq_test.go b/runway/controller/dlq/dlq_test.go index 9b29f5d9c..25c30d2ee 100644 --- a/runway/controller/dlq/dlq_test.go +++ b/runway/controller/dlq/dlq_test.go @@ -33,7 +33,7 @@ import ( ) const ( - testID = "test-queue/1" + testID = "1" testQueue = "test-queue" testPartitionKey = "test-queue" ) diff --git a/runway/controller/merge/merge_test.go b/runway/controller/merge/merge_test.go index 2bd691f1d..9e2ec49d1 100644 --- a/runway/controller/merge/merge_test.go +++ b/runway/controller/merge/merge_test.go @@ -35,7 +35,7 @@ import ( ) const ( - testID = "test-queue/1" + testID = "1" testQueue = "test-queue" testPartitionKey = "test-queue" ) diff --git a/runway/controller/mergeconflictcheck/mergeconflictcheck_test.go b/runway/controller/mergeconflictcheck/mergeconflictcheck_test.go index 8e4bd201a..aee38de34 100644 --- a/runway/controller/mergeconflictcheck/mergeconflictcheck_test.go +++ b/runway/controller/mergeconflictcheck/mergeconflictcheck_test.go @@ -35,7 +35,7 @@ import ( ) const ( - testID = "test-queue/1" + testID = "1" testQueue = "test-queue" testPartitionKey = "test-queue" ) diff --git a/runway/extension/merger/fake/fake_test.go b/runway/extension/merger/fake/fake_test.go index 0bdf25156..bc4dec064 100644 --- a/runway/extension/merger/fake/fake_test.go +++ b/runway/extension/merger/fake/fake_test.go @@ -37,16 +37,16 @@ const baseURI = "github://github.example.com/uber/repo/pull/1/abcdef0123456789ab // URIs, so tests can prove the token is found on any step, not just the first. func requestWith(uris ...string) *runwaymq.MergeRequest { return &runwaymq.MergeRequest{ - Id: "queue-a/42", + Id: "42", QueueName: "queue-a", Steps: []*runwaymq.MergeStep{ { - StepId: "queue-a/1", + StepId: "1", Change: &changepb.Change{Uris: []string{baseURI}}, Strategy: strategypb.Strategy_REBASE, }, { - StepId: "queue-a/2", + StepId: "2", Change: &changepb.Change{Uris: uris}, Strategy: strategypb.Strategy_REBASE, }, @@ -64,9 +64,9 @@ func TestUnmarkedRequestSucceeds(t *testing.T) { assert.Equal(t, req.GetId(), res.GetId()) assert.Equal(t, runwaypb.Outcome_SUCCEEDED, res.GetOutcome()) require.Len(t, res.GetSteps(), 2) - assert.Equal(t, "queue-a/1", res.GetSteps()[0].GetStepId()) + assert.Equal(t, "1", res.GetSteps()[0].GetStepId()) assert.Empty(t, res.GetSteps()[0].GetOutputs()) - assert.Equal(t, "queue-a/2", res.GetSteps()[1].GetStepId()) + assert.Equal(t, "2", res.GetSteps()[1].GetStepId()) assert.Empty(t, res.GetSteps()[1].GetOutputs()) }) @@ -140,11 +140,11 @@ func TestUnrecognizedTokenSucceeds(t *testing.T) { func TestFirstRecognizedTokenWins(t *testing.T) { req := &runwaymq.MergeRequest{ - Id: "queue-a/42", + Id: "42", QueueName: "queue-a", Steps: []*runwaymq.MergeStep{ - {StepId: "queue-a/1", Change: &changepb.Change{Uris: []string{baseURI + "?sq-fake=" + tokenConflict}}}, - {StepId: "queue-a/2", Change: &changepb.Change{Uris: []string{baseURI + "?sq-fake=" + tokenInvalid}}}, + {StepId: "1", Change: &changepb.Change{Uris: []string{baseURI + "?sq-fake=" + tokenConflict}}}, + {StepId: "2", Change: &changepb.Change{Uris: []string{baseURI + "?sq-fake=" + tokenInvalid}}}, }, } diff --git a/runway/extension/merger/noop/noop_test.go b/runway/extension/merger/noop/noop_test.go index 876a1cc6f..c29df812d 100644 --- a/runway/extension/merger/noop/noop_test.go +++ b/runway/extension/merger/noop/noop_test.go @@ -34,16 +34,16 @@ var testCfg = merger.Config{QueueName: "test-queue"} func testRequest() *runwaymq.MergeRequest { return &runwaymq.MergeRequest{ - Id: "queue-a/42", + Id: "42", QueueName: "queue-a", Steps: []*runwaymq.MergeStep{ { - StepId: "queue-a/1", + StepId: "1", Change: &changepb.Change{Uris: []string{"github://github.example.com/uber/repo/pull/1/abcdef0123456789abcdef0123456789abcdef01"}}, Strategy: strategypb.Strategy_REBASE, }, { - StepId: "queue-a/2", + StepId: "2", Change: &changepb.Change{Uris: []string{"github://github.example.com/uber/repo/pull/2/89abcdef0123456789abcdef0123456789abcdef"}}, Strategy: strategypb.Strategy_MERGE, }, @@ -61,9 +61,9 @@ func TestCheckMergeability(t *testing.T) { assert.Equal(t, req.GetId(), res.GetId()) assert.Equal(t, runwaypb.Outcome_SUCCEEDED, res.GetOutcome()) require.Len(t, res.GetSteps(), 2) - assert.Equal(t, "queue-a/1", res.GetSteps()[0].GetStepId()) + assert.Equal(t, "1", res.GetSteps()[0].GetStepId()) assert.Empty(t, res.GetSteps()[0].GetOutputs()) - assert.Equal(t, "queue-a/2", res.GetSteps()[1].GetStepId()) + assert.Equal(t, "2", res.GetSteps()[1].GetStepId()) assert.Empty(t, res.GetSteps()[1].GetOutputs()) } @@ -77,10 +77,10 @@ func TestMerge(t *testing.T) { assert.Equal(t, req.GetId(), res.GetId()) assert.Equal(t, runwaypb.Outcome_SUCCEEDED, res.GetOutcome()) require.Len(t, res.GetSteps(), 2) - assert.Equal(t, "queue-a/1", res.GetSteps()[0].GetStepId()) + assert.Equal(t, "1", res.GetSteps()[0].GetStepId()) require.Len(t, res.GetSteps()[0].GetOutputs(), 1) assert.NotEmpty(t, res.GetSteps()[0].GetOutputs()[0].GetId()) - assert.Equal(t, "queue-a/2", res.GetSteps()[1].GetStepId()) + assert.Equal(t, "2", res.GetSteps()[1].GetStepId()) require.Len(t, res.GetSteps()[1].GetOutputs(), 1) assert.NotEmpty(t, res.GetSteps()[1].GetOutputs()[0].GetId()) } diff --git a/service/stovepipe/README.md b/service/stovepipe/README.md index 78668eef2..8b66bec6c 100644 --- a/service/stovepipe/README.md +++ b/service/stovepipe/README.md @@ -74,7 +74,7 @@ Attach with `.vscode/launch.json` (**Debug: attach (dlv in docker)**), then send grpcurl -plaintext -d '{"queue":"monorepo/main"}' localhost:PORT uber.submitqueue.stovepipe.Stovepipe/Ingest # Retained history by request ID -grpcurl -plaintext -d '{"queue":"monorepo/main","request_id":"request/monorepo/main/1"}' localhost:PORT uber.submitqueue.stovepipe.Stovepipe/GetRequestHistoryByID +grpcurl -plaintext -d '{"queue":"monorepo/main","request_id":"1"}' localhost:PORT uber.submitqueue.stovepipe.Stovepipe/GetRequestHistoryByID # Retained history by exact commit URI grpcurl -plaintext -d '{"queue":"monorepo/main","uri":"git://monorepo/main/HEAD"}' localhost:PORT uber.submitqueue.stovepipe.Stovepipe/GetRequestHistoryByURI diff --git a/service/stovepipe/server/BUILD.bazel b/service/stovepipe/server/BUILD.bazel index 1e015a8f1..90960f92e 100644 --- a/service/stovepipe/server/BUILD.bazel +++ b/service/stovepipe/server/BUILD.bazel @@ -16,6 +16,7 @@ go_library( "//platform/errs/yarpc:go_default_library", "//platform/extension/consumergate/noop:go_default_library", "//platform/extension/counter:go_default_library", + "//platform/extension/counter/mysql:go_default_library", "//platform/extension/hook:go_default_library", "//platform/extension/hook/noop:go_default_library", "//platform/extension/messagequeue:go_default_library", diff --git a/service/stovepipe/server/handler/server_test.go b/service/stovepipe/server/handler/server_test.go index ebec7715a..42c035e95 100644 --- a/service/stovepipe/server/handler/server_test.go +++ b/service/stovepipe/server/handler/server_test.go @@ -70,10 +70,10 @@ func TestGetRequestHistoryByID(t *testing.T) { srv := &StovepipeServer{requestHistoryController: fake} resp, err := srv.GetRequestHistoryByID(context.Background(), &pb.GetRequestHistoryByIDRequest{ - Queue: "monorepo/main", RequestId: "request/1", + Queue: "monorepo/main", RequestId: "1", }) - assert.Equal(t, entity.GetRequestHistoryByIDRequest{Queue: "monorepo/main", ID: "request/1"}, gotReq) + assert.Equal(t, entity.GetRequestHistoryByIDRequest{Queue: "monorepo/main", ID: "1"}, gotReq) if tt.err != nil { require.ErrorIs(t, err, tt.err) assert.Nil(t, resp) @@ -92,7 +92,7 @@ func TestGetRequestHistoryByID(t *testing.T) { func TestGetRequestHistoryByURI(t *testing.T) { controllerErr := errors.New("controller failed") histories := []entity.RequestHistory{{ - RequestID: "request/1", + RequestID: "1", Events: []entity.RequestLog{{ID: "occurrence/1", State: entity.RequestStateAccepted}}, }} tests := []struct { @@ -130,7 +130,7 @@ func TestGetRequestHistoryByURI(t *testing.T) { require.NoError(t, err) require.Len(t, resp.Histories, tt.wantHistories) if tt.wantHistories > 0 { - assert.Equal(t, "request/1", resp.Histories[0].RequestId) + assert.Equal(t, "1", resp.Histories[0].RequestId) require.Len(t, resp.Histories[0].Events, 1) assert.Equal(t, "accepted", resp.Histories[0].Events[0].GetRequestState()) } diff --git a/service/stovepipe/server/main.go b/service/stovepipe/server/main.go index 00e5811a6..dee404ef9 100644 --- a/service/stovepipe/server/main.go +++ b/service/stovepipe/server/main.go @@ -38,6 +38,7 @@ import ( yarpcerrs "github.com/uber/submitqueue/platform/errs/yarpc" consumergatenoop "github.com/uber/submitqueue/platform/extension/consumergate/noop" "github.com/uber/submitqueue/platform/extension/counter" + mysqlcounter "github.com/uber/submitqueue/platform/extension/counter/mysql" hookext "github.com/uber/submitqueue/platform/extension/hook" hooknoop "github.com/uber/submitqueue/platform/extension/hook/noop" extqueue "github.com/uber/submitqueue/platform/extension/messagequeue" @@ -66,54 +67,6 @@ import ( "google.golang.org/grpc/reflection" ) -// inMemoryCounter is a minimal, process-local counter.Counter used to wire the example -// server. It is not durable; a real deployment supplies a persistent implementation -// (e.g. platform/extension/counter/mysql). -type inMemoryCounter struct { - mu sync.Mutex - values map[string]int64 - queue string -} - -func newInMemoryCounter(queue string) *inMemoryCounter { - return &inMemoryCounter{values: make(map[string]int64), queue: queue} -} - -// Next returns the next value in the sequence for the given domain, starting at 1. -func (c *inMemoryCounter) Next(_ context.Context, domain string) (int64, error) { - c.mu.Lock() - defer c.mu.Unlock() - key := c.queue + "\x00" + domain - c.values[key]++ - return c.values[key], nil -} - -// inMemoryCounterFactory is the example counter.Factory: one process-local sequence -// set per queue, minted on first use. A real deployment supplies a persistent factory. -type inMemoryCounterFactory struct { - mu sync.Mutex - counters map[string]*inMemoryCounter -} - -func newInMemoryCounterFactory() *inMemoryCounterFactory { - return &inMemoryCounterFactory{counters: make(map[string]*inMemoryCounter)} -} - -// For returns the process-local Counter bound to the queue named in config. -func (f *inMemoryCounterFactory) For(config counter.Config) (counter.Counter, error) { - if config.QueueName == "" { - return nil, fmt.Errorf("queue name must not be empty") - } - f.mu.Lock() - defer f.mu.Unlock() - if existing, ok := f.counters[config.QueueName]; ok { - return existing, nil - } - created := newInMemoryCounter(config.QueueName) - f.counters[config.QueueName] = created - return created, nil -} - // fakeSourceControlFactory is the example SourceControl factory. It seeds each queue with a // deterministic single-commit history so ingest resolves a stable head URI (and re-ingesting // the same queue exercises the dedup path). It has no ref to promote onto, so a promotion @@ -324,7 +277,7 @@ func run() error { ingestController := controller.NewIngestController( logger.Sugar(), scope, - newInMemoryCounterFactory(), + counterFactory{db: storageDB, scope: scope.SubScope("counter")}, sourceControl, storageFty, materializer, @@ -596,3 +549,17 @@ type storageFactory struct { func (f storageFactory) For(config storage.Config) (storage.Storage, error) { return f.backend.For(config.QueueName) } + +// counterFactory binds a queue to Stovepipe's MySQL backend. +type counterFactory struct { + db *sql.DB + scope tally.Scope +} + +// For returns the durable Counter bound to the queue named in config. +func (f counterFactory) For(config counter.Config) (counter.Counter, error) { + if config.QueueName == "" { + return nil, fmt.Errorf("queue name must not be empty") + } + return mysqlcounter.NewCounter(f.db, f.scope, config.QueueName), nil +} diff --git a/service/stovepipe/server/mapper/ingest_test.go b/service/stovepipe/server/mapper/ingest_test.go index 2c715f682..3861049b7 100644 --- a/service/stovepipe/server/mapper/ingest_test.go +++ b/service/stovepipe/server/mapper/ingest_test.go @@ -56,8 +56,8 @@ func TestIngestResultToProto(t *testing.T) { }{ { name: "maps ID", - result: entity.IngestResult{ID: "request/monorepo/main/7"}, - expected: &pb.IngestResponse{Id: "request/monorepo/main/7"}, + result: entity.IngestResult{ID: "7"}, + expected: &pb.IngestResponse{Id: "7"}, }, { name: "zero value result yields empty response", diff --git a/service/stovepipe/server/mapper/project_status_test.go b/service/stovepipe/server/mapper/project_status_test.go index 6cb464671..fed19c252 100644 --- a/service/stovepipe/server/mapper/project_status_test.go +++ b/service/stovepipe/server/mapper/project_status_test.go @@ -32,13 +32,13 @@ func TestProjectStatusRequestAndResponseMapping(t *testing.T) { degree := entity.DegreeGreen response := GetProjectStatusByURIResultToProto(entity.GetProjectStatusByURIResult{ - RequestSummary: entity.RequestSummary{RequestID: "request/1", Queue: "queue", URI: "uri", BaseURI: "base", State: entity.RequestStateSucceeded}, + RequestSummary: entity.RequestSummary{RequestID: "1", Queue: "queue", URI: "uri", BaseURI: "base", State: entity.RequestStateSucceeded}, RepositoryValidationFact: entity.ValidationFact{Degree: degree}, HasRepositoryValidationFact: true, ProjectValidationFacts: []entity.ValidationFact{{Project: "project-a", Degree: entity.DegreeBroken}}, NextPageToken: "next-token", }) - assert.Equal(t, "request/1", response.GetRequestId()) + assert.Equal(t, "1", response.GetRequestId()) assert.Equal(t, "succeeded", response.GetRequestState()) assert.Equal(t, degree, response.GetRepositoryBreakageDegree()) assert.IsType(t, &pb.GetProjectStatusByURIResponse_RepositoryBreakageDegree{}, response.GetRepositoryResult()) diff --git a/service/stovepipe/server/mapper/request_history_test.go b/service/stovepipe/server/mapper/request_history_test.go index 6292b4651..61add03cb 100644 --- a/service/stovepipe/server/mapper/request_history_test.go +++ b/service/stovepipe/server/mapper/request_history_test.go @@ -30,8 +30,8 @@ func TestProtoToGetRequestHistoryByIDRequest(t *testing.T) { }{ { name: "maps selector", - req: &pb.GetRequestHistoryByIDRequest{Queue: "monorepo/main", RequestId: "request/monorepo/main/1"}, - want: entity.GetRequestHistoryByIDRequest{Queue: "monorepo/main", ID: "request/monorepo/main/1"}, + req: &pb.GetRequestHistoryByIDRequest{Queue: "monorepo/main", RequestId: "1"}, + want: entity.GetRequestHistoryByIDRequest{Queue: "monorepo/main", ID: "1"}, }, { name: "empty request yields zero value", @@ -147,19 +147,19 @@ func TestRequestHistoriesToProto(t *testing.T) { { name: "maps groups in order", histories: []entity.RequestHistory{ - {RequestID: "request/monorepo/main/1", Events: logs}, - {RequestID: "request/monorepo/main/2", Events: []entity.RequestLog{}}, + {RequestID: "1", Events: logs}, + {RequestID: "2", Events: []entity.RequestLog{}}, }, want: []*pb.RequestHistory{ { - RequestId: "request/monorepo/main/1", + RequestId: "1", Events: []*pb.HistoryEvent{{ EventId: "state/1", TimestampMs: 10, Occurrence: &pb.HistoryEvent_RequestState{RequestState: "accepted"}, }}, }, - {RequestId: "request/monorepo/main/2", Events: []*pb.HistoryEvent{}}, + {RequestId: "2", Events: []*pb.HistoryEvent{}}, }, }, {name: "nil input", histories: nil, want: []*pb.RequestHistory{}}, diff --git a/service/submitqueue/gateway/client/main.go b/service/submitqueue/gateway/client/main.go index ebef5b45b..aa88c79b3 100644 --- a/service/submitqueue/gateway/client/main.go +++ b/service/submitqueue/gateway/client/main.go @@ -63,7 +63,7 @@ Examples: client land -queue my-queue -pr https://github.com/uber/sq-sandbox/pull/7 client land -queue my-queue -uri github://github.com/uber/r/pull/7/ -strategy SQUASH_REBASE client land -queue my-queue -pr -pr - client status -queue my-queue -sqid my-queue/12 + client status -queue my-queue -sqid 12 client list -queue my-queue -since 1h client watch -queue my-queue client -addr sq.example.com:443 -tls list -queue my-queue diff --git a/service/submitqueue/gateway/server/main_test.go b/service/submitqueue/gateway/server/main_test.go index a86781061..2fc6cce17 100644 --- a/service/submitqueue/gateway/server/main_test.go +++ b/service/submitqueue/gateway/server/main_test.go @@ -107,7 +107,7 @@ func TestGatewayStatusError(t *testing.T) { }, { name: "request not found", - err: &controller.RequestNotFoundError{Sqid: "queue/1"}, + err: &controller.RequestNotFoundError{Sqid: "1"}, code: codes.NotFound, }, { diff --git a/service/submitqueue/gateway/server/mapper/cancel_test.go b/service/submitqueue/gateway/server/mapper/cancel_test.go index cc9525663..edd0f03d5 100644 --- a/service/submitqueue/gateway/server/mapper/cancel_test.go +++ b/service/submitqueue/gateway/server/mapper/cancel_test.go @@ -30,13 +30,13 @@ func TestProtoToCancelRequest(t *testing.T) { }{ { name: "maps sqid, queue and reason", - req: &pb.CancelRequest{Sqid: "test-queue/42", Queue: "test-queue", Reason: "obsolete change"}, - expected: entity.CancelRequest{ID: "test-queue/42", Queue: "test-queue", Reason: "obsolete change"}, + req: &pb.CancelRequest{Sqid: "42", Queue: "test-queue", Reason: "obsolete change"}, + expected: entity.CancelRequest{ID: "42", Queue: "test-queue", Reason: "obsolete change"}, }, { name: "maps sqid and queue without reason", - req: &pb.CancelRequest{Sqid: "test-queue/1", Queue: "test-queue"}, - expected: entity.CancelRequest{ID: "test-queue/1", Queue: "test-queue"}, + req: &pb.CancelRequest{Sqid: "1", Queue: "test-queue"}, + expected: entity.CancelRequest{ID: "1", Queue: "test-queue"}, }, { name: "empty request yields zero value", diff --git a/service/submitqueue/gateway/server/mapper/list_test.go b/service/submitqueue/gateway/server/mapper/list_test.go index 0c9e62610..dc9e77a38 100644 --- a/service/submitqueue/gateway/server/mapper/list_test.go +++ b/service/submitqueue/gateway/server/mapper/list_test.go @@ -43,7 +43,7 @@ func TestProtoToListRequest(t *testing.T) { func TestListResultToProto(t *testing.T) { result := entity.ListResult{ Requests: []entity.RequestQueueSummary{{ - RequestID: "q/1", + RequestID: "1", Queue: "q", ChangeURIs: []string{"github://uber/repo/pull/1/abc"}, ReceivedAtMs: 100, @@ -55,6 +55,6 @@ func TestListResultToProto(t *testing.T) { response := ListResultToProto(result) - assert.Equal(t, "q/1", response.Requests[0].Sqid) + assert.Equal(t, "1", response.Requests[0].Sqid) assert.Equal(t, "next", response.NextPageToken) } diff --git a/service/submitqueue/gateway/server/mapper/request_history_test.go b/service/submitqueue/gateway/server/mapper/request_history_test.go index 53c14e257..9f90bb846 100644 --- a/service/submitqueue/gateway/server/mapper/request_history_test.go +++ b/service/submitqueue/gateway/server/mapper/request_history_test.go @@ -24,8 +24,8 @@ import ( func TestProtoToGetRequestHistoryRequests(t *testing.T) { assert.Equal(t, - entity.GetRequestHistoryByIDRequest{ID: "q/1", Queue: "q"}, - ProtoToGetRequestHistoryByIDRequest(&pb.GetRequestHistoryByIDRequest{Sqid: "q/1", Queue: "q"}), + entity.GetRequestHistoryByIDRequest{ID: "1", Queue: "q"}, + ProtoToGetRequestHistoryByIDRequest(&pb.GetRequestHistoryByIDRequest{Sqid: "1", Queue: "q"}), ) assert.Equal(t, entity.GetRequestHistoryByChangeURIRequest{ChangeURI: "uri", Queue: "q"}, @@ -51,10 +51,10 @@ func TestHistoryEventsToProto(t *testing.T) { func TestRequestHistoriesToProto(t *testing.T) { histories := RequestHistoriesToProto([]entity.RequestHistory{{ - RequestID: "q/1", + RequestID: "1", Events: []entity.RequestLog{{TimestampMs: 10, Status: entity.RequestStatusAccepted}}, }}) - assert.Equal(t, "q/1", histories[0].Sqid) + assert.Equal(t, "1", histories[0].Sqid) assert.Equal(t, string(entity.RequestStatusAccepted), histories[0].Events[0].Status) } diff --git a/service/submitqueue/gateway/server/mapper/request_summary_test.go b/service/submitqueue/gateway/server/mapper/request_summary_test.go index 275df3757..3c9e838bd 100644 --- a/service/submitqueue/gateway/server/mapper/request_summary_test.go +++ b/service/submitqueue/gateway/server/mapper/request_summary_test.go @@ -24,8 +24,8 @@ import ( func TestProtoToGetRequestSummaryByIDRequest(t *testing.T) { assert.Equal(t, - entity.GetRequestSummaryByIDRequest{ID: "test-queue/42", Queue: "test-queue"}, - ProtoToGetRequestSummaryByIDRequest(&pb.GetRequestSummaryByIDRequest{Sqid: "test-queue/42", Queue: "test-queue"}), + entity.GetRequestSummaryByIDRequest{ID: "42", Queue: "test-queue"}, + ProtoToGetRequestSummaryByIDRequest(&pb.GetRequestSummaryByIDRequest{Sqid: "42", Queue: "test-queue"}), ) } @@ -38,7 +38,7 @@ func TestProtoToGetRequestSummaryByChangeURIRequest(t *testing.T) { func TestRequestSummaryToProto(t *testing.T) { summary := entity.RequestSummary{ - RequestID: "test-queue/42", + RequestID: "42", Queue: "test-queue", ChangeURIs: []string{"github://uber/repo/pull/1/abc"}, ReceivedAtMs: 100, @@ -48,7 +48,7 @@ func TestRequestSummaryToProto(t *testing.T) { } assert.Equal(t, &pb.RequestSummary{ - Sqid: "test-queue/42", + Sqid: "42", Queue: "test-queue", ChangeUris: []string{"github://uber/repo/pull/1/abc"}, ReceivedAtMs: 100, @@ -60,11 +60,11 @@ func TestRequestSummaryToProto(t *testing.T) { func TestRequestSummariesToProto(t *testing.T) { summaries := []entity.RequestSummary{ - {RequestID: "test-queue/2", Status: entity.RequestStatusLanded}, - {RequestID: "test-queue/1", Status: entity.RequestStatusError}, + {RequestID: "2", Status: entity.RequestStatusLanded}, + {RequestID: "1", Status: entity.RequestStatusError}, } requests := RequestSummariesToProto(summaries) - assert.Equal(t, []string{"test-queue/2", "test-queue/1"}, []string{requests[0].Sqid, requests[1].Sqid}) + assert.Equal(t, []string{"2", "1"}, []string{requests[0].Sqid, requests[1].Sqid}) } diff --git a/service/submitqueue/orchestrator/server/config_test.go b/service/submitqueue/orchestrator/server/config_test.go index b6e0a44e6..45b923306 100644 --- a/service/submitqueue/orchestrator/server/config_test.go +++ b/service/submitqueue/orchestrator/server/config_test.go @@ -262,8 +262,8 @@ queues: // result to what distinguishes the three kinds from each other. func analyzeOutcome(t *testing.T, analyzer conflict.Analyzer) (conflicts int, failed bool) { t.Helper() - batch := entity.Batch{ID: "q/batch/2", Queue: "q"} - inFlight := []entity.Batch{{ID: "q/batch/1", Queue: "q"}} + batch := entity.Batch{ID: "2", Queue: "q"} + inFlight := []entity.Batch{{ID: "1", Queue: "q"}} found, err := analyzer.Analyze(context.Background(), batch, inFlight) if err != nil { diff --git a/stovepipe/controller/BUILD.bazel b/stovepipe/controller/BUILD.bazel index 9a0a3f2e0..38ff0fb08 100644 --- a/stovepipe/controller/BUILD.bazel +++ b/stovepipe/controller/BUILD.bazel @@ -13,6 +13,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//api/stovepipe/protopb:go_default_library", + "//platform/base/id:go_default_library", "//platform/consumer:go_default_library", "//platform/errs:go_default_library", "//platform/extension/counter:go_default_library", diff --git a/stovepipe/controller/build/build_test.go b/stovepipe/controller/build/build_test.go index 905f41dd8..bba48204d 100644 --- a/stovepipe/controller/build/build_test.go +++ b/stovepipe/controller/build/build_test.go @@ -42,7 +42,7 @@ import ( const ( testQueue = "monorepo/main" - testID = "request/monorepo/main/7" + testID = "7" testHeadURI = "git://repo/monorepo/main/head" testBaseURI = "git://repo/monorepo/main/base" testBuildID = "bk-1" diff --git a/stovepipe/controller/buildsignal/buildsignal_test.go b/stovepipe/controller/buildsignal/buildsignal_test.go index c82827d46..aac4eecf7 100644 --- a/stovepipe/controller/buildsignal/buildsignal_test.go +++ b/stovepipe/controller/buildsignal/buildsignal_test.go @@ -42,7 +42,7 @@ import ( const ( testQueue = "monorepo/main" - testID = "request/monorepo/main/7" + testID = "7" testBuildID = "bk-1" ) diff --git a/stovepipe/controller/dlq/dlq_test.go b/stovepipe/controller/dlq/dlq_test.go index 404f020ec..eeb7c838c 100644 --- a/stovepipe/controller/dlq/dlq_test.go +++ b/stovepipe/controller/dlq/dlq_test.go @@ -38,7 +38,7 @@ import ( const ( testQueue = "monorepo/main" - testID = "request/monorepo/main/7" + testID = "7" ) func queueContext() context.Context { diff --git a/stovepipe/controller/get_project_status_by_uri_test.go b/stovepipe/controller/get_project_status_by_uri_test.go index 03ce84f99..dbe04e662 100644 --- a/stovepipe/controller/get_project_status_by_uri_test.go +++ b/stovepipe/controller/get_project_status_by_uri_test.go @@ -34,7 +34,7 @@ import ( const ( projectStatusQueue = "monorepo/main" projectStatusURI = "git://monorepo/main/abc" - projectStatusID = "request/monorepo/main/7" + projectStatusID = "7" ) func TestGetProjectStatusByURI(t *testing.T) { diff --git a/stovepipe/controller/ingest.go b/stovepipe/controller/ingest.go index dc7b09358..0208a5bfb 100644 --- a/stovepipe/controller/ingest.go +++ b/stovepipe/controller/ingest.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/uber-go/tally" + "github.com/uber/submitqueue/platform/base/id" "github.com/uber/submitqueue/platform/consumer" "github.com/uber/submitqueue/platform/extension/counter" "github.com/uber/submitqueue/platform/metrics" @@ -32,9 +33,7 @@ import ( "go.uber.org/zap" ) -// counterDomainRequest names the per-queue sequence that mints request IDs. It also -// happens to be the leading segment of the ID, but the two are written independently -// (see resolveID) so they cannot drift into each other. +// counterDomainRequest names the per-queue sequence that mints request IDs. const counterDomainRequest = "request" // IngestController handles ingest business logic for stovepipe: it admits a queue's newly @@ -183,10 +182,7 @@ func (c *IngestController) resolveID(ctx context.Context, store storage.Storage, return "", fmt.Errorf("failed to look up existing request for queue=%s: %w", queue, err) } - // Mint a globally unique request ID namespaced by the queue. The ID format - // ("request//") is written out here rather than derived from the counter - // domain: the domain is a per-queue sequence name only, and the two must stay independent - // so re-keying the counter cannot change the emitted ID. + // Mint a request ID within the queue's request sequence. queueCounter, err := c.counters.For(counter.Config{QueueName: queue}) if err != nil { return "", fmt.Errorf("failed to resolve counter for queue=%s: %w", queue, err) @@ -195,9 +191,12 @@ func (c *IngestController) resolveID(ctx context.Context, store storage.Storage, if err != nil { return "", fmt.Errorf("failed to generate request ID for queue=%s: %w", queue, err) } - id := fmt.Sprintf("%s/%s/%d", counterDomainRequest, queue, seq) + requestID, err := id.FromCounter(seq) + if err != nil { + return "", fmt.Errorf("generated invalid request ID for queue=%s: %w", queue, err) + } - if err := uriStore.Create(ctx, uri, id); err != nil { + if err := uriStore.Create(ctx, uri, requestID); err != nil { if errors.Is(err, storage.ErrAlreadyExists) { existing, getErr := uriStore.GetIDByURI(ctx, uri) if getErr != nil { @@ -207,7 +206,7 @@ func (c *IngestController) resolveID(ctx context.Context, store storage.Storage, } return "", fmt.Errorf("failed to map URI for queue=%s: %w", queue, err) } - return id, nil + return requestID, nil } // ensureRequest returns the request for id, creating it in the Accepted state if it does not yet @@ -278,7 +277,7 @@ func (c *IngestController) advanceQueueLatestRequestID(ctx context.Context, stor return err } if queueRow.LatestRequestID != "" { - cmp, err := entity.CompareRequestID(queue, id, queueRow.LatestRequestID) + cmp, err := entity.CompareRequestID(id, queueRow.LatestRequestID) if err != nil { return fmt.Errorf("failed to compare request ids for queue %s: %w", queue, err) } diff --git a/stovepipe/controller/ingest_test.go b/stovepipe/controller/ingest_test.go index 74492d19d..78283488d 100644 --- a/stovepipe/controller/ingest_test.go +++ b/stovepipe/controller/ingest_test.go @@ -153,7 +153,7 @@ func TestPublishProcessCarriesQueueMetadata(t *testing.T) { return nil }) - require.NoError(t, c.publishProcess(context.Background(), "request/monorepo/main/7", testQueue)) + require.NoError(t, c.publishProcess(context.Background(), "7", testQueue)) assert.Equal(t, testQueue, got.PartitionKey) assert.Equal(t, testQueue, got.Tenant) assert.Equal(t, testQueue, got.Metadata[entityqueue.MetadataKeyQueueName]) @@ -175,41 +175,41 @@ func TestIngestController_Ingest(t *testing.T) { expectResolve(m) m.uriStore.EXPECT().GetIDByURI(gomock.Any(), testURI).Return("", storage.ErrNotFound) m.counter.EXPECT().Next(gomock.Any(), counterDomainRequest).Return(int64(7), nil) - m.uriStore.EXPECT().Create(gomock.Any(), testURI, "request/monorepo/main/7").Return(nil) - m.reqStore.EXPECT().Get(gomock.Any(), "request/monorepo/main/7").Return(entity.Request{}, storage.ErrNotFound) + m.uriStore.EXPECT().Create(gomock.Any(), testURI, "7").Return(nil) + m.reqStore.EXPECT().Get(gomock.Any(), "7").Return(entity.Request{}, storage.ErrNotFound) m.reqStore.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil) - expectMaterializeAccepted(m, "request/monorepo/main/7") - expectAdvanceLatestRequestID(m, testQueue, "request/monorepo/main/7") + expectMaterializeAccepted(m, "7") + expectAdvanceLatestRequestID(m, testQueue, "7") m.publisher.EXPECT().Publish(gomock.Any(), "process", gomock.Any()).Return(nil) }, - wantID: "request/monorepo/main/7", + wantID: "7", }, { name: "dedup with existing accepted request republishes without minting", queue: testQueue, setup: func(m ingestMocks) { expectResolve(m) - m.uriStore.EXPECT().GetIDByURI(gomock.Any(), testURI).Return("request/monorepo/main/3", nil) - m.reqStore.EXPECT().Get(gomock.Any(), "request/monorepo/main/3").Return(acceptedRequest("request/monorepo/main/3"), nil) - expectMaterializeAccepted(m, "request/monorepo/main/3") - expectAdvanceLatestRequestIDNoOp(m, testQueue, "request/monorepo/main/3") + m.uriStore.EXPECT().GetIDByURI(gomock.Any(), testURI).Return("3", nil) + m.reqStore.EXPECT().Get(gomock.Any(), "3").Return(acceptedRequest("3"), nil) + expectMaterializeAccepted(m, "3") + expectAdvanceLatestRequestIDNoOp(m, testQueue, "3") m.publisher.EXPECT().Publish(gomock.Any(), "process", gomock.Any()).Return(nil) }, - wantID: "request/monorepo/main/3", + wantID: "3", }, { name: "heals when uri mapped but request missing", queue: testQueue, setup: func(m ingestMocks) { expectResolve(m) - m.uriStore.EXPECT().GetIDByURI(gomock.Any(), testURI).Return("request/monorepo/main/3", nil) - m.reqStore.EXPECT().Get(gomock.Any(), "request/monorepo/main/3").Return(entity.Request{}, storage.ErrNotFound) + m.uriStore.EXPECT().GetIDByURI(gomock.Any(), testURI).Return("3", nil) + m.reqStore.EXPECT().Get(gomock.Any(), "3").Return(entity.Request{}, storage.ErrNotFound) m.reqStore.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil) - expectMaterializeAccepted(m, "request/monorepo/main/3") - expectAdvanceLatestRequestID(m, testQueue, "request/monorepo/main/3") + expectMaterializeAccepted(m, "3") + expectAdvanceLatestRequestID(m, testQueue, "3") m.publisher.EXPECT().Publish(gomock.Any(), "process", gomock.Any()).Return(nil) }, - wantID: "request/monorepo/main/3", + wantID: "3", }, { name: "dedup race returns winner id and completes", @@ -218,14 +218,14 @@ func TestIngestController_Ingest(t *testing.T) { expectResolve(m) m.uriStore.EXPECT().GetIDByURI(gomock.Any(), testURI).Return("", storage.ErrNotFound) m.counter.EXPECT().Next(gomock.Any(), counterDomainRequest).Return(int64(7), nil) - m.uriStore.EXPECT().Create(gomock.Any(), testURI, "request/monorepo/main/7").Return(storage.ErrAlreadyExists) - m.uriStore.EXPECT().GetIDByURI(gomock.Any(), testURI).Return("request/monorepo/main/3", nil) - m.reqStore.EXPECT().Get(gomock.Any(), "request/monorepo/main/3").Return(acceptedRequest("request/monorepo/main/3"), nil) - expectMaterializeAccepted(m, "request/monorepo/main/3") - expectAdvanceLatestRequestIDNoOp(m, testQueue, "request/monorepo/main/3") + m.uriStore.EXPECT().Create(gomock.Any(), testURI, "7").Return(storage.ErrAlreadyExists) + m.uriStore.EXPECT().GetIDByURI(gomock.Any(), testURI).Return("3", nil) + m.reqStore.EXPECT().Get(gomock.Any(), "3").Return(acceptedRequest("3"), nil) + expectMaterializeAccepted(m, "3") + expectAdvanceLatestRequestIDNoOp(m, testQueue, "3") m.publisher.EXPECT().Publish(gomock.Any(), "process", gomock.Any()).Return(nil) }, - wantID: "request/monorepo/main/3", + wantID: "3", }, { name: "empty queue is invalid", @@ -270,6 +270,16 @@ func TestIngestController_Ingest(t *testing.T) { }, wantErr: true, }, + { + name: "non-positive counter value", + queue: testQueue, + setup: func(m ingestMocks) { + expectResolve(m) + m.uriStore.EXPECT().GetIDByURI(gomock.Any(), testURI).Return("", storage.ErrNotFound) + m.counter.EXPECT().Next(gomock.Any(), gomock.Any()).Return(int64(0), nil) + }, + wantErr: true, + }, { name: "request store create error", queue: testQueue, @@ -293,8 +303,8 @@ func TestIngestController_Ingest(t *testing.T) { m.uriStore.EXPECT().Create(gomock.Any(), testURI, gomock.Any()).Return(nil) m.reqStore.EXPECT().Get(gomock.Any(), gomock.Any()).Return(entity.Request{}, storage.ErrNotFound) m.reqStore.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil) - expectMaterializeAccepted(m, "request/monorepo/main/7") - expectAdvanceLatestRequestID(m, testQueue, "request/monorepo/main/7") + expectMaterializeAccepted(m, "7") + expectAdvanceLatestRequestID(m, testQueue, "7") m.publisher.EXPECT().Publish(gomock.Any(), "process", gomock.Any()).Return(errors.New("queue down")) }, wantErr: true, @@ -304,12 +314,12 @@ func TestIngestController_Ingest(t *testing.T) { queue: testQueue, setup: func(m ingestMocks) { expectResolve(m) - m.uriStore.EXPECT().GetIDByURI(gomock.Any(), testURI).Return("request/monorepo/main/3", nil) - m.reqStore.EXPECT().Get(gomock.Any(), "request/monorepo/main/3").Return(acceptedRequest("request/monorepo/main/3"), nil) + m.uriStore.EXPECT().GetIDByURI(gomock.Any(), testURI).Return("3", nil) + m.reqStore.EXPECT().Get(gomock.Any(), "3").Return(acceptedRequest("3"), nil) m.materializer.EXPECT().PersistLog( gomock.Any(), m.store, - requestlog.NewRequestStateLog(acceptedRequest("request/monorepo/main/3"), entity.RequestOutcomeReasonUnknown), + requestlog.NewRequestStateLog(acceptedRequest("3"), entity.RequestOutcomeReasonUnknown), ).Return(errors.New("log unavailable")) }, wantErr: true, diff --git a/stovepipe/controller/process/process.go b/stovepipe/controller/process/process.go index 991899ea0..f334f70bd 100644 --- a/stovepipe/controller/process/process.go +++ b/stovepipe/controller/process/process.go @@ -198,7 +198,7 @@ func (c *Controller) processAccepted(ctx context.Context, store storage.Storage, // true so the caller acks. It returns false when request is still the latest head and // should proceed to the gate. Superseding consumes no build slot. func (c *Controller) coalesce(ctx context.Context, store storage.Storage, request entity.Request, latestRequestID string) (bool, error) { - cmp, err := entity.CompareRequestID(request.Queue, request.ID, latestRequestID) + cmp, err := entity.CompareRequestID(request.ID, latestRequestID) if err != nil { return false, fmt.Errorf("failed to compare request ids for queue %s: %w", request.Queue, err) } diff --git a/stovepipe/controller/process/process_test.go b/stovepipe/controller/process/process_test.go index 5020a7a96..cd8ef4c61 100644 --- a/stovepipe/controller/process/process_test.go +++ b/stovepipe/controller/process/process_test.go @@ -45,8 +45,8 @@ import ( const ( testQueue = "monorepo/main" - testID = "request/monorepo/main/7" - testOlderID = "request/monorepo/main/3" + testID = "7" + testOlderID = "3" testURI = "git://repo/monorepo/main/abc123" ) @@ -797,7 +797,7 @@ func TestProcess(t *testing.T) { }, int32(1), int32(2)).Return(storage.ErrVersionMismatch) // Reload: ingest stamped a newer head — our head is no longer latest. m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ - Name: testQueue, LatestRequestID: "request/monorepo/main/9", Version: 2, + Name: testQueue, LatestRequestID: "9", Version: 2, }, nil) superseded := acceptedRequest(testID) superseded.State = entity.RequestStateSuperseded @@ -964,7 +964,7 @@ func TestProcess(t *testing.T) { m.reqStore.EXPECT().Get(gomock.Any(), testID).Return(acceptedRequest(testID), nil) m.queueStore.EXPECT().Get(gomock.Any(), testQueue).Return(entity.Queue{ Name: testQueue, - LatestRequestID: "request/other-queue/99", + LatestRequestID: "request.99", Version: 1, }, nil) }, diff --git a/stovepipe/controller/read_errors.go b/stovepipe/controller/read_errors.go index 127fa436f..5271c4bb3 100644 --- a/stovepipe/controller/read_errors.go +++ b/stovepipe/controller/read_errors.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" + "github.com/uber/submitqueue/platform/base/id" "github.com/uber/submitqueue/platform/errs" ) @@ -41,6 +42,13 @@ func validateHistoryIdentifier(name, value string) error { return nil } +func validateRequestID(value string) error { + if err := id.Validate(value); err != nil { + return fmt.Errorf("invalid request ID %q: %v: %w", value, err, ErrInvalidRequest) + } + return nil +} + // RequestHistoryByIDNotFoundError indicates that no retained history exists for a request ID. type RequestHistoryByIDNotFoundError struct { // RequestID is the selected request identifier. diff --git a/stovepipe/controller/record/record.go b/stovepipe/controller/record/record.go index a4bd64165..f3cb9956e 100644 --- a/stovepipe/controller/record/record.go +++ b/stovepipe/controller/record/record.go @@ -623,7 +623,7 @@ func compareToBookmark(queue, candidate, current string) (int, error) { if current == "" { return 1, nil } - cmp, err := entity.CompareRequestID(queue, candidate, current) + cmp, err := entity.CompareRequestID(candidate, current) if err != nil { return 0, fmt.Errorf("failed to compare request ids for queue %s: %w", queue, err) } diff --git a/stovepipe/controller/record/record_test.go b/stovepipe/controller/record/record_test.go index 0559c6a80..871cdeda4 100644 --- a/stovepipe/controller/record/record_test.go +++ b/stovepipe/controller/record/record_test.go @@ -48,7 +48,7 @@ import ( const ( testQueue = "monorepo/main" - testID = "request/monorepo/main/7" + testID = "7" testURI = "git://remote/monorepo/main/head-sha" testBaseURI = "git://remote/monorepo/main/base-sha" ) @@ -288,7 +288,7 @@ func TestProcess_AdvancesBookmarkOnSuccess(t *testing.T) { }, { name: "stored bookmark is older", - stored: queueRow("git://remote/monorepo/main/old", "request/monorepo/main/3", 4), + stored: queueRow("git://remote/monorepo/main/old", "3", 4), wantURI: testURI, }, } @@ -728,7 +728,7 @@ func TestProcess_ExistingFactFromDifferentRequestFails(t *testing.T) { Return(requestWithState(entity.RequestStateSucceeded), nil) m.factStore.EXPECT().Create(gomock.Any(), gomock.Any()).Return(storage.ErrAlreadyExists) m.factStore.EXPECT().Get(gomock.Any(), testURI, wholeRepositoryProject). - Return(entity.ValidationFact{URI: testURI, Degree: entity.DegreeGreen, RequestID: "request/monorepo/main/1"}, nil) + Return(entity.ValidationFact{URI: testURI, Degree: entity.DegreeGreen, RequestID: "1"}, nil) // The bookmark must not move on an identity this request does not own. require.Error(t, c.Process(queueContext(), delivery(t, ctrl, recordPayload(t, testID)))) @@ -751,7 +751,7 @@ func TestProcess_SkipsBookmarkWhenNotNewer(t *testing.T) { // A newer green commit owns the bookmark and the ref, so promoting // this one would move the ref backwards. name: "stored bookmark is newer", - stored: queueRow("git://remote/monorepo/main/newer", "request/monorepo/main/9", 5), + stored: queueRow("git://remote/monorepo/main/newer", "9", 5), }, } diff --git a/stovepipe/controller/request_history.go b/stovepipe/controller/request_history.go index cc79cb973..4f320b968 100644 --- a/stovepipe/controller/request_history.go +++ b/stovepipe/controller/request_history.go @@ -70,7 +70,7 @@ func (c *requestHistoryController) readHistoryByID(ctx context.Context, req enti if err := validateHistoryIdentifier("queue", req.Queue); err != nil { return nil, fmt.Errorf("GetRequestHistoryByID invalid queue=%q: %w", req.Queue, err) } - if err := validateHistoryIdentifier("request ID", req.ID); err != nil { + if err := validateRequestID(req.ID); err != nil { return nil, fmt.Errorf("GetRequestHistoryByID invalid request_id=%q queue=%q: %w", req.ID, req.Queue, err) } diff --git a/stovepipe/controller/request_history_test.go b/stovepipe/controller/request_history_test.go index 866d41a83..3ad919674 100644 --- a/stovepipe/controller/request_history_test.go +++ b/stovepipe/controller/request_history_test.go @@ -37,7 +37,7 @@ import ( func TestGetRequestHistoryByID(t *testing.T) { const ( queue = "monorepo/main" - requestID = "request/monorepo/main/42" + requestID = "42" ) backendErr := errors.New("backend unavailable") ordered := []entity.RequestLog{ @@ -69,6 +69,7 @@ func TestGetRequestHistoryByID(t *testing.T) { {name: "empty queue", req: entity.GetRequestHistoryByIDRequest{ID: requestID}, wantInvalid: true, wantUser: true}, {name: "oversized queue", req: entity.GetRequestHistoryByIDRequest{Queue: strings.Repeat("q", maxHistoryIdentifierBytes+1), ID: requestID}, wantInvalid: true, wantUser: true}, {name: "empty request ID", req: entity.GetRequestHistoryByIDRequest{Queue: queue}, wantInvalid: true, wantUser: true}, + {name: "composite request ID", req: entity.GetRequestHistoryByIDRequest{Queue: queue, ID: "request/monorepo/main/42"}, wantInvalid: true, wantUser: true}, {name: "oversized request ID", req: entity.GetRequestHistoryByIDRequest{Queue: queue, ID: strings.Repeat("r", maxHistoryIdentifierBytes+1)}, wantInvalid: true, wantUser: true}, {name: "storage factory failure", req: entity.GetRequestHistoryByIDRequest{Queue: queue, ID: requestID}, factoryErr: backendErr, wantCause: backendErr}, {name: "history not found", req: entity.GetRequestHistoryByIDRequest{Queue: queue, ID: requestID}, listErr: fmt.Errorf("query: %w", storage.ErrNotFound), wantNotFound: true, wantUser: true}, @@ -139,7 +140,7 @@ func TestGetRequestHistoryByURI(t *testing.T) { const ( queue = "monorepo/main" uri = "git://example.com/repo.git/commit/deadbeef" - requestID = "request/monorepo/main/42" + requestID = "42" ) backendErr := errors.New("backend unavailable") logs := []entity.RequestLog{ @@ -248,11 +249,11 @@ func TestRequestHistoryNotFoundErrors(t *testing.T) { }{ { name: "request ID", - err: fmt.Errorf("lookup failed: %w", &RequestHistoryByIDNotFoundError{RequestID: "request/queue/1"}), + err: fmt.Errorf("lookup failed: %w", &RequestHistoryByIDNotFoundError{RequestID: "1"}), assert: func(t *testing.T, err error) { var notFound *RequestHistoryByIDNotFoundError require.ErrorAs(t, err, ¬Found) - assert.Equal(t, "request/queue/1", notFound.RequestID) + assert.Equal(t, "1", notFound.RequestID) }, }, { diff --git a/stovepipe/core/hookevent/hookevent_test.go b/stovepipe/core/hookevent/hookevent_test.go index 63278440a..e65692ed4 100644 --- a/stovepipe/core/hookevent/hookevent_test.go +++ b/stovepipe/core/hookevent/hookevent_test.go @@ -25,7 +25,7 @@ import ( const ( testQueue = "monorepo/main" - testID = "request/monorepo/main/7" + testID = "7" ) func testRequest() entity.Request { diff --git a/stovepipe/core/messagequeue/messagequeue_test.go b/stovepipe/core/messagequeue/messagequeue_test.go index 141ddddab..e0d0aa88d 100644 --- a/stovepipe/core/messagequeue/messagequeue_test.go +++ b/stovepipe/core/messagequeue/messagequeue_test.go @@ -23,7 +23,7 @@ import ( ) func TestProcessRequestRoundTrip(t *testing.T) { - req := &ProcessRequest{Id: "request/monorepo/main/42"} + req := &ProcessRequest{Id: "42"} data, err := Marshal(req) require.NoError(t, err) @@ -34,7 +34,7 @@ func TestProcessRequestRoundTrip(t *testing.T) { } func TestBuildRequestRoundTrip(t *testing.T) { - req := &BuildRequest{Id: "request/monorepo/main/42"} + req := &BuildRequest{Id: "42"} data, err := Marshal(req) require.NoError(t, err) @@ -69,7 +69,7 @@ func TestRecordRoundTrip(t *testing.T) { // TestWireFormat locks the protojson encoding decision the contract relies on: // snake_case field names (UseProtoNames). func TestWireFormat(t *testing.T) { - data, err := Marshal(&ProcessRequest{Id: "request/monorepo/main/42"}) + data, err := Marshal(&ProcessRequest{Id: "42"}) require.NoError(t, err) assert.Contains(t, string(data), `"id"`, "fields must serialize as snake_case") diff --git a/stovepipe/core/messagequeue/proto/build.proto b/stovepipe/core/messagequeue/proto/build.proto index 5b9b03bda..2d9573db3 100644 --- a/stovepipe/core/messagequeue/proto/build.proto +++ b/stovepipe/core/messagequeue/proto/build.proto @@ -31,7 +31,7 @@ option java_package = "com.uber.submitqueue.stovepipe.messagequeue"; message BuildRequest { option (uber.base.messagequeue.topic_keys) = "build"; - // id is the request id to build. Format: "request//". + // id is the request id to build. Canonical positive decimal string; queue is carried separately. string id = 1; // queue_name is the name of the queue processing the request, carried so // the consumer can route by queue without loading state first. Empty on diff --git a/stovepipe/core/messagequeue/proto/process.proto b/stovepipe/core/messagequeue/proto/process.proto index bb0059c7c..83a11f7b3 100644 --- a/stovepipe/core/messagequeue/proto/process.proto +++ b/stovepipe/core/messagequeue/proto/process.proto @@ -30,7 +30,7 @@ option java_package = "com.uber.submitqueue.stovepipe.messagequeue"; message ProcessRequest { option (uber.base.messagequeue.topic_keys) = "process"; - // id is the minted request id to process. Format: "request//". + // id is the minted request id to process. Canonical positive decimal string; queue is carried separately. string id = 1; // queue_name is the name of the queue processing the request, carried so // the consumer can route by queue without loading state first. Empty on diff --git a/stovepipe/core/messagequeue/proto/record.proto b/stovepipe/core/messagequeue/proto/record.proto index dffabaf84..7f9713c35 100644 --- a/stovepipe/core/messagequeue/proto/record.proto +++ b/stovepipe/core/messagequeue/proto/record.proto @@ -33,7 +33,7 @@ message Record { option (uber.base.messagequeue.topic_keys) = "record"; // id is the request id whose build reached a terminal status. Format: - // "request//" (entity.Request.ID). + // a canonical positive decimal string (entity.Request.ID). string id = 1; // queue_name is the name of the queue processing the request, carried so // the consumer can route by queue without loading state first. Empty on diff --git a/stovepipe/core/messagequeue/protopb/build.pb.go b/stovepipe/core/messagequeue/protopb/build.pb.go index 940f0cfab..4d27a588d 100644 --- a/stovepipe/core/messagequeue/protopb/build.pb.go +++ b/stovepipe/core/messagequeue/protopb/build.pb.go @@ -44,7 +44,7 @@ const ( // redelivery stays idempotent). See doc/rfc/stovepipe/steps/build.md. type BuildRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - // id is the request id to build. Format: "request//". + // id is the request id to build. Canonical positive decimal string; queue is carried separately. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // queue_name is the name of the queue processing the request, carried so // the consumer can route by queue without loading state first. Empty on diff --git a/stovepipe/core/messagequeue/protopb/process.pb.go b/stovepipe/core/messagequeue/protopb/process.pb.go index 943baa089..12bbdf93d 100644 --- a/stovepipe/core/messagequeue/protopb/process.pb.go +++ b/stovepipe/core/messagequeue/protopb/process.pb.go @@ -43,7 +43,7 @@ const ( // the store, so the id is enough and redelivery stays idempotent). type ProcessRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - // id is the minted request id to process. Format: "request//". + // id is the minted request id to process. Canonical positive decimal string; queue is carried separately. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // queue_name is the name of the queue processing the request, carried so // the consumer can route by queue without loading state first. Empty on diff --git a/stovepipe/core/messagequeue/protopb/record.pb.go b/stovepipe/core/messagequeue/protopb/record.pb.go index 0d99d5658..9ad042359 100644 --- a/stovepipe/core/messagequeue/protopb/record.pb.go +++ b/stovepipe/core/messagequeue/protopb/record.pb.go @@ -46,7 +46,7 @@ const ( type Record struct { state protoimpl.MessageState `protogen:"open.v1"` // id is the request id whose build reached a terminal status. Format: - // "request//" (entity.Request.ID). + // a canonical positive decimal string (entity.Request.ID). Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // queue_name is the name of the queue processing the request, carried so // the consumer can route by queue without loading state first. Empty on diff --git a/stovepipe/core/requestlog/materializer_test.go b/stovepipe/core/requestlog/materializer_test.go index 425560ff0..5b02ad25b 100644 --- a/stovepipe/core/requestlog/materializer_test.go +++ b/stovepipe/core/requestlog/materializer_test.go @@ -32,7 +32,7 @@ import ( const ( testQueue = "monorepo/main" - testRequestID = "request/monorepo/main/1" + testRequestID = "1" testRequestURI = "git://repo/head" testBaseURI = "git://repo/base" testNowMs = int64(1735689600000) diff --git a/stovepipe/entity/BUILD.bazel b/stovepipe/entity/BUILD.bazel index a1d7e5857..4fc9fb0c3 100644 --- a/stovepipe/entity/BUILD.bazel +++ b/stovepipe/entity/BUILD.bazel @@ -17,6 +17,7 @@ go_library( ], importpath = "github.com/uber/submitqueue/stovepipe/entity", visibility = ["//visibility:public"], + deps = ["//platform/base/id:go_default_library"], ) go_test( diff --git a/stovepipe/entity/ingest.go b/stovepipe/entity/ingest.go index aa1b42aa8..39c0c4591 100644 --- a/stovepipe/entity/ingest.go +++ b/stovepipe/entity/ingest.go @@ -23,7 +23,7 @@ type IngestRequest struct { // IngestResult is the outcome of a successful ingest operation. type IngestResult struct { - // ID is the globally unique request identifier assigned to the ingested commit. - // Format: "request//". + // ID is the canonical decimal request identifier assigned within the selected queue. + // The value is a canonical positive decimal string scoped by the request queue. ID string } diff --git a/stovepipe/entity/request.go b/stovepipe/entity/request.go index 00e99e8e6..850e751d2 100644 --- a/stovepipe/entity/request.go +++ b/stovepipe/entity/request.go @@ -83,18 +83,17 @@ const ( ) // Request represents a single validation of a queue at a particular commit. The queue reports -// a newly observed commit, Stovepipe mints a Request (identity namespaced by the queue), and +// a newly observed commit, Stovepipe mints a Request (identity scoped by the queue), and // the request flows through the pipeline accumulating state. type Request struct { // **************** // Immutable fields, fixed at request entity creation // **************** - // ID is the globally unique identifier for the request. Format: "request//" - // (e.g. "request/monorepo/main/42"). + // ID is the canonical decimal request identifier, unique within Queue. ID string `json:"id"` - // Queue is the name of the queue (a named repo+ref) being validated. It namespaces the ID - // and is the stable handle the ingest caller supplies. + // Queue is the name of the queue (a named repo+ref) being validated. Together with ID it + // identifies the request and is the stable handle the ingest caller supplies. Queue string `json:"queue"` // URI is the opaque, VCS-agnostic locator of the commit under validation, as produced by the // SourceControl extension. diff --git a/stovepipe/entity/request_history.go b/stovepipe/entity/request_history.go index 6c2313417..e3c6b7d9d 100644 --- a/stovepipe/entity/request_history.go +++ b/stovepipe/entity/request_history.go @@ -16,7 +16,7 @@ package entity // GetRequestHistoryByIDRequest identifies one retained request history by request ID. type GetRequestHistoryByIDRequest struct { - // ID is the globally unique identifier of the request. + // ID is the queue-scoped identifier of the request. ID string // Queue is the logical queue containing the request and scopes the lookup. Queue string @@ -32,7 +32,7 @@ type GetRequestHistoryByURIRequest struct { // RequestHistory groups retained events for one request. type RequestHistory struct { - // RequestID is the globally unique identifier of the request. + // RequestID is the queue-scoped identifier of the request. RequestID string // Events are retained request-log events ordered by occurrence time and stable identity. Events []RequestLog diff --git a/stovepipe/entity/request_id.go b/stovepipe/entity/request_id.go index df5614dca..def123da7 100644 --- a/stovepipe/entity/request_id.go +++ b/stovepipe/entity/request_id.go @@ -14,47 +14,11 @@ package entity -import ( - "fmt" - "strconv" - "strings" -) +import "github.com/uber/submitqueue/platform/base/id" // CompareRequestID compares ingest order of two request IDs in the same queue. // Returns -1 if a is older than b, 0 if equal, 1 if a is newer than b. -// IDs must follow the format request//. -func CompareRequestID(queue, a, b string) (int, error) { - if a == b { - return 0, nil - } - aCounter, err := requestCounter(a, queue) - if err != nil { - return 0, err - } - bCounter, err := requestCounter(b, queue) - if err != nil { - return 0, err - } - switch { - case aCounter < bCounter: - return -1, nil - case aCounter > bCounter: - return 1, nil - default: - return 0, nil - } -} - -// requestIDPrefix returns the expected prefix for request ids in queue: "request//". -func requestIDPrefix(queue string) string { - return "request/" + queue + "/" -} - -// requestCounter extracts the per-queue counter suffix from a request id. -func requestCounter(id, queue string) (int64, error) { - prefix := requestIDPrefix(queue) - if !strings.HasPrefix(id, prefix) { - return 0, fmt.Errorf("request id %q does not match queue %q", id, queue) - } - return strconv.ParseInt(id[len(prefix):], 10, 64) +// IDs are canonical decimal strings whose scope is carried separately. +func CompareRequestID(a, b string) (int, error) { + return id.Compare(a, b) } diff --git a/stovepipe/entity/request_id_test.go b/stovepipe/entity/request_id_test.go index 047c607b0..716ae0634 100644 --- a/stovepipe/entity/request_id_test.go +++ b/stovepipe/entity/request_id_test.go @@ -22,8 +22,6 @@ import ( ) func TestCompareRequestID(t *testing.T) { - const queue = "monorepo/main" - tests := []struct { name string a string @@ -33,39 +31,34 @@ func TestCompareRequestID(t *testing.T) { }{ { name: "older vs newer", - a: "request/monorepo/main/7", - b: "request/monorepo/main/10", + a: "7", + b: "10", want: -1, }, { name: "newer vs older", - a: "request/monorepo/main/10", - b: "request/monorepo/main/7", + a: "10", + b: "7", want: 1, }, { name: "equal", - a: "request/monorepo/main/42", - b: "request/monorepo/main/42", + a: "42", + b: "42", want: 0, }, { name: "numeric not lexicographic", - a: "request/monorepo/main/9", - b: "request/monorepo/main/10", + a: "9", + b: "10", want: -1, }, - { - name: "wrong queue prefix", - a: "request/other/1", - b: "request/monorepo/main/2", - wantErr: true, - }, + {name: "prefixed ID", a: "request.1", b: "2", wantErr: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := CompareRequestID(queue, tt.a, tt.b) + got, err := CompareRequestID(tt.a, tt.b) if tt.wantErr { require.Error(t, err) return diff --git a/stovepipe/entity/request_log_test.go b/stovepipe/entity/request_log_test.go index 9dc4daaf5..67954c209 100644 --- a/stovepipe/entity/request_log_test.go +++ b/stovepipe/entity/request_log_test.go @@ -24,7 +24,7 @@ func TestRequestLogValidate(t *testing.T) { base := RequestLog{ ID: "state/1", Queue: "monorepo/main", - RequestID: "request/monorepo/main/1", + RequestID: "1", TimestampMs: 1735689600000, State: RequestStateAccepted, RequestVersion: 1, @@ -40,7 +40,7 @@ func TestRequestLogValidate(t *testing.T) { name: "superseded state", mutate: func(entry RequestLog) RequestLog { entry.State = RequestStateSuperseded - entry.Metadata = map[string]string{"superseded_by_request_id": "request/monorepo/main/2"} + entry.Metadata = map[string]string{"superseded_by_request_id": "2"} entry.OutcomeReason = RequestOutcomeReasonSupersededByNewerHead return entry }, diff --git a/stovepipe/entity/request_summary.go b/stovepipe/entity/request_summary.go index 2a217956b..b42c198ef 100644 --- a/stovepipe/entity/request_summary.go +++ b/stovepipe/entity/request_summary.go @@ -16,7 +16,7 @@ package entity // RequestSummary is the materialized current view of a validation request. type RequestSummary struct { - // RequestID is the globally unique request identifier. + // RequestID is the canonical decimal request identifier, unique within Queue. RequestID string // Queue is the queue containing the request. Queue string diff --git a/stovepipe/extension/storage/mysql/build_store_test.go b/stovepipe/extension/storage/mysql/build_store_test.go index 6e6e59118..c2af2c000 100644 --- a/stovepipe/extension/storage/mysql/build_store_test.go +++ b/stovepipe/extension/storage/mysql/build_store_test.go @@ -43,7 +43,7 @@ func setupBuildStoreTest(t *testing.T) (*sql.DB, sqlmock.Sqlmock, storage.BuildS func TestBuildStore_Create(t *testing.T) { build := entity.Build{ ID: "bk-1001", - RequestID: "request/monorepo/main/1", + RequestID: "1", Status: entity.BuildStatusAccepted, Version: 1, } @@ -107,7 +107,7 @@ func TestBuildStore_Create(t *testing.T) { func TestBuildStore_Get(t *testing.T) { want := entity.Build{ ID: "bk-1001", - RequestID: "request/monorepo/main/1", + RequestID: "1", Status: entity.BuildStatusRunning, Version: 2, } diff --git a/stovepipe/extension/storage/mysql/queue_store_test.go b/stovepipe/extension/storage/mysql/queue_store_test.go index 4bf6be9aa..35f5b2749 100644 --- a/stovepipe/extension/storage/mysql/queue_store_test.go +++ b/stovepipe/extension/storage/mysql/queue_store_test.go @@ -44,9 +44,9 @@ func TestQueueStore_Create(t *testing.T) { queue := entity.Queue{ Name: "monorepo/main", LastGreenURI: "git://remote/monorepo/main/green", - LastGreenRequestID: "request/monorepo/main/1", + LastGreenRequestID: "1", InFlightCount: 0, - LatestRequestID: "request/monorepo/main/1", + LatestRequestID: "1", Version: 1, } @@ -110,9 +110,9 @@ func TestQueueStore_Get(t *testing.T) { want := entity.Queue{ Name: "monorepo/main", LastGreenURI: "git://remote/monorepo/main/green", - LastGreenRequestID: "request/monorepo/main/2", + LastGreenRequestID: "2", InFlightCount: 2, - LatestRequestID: "request/monorepo/main/3", + LatestRequestID: "3", Version: 3, } @@ -185,9 +185,9 @@ func TestQueueStore_Update(t *testing.T) { queue := entity.Queue{ Name: "monorepo/main", LastGreenURI: "git://remote/monorepo/main/green", - LastGreenRequestID: "request/monorepo/main/1", + LastGreenRequestID: "1", InFlightCount: 1, - LatestRequestID: "request/monorepo/main/2", + LatestRequestID: "2", } const oldVersion, newVersion = int32(1), int32(2) diff --git a/stovepipe/extension/storage/mysql/request_log_store_test.go b/stovepipe/extension/storage/mysql/request_log_store_test.go index 1c46a5678..8eaad2aa0 100644 --- a/stovepipe/extension/storage/mysql/request_log_store_test.go +++ b/stovepipe/extension/storage/mysql/request_log_store_test.go @@ -32,7 +32,7 @@ import ( const ( testLogQueue = "monorepo/main" - testLogRequestID = "request/monorepo/main/1" + testLogRequestID = "1" ) var requestLogColumnNames = []string{ diff --git a/stovepipe/extension/storage/mysql/request_store_test.go b/stovepipe/extension/storage/mysql/request_store_test.go index ab912136e..190ac279b 100644 --- a/stovepipe/extension/storage/mysql/request_store_test.go +++ b/stovepipe/extension/storage/mysql/request_store_test.go @@ -43,7 +43,7 @@ func setupRequestStoreTest(t *testing.T) (*sql.DB, sqlmock.Sqlmock, storage.Requ func TestRequestStore_Create(t *testing.T) { request := entity.Request{ - ID: "request/monorepo/main/1", + ID: "1", Queue: "monorepo/main", URI: "git://remote/monorepo/main/deadbeef", State: entity.RequestStateAccepted, @@ -110,7 +110,7 @@ func TestRequestStore_Create(t *testing.T) { func TestRequestStore_Get(t *testing.T) { want := entity.Request{ - ID: "request/monorepo/main/1", + ID: "1", Queue: "monorepo/main", URI: "git://remote/monorepo/main/deadbeef", State: entity.RequestStateProcessing, @@ -187,7 +187,7 @@ func TestRequestStore_Get(t *testing.T) { func TestRequestStore_Update(t *testing.T) { request := entity.Request{ - ID: "request/monorepo/main/1", + ID: "1", Queue: "monorepo/main", URI: "git://remote/monorepo/main/deadbeef", State: entity.RequestStateProcessing, diff --git a/stovepipe/extension/storage/mysql/request_summary_store_test.go b/stovepipe/extension/storage/mysql/request_summary_store_test.go index dfdc3c53d..beaaf01d5 100644 --- a/stovepipe/extension/storage/mysql/request_summary_store_test.go +++ b/stovepipe/extension/storage/mysql/request_summary_store_test.go @@ -33,7 +33,7 @@ const testRequestSummaryQueue = "monorepo/main" func testRequestSummary() entity.RequestSummary { return entity.RequestSummary{ - RequestID: "request/monorepo/main/1", + RequestID: "1", Queue: testRequestSummaryQueue, URI: "git://repo/head", BaseURI: "git://repo/base", diff --git a/stovepipe/extension/storage/mysql/request_uri_store_test.go b/stovepipe/extension/storage/mysql/request_uri_store_test.go index fae0185f2..7ee5f1cdc 100644 --- a/stovepipe/extension/storage/mysql/request_uri_store_test.go +++ b/stovepipe/extension/storage/mysql/request_uri_store_test.go @@ -40,7 +40,7 @@ func setupRequestURIStoreTest(t *testing.T) (*sql.DB, sqlmock.Sqlmock, storage.R } func TestRequestURIStore_Create(t *testing.T) { - const queue, uri, id = "monorepo/main", "git://remote/monorepo/main/deadbeef", "request/monorepo/main/1" + const queue, uri, id = "monorepo/main", "git://remote/monorepo/main/deadbeef", "1" tests := []struct { name string @@ -99,7 +99,7 @@ func TestRequestURIStore_Create(t *testing.T) { } func TestRequestURIStore_GetIDByURI(t *testing.T) { - const queue, uri, wantID = "monorepo/main", "git://remote/monorepo/main/deadbeef", "request/monorepo/main/1" + const queue, uri, wantID = "monorepo/main", "git://remote/monorepo/main/deadbeef", "1" tests := []struct { name string diff --git a/stovepipe/extension/storage/mysql/validation_fact_store_test.go b/stovepipe/extension/storage/mysql/validation_fact_store_test.go index 3221e035c..e5628f87d 100644 --- a/stovepipe/extension/storage/mysql/validation_fact_store_test.go +++ b/stovepipe/extension/storage/mysql/validation_fact_store_test.go @@ -50,7 +50,7 @@ func TestValidationFactStore_Create(t *testing.T) { URI: testFactURI, Project: "", Degree: entity.DegreeGreen, - RequestID: "request/monorepo/main/1", + RequestID: "1", CreatedAt: 1735689600000, } @@ -115,7 +115,7 @@ func TestValidationFactStore_Get(t *testing.T) { URI: testFactURI, Project: "", Degree: entity.DegreeBroken, - RequestID: "request/monorepo/main/2", + RequestID: "2", CreatedAt: 1735689600000, } diff --git a/submitqueue/client/query_test.go b/submitqueue/client/query_test.go index 6954c8d42..92c9c5012 100644 --- a/submitqueue/client/query_test.go +++ b/submitqueue/client/query_test.go @@ -94,9 +94,9 @@ func TestListRequiresAQueue(t *testing.T) { func TestListFollowsPages(t *testing.T) { gw := &pagingGateway{pages: [][]string{ - {"q/1", "q/2"}, - {"q/3", "q/4"}, - {"q/5"}, + {"1", "2"}, + {"3", "4"}, + {"5"}, }} sq, stop := dial(t, gw) defer stop() @@ -104,7 +104,7 @@ func TestListFollowsPages(t *testing.T) { got, err := sq.List(context.Background(), ListQuery{Queue: "q"}) require.NoError(t, err) - assert.Equal(t, []string{"q/1", "q/2", "q/3", "q/4", "q/5"}, sqidsOf(got)) + assert.Equal(t, []string{"1", "2", "3", "4", "5"}, sqidsOf(got)) assert.Equal(t, 3, gw.calls, "every page is fetched") } @@ -112,9 +112,9 @@ func TestListStopsAtTheLimit(t *testing.T) { // The limit is across pages, not per page, so it has to cut a page short // and stop asking for more rather than over-fetching the whole queue. gw := &pagingGateway{pages: [][]string{ - {"q/1", "q/2"}, - {"q/3", "q/4"}, - {"q/5"}, + {"1", "2"}, + {"3", "4"}, + {"5"}, }} sq, stop := dial(t, gw) defer stop() @@ -122,26 +122,26 @@ func TestListStopsAtTheLimit(t *testing.T) { got, err := sq.List(context.Background(), ListQuery{Queue: "q", Limit: 3}) require.NoError(t, err) - assert.Equal(t, []string{"q/1", "q/2", "q/3"}, sqidsOf(got)) + assert.Equal(t, []string{"1", "2", "3"}, sqidsOf(got)) assert.Equal(t, 2, gw.calls, "the third page is never asked for") } func TestListStopsOnAnEmptyPage(t *testing.T) { // A server that keeps handing back a token with nothing in the page would // otherwise spin forever. - gw := &pagingGateway{pages: [][]string{{"q/1"}, {}}, alwaysToken: true} + gw := &pagingGateway{pages: [][]string{{"1"}, {}}, alwaysToken: true} sq, stop := dial(t, gw) defer stop() got, err := sq.List(context.Background(), ListQuery{Queue: "q"}) require.NoError(t, err) - assert.Equal(t, []string{"q/1"}, sqidsOf(got)) + assert.Equal(t, []string{"1"}, sqidsOf(got)) assert.Equal(t, 2, gw.calls) } func TestListSinceBecomesAReceiptBound(t *testing.T) { - gw := &pagingGateway{pages: [][]string{{"q/1"}}} + gw := &pagingGateway{pages: [][]string{{"1"}}} sq, stop := dial(t, gw) defer stop() @@ -155,7 +155,7 @@ func TestListSinceBecomesAReceiptBound(t *testing.T) { } func TestListWithoutSinceLeavesTheWindowOpen(t *testing.T) { - gw := &pagingGateway{pages: [][]string{{"q/1"}}} + gw := &pagingGateway{pages: [][]string{{"1"}}} sq, stop := dial(t, gw) defer stop() @@ -169,7 +169,7 @@ func TestListWithoutSinceLeavesTheWindowOpen(t *testing.T) { func TestListClosesTheWindowAtTheCallTime(t *testing.T) { // The gateway rejects a list whose lower bound is not strictly below its // upper one, so leaving the upper bound unset fails every call. - gw := &pagingGateway{pages: [][]string{{"q/1"}}} + gw := &pagingGateway{pages: [][]string{{"1"}}} sq, stop := dial(t, gw) defer stop() @@ -188,7 +188,7 @@ func TestListClosesTheWindowAtTheCallTime(t *testing.T) { func TestListKeepsTheWindowFixedAcrossPages(t *testing.T) { // The continuation token pins both bounds, so a bound recomputed per page // would be rejected from the second page on. - gw := &pagingGateway{pages: [][]string{{"q/1"}, {"q/2"}, {"q/3"}}} + gw := &pagingGateway{pages: [][]string{{"1"}, {"2"}, {"3"}}} sq, stop := dial(t, gw) defer stop() @@ -206,23 +206,23 @@ func TestRowsFromSummaries(t *testing.T) { received := time.Now().Add(-5 * time.Minute) rows := RowsFromSummaries([]*pb.RequestSummary{ { - Sqid: "q/1", + Sqid: "1", Status: "landed", ChangeUris: []string{"github://h/o/r/pull/1/abc"}, ReceivedAtMs: received.UnixMilli(), }, nil, // a hole in the page is skipped rather than becoming a blank row - {Sqid: "q/2", Status: "batched", LastError: "still going"}, + {Sqid: "2", Status: "batched", LastError: "still going"}, }) require.Len(t, rows, 2) - assert.Equal(t, "q/1", rows[0].SQID) + assert.Equal(t, "1", rows[0].SQID) assert.Equal(t, []Cell{{Text: "github://h/o/r/pull/1/abc"}}, rows[0].Cells) assert.True(t, rows[0].Done, "a terminal status arrives already settled") assert.Equal(t, received.UnixMilli(), rows[0].Submitted.UnixMilli()) - assert.Equal(t, "q/2", rows[1].SQID) + assert.Equal(t, "2", rows[1].SQID) assert.False(t, rows[1].Done, "an active request is still outstanding") assert.Equal(t, "still going", rows[1].Note) } diff --git a/submitqueue/client/view_test.go b/submitqueue/client/view_test.go index 9821a3308..aa1491959 100644 --- a/submitqueue/client/view_test.go +++ b/submitqueue/client/view_test.go @@ -240,12 +240,12 @@ func TestRowStage(t *testing.T) { }, { name: "accepted but nothing recorded yet", - row: Row{SQID: "demo-queue/17"}, + row: Row{SQID: "17"}, want: "…", }, { name: "the states it passed through", - row: Row{SQID: "demo-queue/17", Trail: []string{"accepted", "started", "landed"}}, + row: Row{SQID: "17", Trail: []string{"accepted", "started", "landed"}}, want: "accepted → started → landed", }, { @@ -253,13 +253,13 @@ func TestRowStage(t *testing.T) { // a request is without knowing how it got there. That still beats // a column of nothing. name: "the position it holds, when the trail was never fetched", - row: Row{SQID: "demo-queue/17", Status: "speculating"}, + row: Row{SQID: "17", Status: "speculating"}, want: "speculating", }, { name: "a fetched trail is preferred over the bare position", row: Row{ - SQID: "demo-queue/17", + SQID: "17", Status: "landed", Trail: []string{"accepted", "landed"}, }, @@ -361,9 +361,9 @@ func TestDrawLineAccounting(t *testing.T) { r := newRenderer() r.inPlace = true rows := []*Row{ - {SQID: "demo-queue/17", Cells: []Cell{{Text: "#41", URL: "https://github.com/o/r/pull/41"}}, + {SQID: "17", Cells: []Cell{{Text: "#41", URL: "https://github.com/o/r/pull/41"}}, Submitted: time.Now(), Trail: []string{"accepted", "started"}}, - {SQID: "demo-queue/18", Cells: []Cell{{Text: "#42", URL: "https://github.com/o/r/pull/42"}}, + {SQID: "18", Cells: []Cell{{Text: "#42", URL: "https://github.com/o/r/pull/42"}}, Submitted: time.Now(), Trail: []string{"accepted"}}, {}, } @@ -385,7 +385,7 @@ func TestDrawStaysWithinLineWidth(t *testing.T) { r := newRenderer() r.inPlace = true rows := []*Row{{ - SQID: "demo-queue/17", + SQID: "17", Submitted: time.Now(), Trail: strings.Split(strings.Repeat("speculating ", 30), " "), Note: strings.Repeat("a very long error message ", 10), @@ -412,7 +412,7 @@ func TestDrawFollowsAResize(t *testing.T) { r.width = width r.size = func() (int, int, bool) { return width, 40, true } - rows := []*Row{{SQID: "demo-queue/72", Submitted: time.Now(), Trail: longTrail}} + rows := []*Row{{SQID: "72", Submitted: time.Now(), Trail: longTrail}} wide := captureStdout(t, func() { r.draw(rows, "watching") }) for _, line := range strings.Split(wide, "\n") { @@ -459,7 +459,7 @@ func TestNewRendererNeedsASizeToRedrawInPlace(t *testing.T) { func TestDrawPipedSkipsClockOnlyRedraws(t *testing.T) { r := newRenderer() r.inPlace = false - rows := []*Row{{SQID: "demo-queue/17", Submitted: time.Now().Add(-5 * time.Second), Trail: []string{"accepted"}}} + rows := []*Row{{SQID: "17", Submitted: time.Now().Add(-5 * time.Second), Trail: []string{"accepted"}}} first := captureStdout(t, func() { r.draw(rows, "watching") }) require.NotEmpty(t, first) @@ -479,10 +479,10 @@ func TestDrawPipedSkipsClockOnlyRedraws(t *testing.T) { // that shrank would make the table jitter as rows fill in. func TestFitGrowsColumnsOnly(t *testing.T) { r := newRenderer() - wide := []*Row{{SQID: "some-very-long-queue-name/1234"}} + wide := []*Row{{SQID: "12345678"}} r.fit(wide) grown := r.wRequest - assert.Equal(t, len("some-very-long-queue-name/1234"), grown) + assert.Equal(t, len("12345678"), grown) r.fit([]*Row{{}}) assert.Equal(t, grown, r.wRequest) @@ -502,14 +502,14 @@ func TestOutcome(t *testing.T) { } func TestSummarize(t *testing.T) { - assert.NoError(t, summarize([]*Row{{SQID: "q/1", Status: "landed"}})) - assert.Error(t, summarize([]*Row{{SQID: "q/1", Status: "landed"}, {SQID: "q/2", Status: "error"}})) + assert.NoError(t, summarize([]*Row{{SQID: "1", Status: "landed"}})) + assert.Error(t, summarize([]*Row{{SQID: "1", Status: "landed"}, {SQID: "2", Status: "error"}})) // A failure reason, when the request carries one, is part of the summary so a // scripted run reports why rather than only that. - err := summarize([]*Row{{SQID: "q/2", Status: "error", Note: "merge failed: conflict in foo.go"}}) + err := summarize([]*Row{{SQID: "2", Status: "error", Note: "merge failed: conflict in foo.go"}}) require.Error(t, err) - assert.Contains(t, err.Error(), "q/2=error: merge failed: conflict in foo.go") + assert.Contains(t, err.Error(), "2=error: merge failed: conflict in foo.go") } // TestRowLineAlignment is the column contract: on every row the stage begins at @@ -566,21 +566,21 @@ func TestTrackerSettlesOnlyWhenSealedAndTerminal(t *testing.T) { ctx := context.Background() captureStdout(t, func() { - tr.Update(func() { tr.rows[0].SQID = "demo-queue/1" }) + tr.Update(func() { tr.rows[0].SQID = "1" }) tr.Seal() }) assert.False(t, isClosed(tr.settled), "a row that was never enqueued is not settled") - captureStdout(t, func() { tr.Update(func() { tr.rows[1].SQID = "demo-queue/2" }) }) - gw.set("demo-queue/1", "accepted", "started") - gw.set("demo-queue/2", "accepted") + captureStdout(t, func() { tr.Update(func() { tr.rows[1].SQID = "2" }) }) + gw.set("1", "accepted", "started") + gw.set("2", "accepted") captureStdout(t, func() { tr.refresh(ctx, gw, "demo-queue") }) assert.Equal(t, []string{"accepted", "started"}, tr.rows[0].Trail) assert.False(t, isClosed(tr.settled), "requests still in flight") - gw.set("demo-queue/1", "accepted", "started", "landed") - gw.set("demo-queue/2", "accepted", "error") + gw.set("1", "accepted", "started", "landed") + gw.set("2", "accepted", "error") captureStdout(t, func() { tr.refresh(ctx, gw, "demo-queue") }) assert.True(t, isClosed(tr.settled), "every request reached a terminal status") @@ -607,10 +607,10 @@ func TestTrackerPollsWhileCreating(t *testing.T) { tr := NewTracker(NewRows(3)) tr.r.inPlace = false gw := &fakeGateway{} - gw.set("demo-queue/1", "accepted", "started", "batched") + gw.set("1", "accepted", "started", "batched") captureStdout(t, func() { - tr.Update(func() { tr.rows[0].SQID = "demo-queue/1" }) + tr.Update(func() { tr.rows[0].SQID = "1" }) tr.refresh(context.Background(), gw, "demo-queue") }) @@ -634,7 +634,7 @@ func TestTrackerConcurrentPollAndUpdate(t *testing.T) { go func() { defer wg.Done() for i := range tr.rows { - sqid := fmt.Sprintf("demo-queue/%d", i) + sqid := fmt.Sprintf("%d", i+1) gw.set(sqid, "accepted", "landed") i := i tr.Update(func() { @@ -702,11 +702,11 @@ func TestNoteLinesRenderErrorInFull(t *testing.T) { r := newRenderer() r.inPlace = true failed := &Row{ - SQID: "demo-queue/1", + SQID: "1", Cells: []Cell{{Text: "#75", URL: "https://github.com/behinddwalls/sq-demo/pull/75"}}, Submitted: time.Now(), Trail: []string{"accepted", "started", "validated", "batched", "error"}, - Note: `speculator failed for queue demo-queue: score dependency "demo-queue/batch/1": ` + + Note: `speculator failed for queue demo-queue: score dependency "1": ` + `failed to resolve storage for queue "": queue name must not be empty`, } r.fit([]*Row{failed}) @@ -738,7 +738,7 @@ func TestNoteLinesRenderErrorInFull(t *testing.T) { func TestNoteLinesIndentToStageColumn(t *testing.T) { r := newRenderer() r.inPlace = true - rows := []*Row{{SQID: "demo-queue/1", Submitted: time.Now(), Trail: []string{"error"}, Note: "boom"}} + rows := []*Row{{SQID: "1", Submitted: time.Now(), Trail: []string{"error"}, Note: "boom"}} r.fit(rows) lines := r.noteLines(rows[0]) @@ -751,12 +751,12 @@ func TestRowLineAlignment(t *testing.T) { r.inPlace = true rows := []*Row{ { - SQID: "demo-queue/17", + SQID: "17", Cells: []Cell{{Text: "#41", URL: "https://github.com/behinddwalls/sq-demo/pull/41"}}, Submitted: time.Now(), Trail: []string{"accepted", "started", "batched", "speculating", "landed"}, }, - {SQID: "demo-queue/1234", Submitted: time.Now(), Trail: []string{"accepted"}}, + {SQID: "1234", Submitted: time.Now(), Trail: []string{"accepted"}}, {}, } r.fit(rows) @@ -829,7 +829,7 @@ var longTrail = []string{ func TestRowLinesWrapsRatherThanTruncates(t *testing.T) { r := newRenderer() r.inPlace = true - rw := &Row{SQID: "demo-queue/1", Submitted: time.Now(), Trail: longTrail} + rw := &Row{SQID: "1", Submitted: time.Now(), Trail: longTrail} r.fit([]*Row{rw}) lines := r.rowLines(rw) @@ -847,7 +847,7 @@ func TestRowLinesWrapsRatherThanTruncates(t *testing.T) { func TestRowLinesContinuationsAlignUnderTheStage(t *testing.T) { r := newRenderer() r.inPlace = true - rw := &Row{SQID: "demo-queue/1", Submitted: time.Now(), Trail: longTrail} + rw := &Row{SQID: "1", Submitted: time.Now(), Trail: longTrail} r.fit([]*Row{rw}) lines := r.rowLines(rw) @@ -869,7 +869,7 @@ func TestRowLinesFitTheWidth(t *testing.T) { r := newRenderer() r.inPlace = true r.width = width - rw := &Row{SQID: "demo-queue/1", Submitted: time.Now(), Trail: longTrail} + rw := &Row{SQID: "1", Submitted: time.Now(), Trail: longTrail} r.fit([]*Row{rw}) for _, line := range r.rowLines(rw) { @@ -885,7 +885,7 @@ func TestRowLinesUseTheWholeWidthBeforeWrapping(t *testing.T) { r := newRenderer() r.inPlace = true r.width = 240 - rw := &Row{SQID: "demo-queue/1", Submitted: time.Now(), Trail: longTrail} + rw := &Row{SQID: "1", Submitted: time.Now(), Trail: longTrail} r.fit([]*Row{rw}) lines := r.rowLines(rw) @@ -897,7 +897,7 @@ func TestRowLinesPipedStayOnOneLine(t *testing.T) { // A log has no width to respect and is easier to read and grep unwrapped. r := newRenderer() r.inPlace = false - rw := &Row{SQID: "demo-queue/1", Submitted: time.Now(), Trail: longTrail} + rw := &Row{SQID: "1", Submitted: time.Now(), Trail: longTrail} r.fit([]*Row{rw}) lines := r.rowLines(rw) @@ -921,7 +921,7 @@ func TestDrawNeverExceedsTheWindowHeight(t *testing.T) { rows := make([]*Row, 0, 60) for i := range 60 { rows = append(rows, &Row{ - SQID: fmt.Sprintf("demo-queue/%d", i+1), + SQID: fmt.Sprintf("%d", i+1), Submitted: time.Now(), Trail: []string{"accepted", "started", "speculating"}, }) @@ -949,7 +949,7 @@ func TestDrawKeepsMovingRowsWhenTheWindowIsShort(t *testing.T) { rows := make([]*Row, 0, 30) for i := range 30 { rows = append(rows, &Row{ - SQID: fmt.Sprintf("demo-queue/%d", i+1), + SQID: fmt.Sprintf("%d", i+1), Submitted: time.Now(), Status: "landed", Trail: []string{"accepted", "landed"}, @@ -957,7 +957,7 @@ func TestDrawKeepsMovingRowsWhenTheWindowIsShort(t *testing.T) { }) } moving := &Row{ - SQID: "demo-queue/moving", + SQID: "999", Submitted: time.Now(), Status: "speculating", Trail: []string{"accepted", "speculating"}, @@ -965,7 +965,7 @@ func TestDrawKeepsMovingRowsWhenTheWindowIsShort(t *testing.T) { rows = append(rows, moving) out := captureStdout(t, func() { r.draw(rows, "watching") }) - assert.Contains(t, out, "demo-queue/moving", "the unsettled row must survive the trim") + assert.Contains(t, out, "999", "the unsettled row must survive the trim") assert.Contains(t, out, "not shown", "and the reader must be told the table is partial") } @@ -975,7 +975,7 @@ func TestDrawOneShotKeepsEveryRow(t *testing.T) { rows := make([]*Row, 0, 40) for i := range 40 { rows = append(rows, &Row{ - SQID: fmt.Sprintf("demo-queue/%d", i+1), + SQID: fmt.Sprintf("%d", i+1), Submitted: time.Now(), Status: "landed", Done: true, @@ -983,7 +983,7 @@ func TestDrawOneShotKeepsEveryRow(t *testing.T) { } out := captureStdout(t, func() { Draw(rows, "40 request(s)") }) - assert.Contains(t, out, "demo-queue/40", "a listing must not drop rows to fit the window") + assert.Contains(t, out, "40", "a listing must not drop rows to fit the window") assert.NotContains(t, out, "not shown") } @@ -996,11 +996,11 @@ func TestDrawPipedKeepsEveryRow(t *testing.T) { rows := make([]*Row, 0, 40) for i := range 40 { - rows = append(rows, &Row{SQID: fmt.Sprintf("demo-queue/%d", i+1), Submitted: time.Now(), Done: true}) + rows = append(rows, &Row{SQID: fmt.Sprintf("%d", i+1), Submitted: time.Now(), Done: true}) } out := captureStdout(t, func() { r.draw(rows, "watching") }) - assert.Contains(t, out, "demo-queue/40") + assert.Contains(t, out, "40") assert.NotContains(t, out, "not shown") } diff --git a/submitqueue/core/changeset/fake/fake_test.go b/submitqueue/core/changeset/fake/fake_test.go index 1471dc1cb..8391aa012 100644 --- a/submitqueue/core/changeset/fake/fake_test.go +++ b/submitqueue/core/changeset/fake/fake_test.go @@ -28,44 +28,44 @@ import ( func TestResolverChanges(t *testing.T) { r := New(). - Set("q/batch/1", change.Change{URIs: []string{"u1"}}). - Set("q/batch/2", change.Change{URIs: []string{"u2"}}, change.Change{URIs: []string{"u3"}}) + Set("1", change.Change{URIs: []string{"u1"}}). + Set("2", change.Change{URIs: []string{"u2"}}, change.Change{URIs: []string{"u3"}}) - got, err := r.ChangesForBatch(context.Background(), entity.Batch{ID: "q/batch/2"}) + got, err := r.ChangesForBatch(context.Background(), entity.Batch{ID: "2"}) require.NoError(t, err) assert.Equal(t, []change.Change{{URIs: []string{"u2"}}, {URIs: []string{"u3"}}}, got) - unseeded, err := r.ChangesForBatch(context.Background(), entity.Batch{ID: "q/batch/unseeded"}) + unseeded, err := r.ChangesForBatch(context.Background(), entity.Batch{ID: "999"}) require.NoError(t, err) assert.Empty(t, unseeded) } func TestResolverDetailed(t *testing.T) { want := entity.BatchChanges{ - BatchID: "q/batch/1", + BatchID: "1", Queue: "q", Changes: []entity.ChangeInfo{{URI: "u1"}}, } - r := New().SetDetailed("q/batch/1", want) + r := New().SetDetailed("1", want) - got, err := r.DetailedForBatch(context.Background(), entity.Batch{ID: "q/batch/1", Queue: "q"}) + got, err := r.DetailedForBatch(context.Background(), entity.Batch{ID: "1", Queue: "q"}) require.NoError(t, err) assert.Equal(t, want, got) } func TestResolverDetailedUnseeded(t *testing.T) { - got, err := New().DetailedForBatch(context.Background(), entity.Batch{ID: "q/batch/9", Queue: "q"}) + got, err := New().DetailedForBatch(context.Background(), entity.Batch{ID: "9", Queue: "q"}) require.NoError(t, err) - assert.Equal(t, entity.BatchChanges{BatchID: "q/batch/9", Queue: "q"}, got) + assert.Equal(t, entity.BatchChanges{BatchID: "9", Queue: "q"}, got) } func TestResolverFailWith(t *testing.T) { sentinel := errors.New("boom") r := New().FailWith(sentinel) - _, err := r.ChangesForBatch(context.Background(), entity.Batch{ID: "q/batch/1"}) + _, err := r.ChangesForBatch(context.Background(), entity.Batch{ID: "1"}) require.ErrorIs(t, err, sentinel) - _, err = r.DetailedForBatch(context.Background(), entity.Batch{ID: "q/batch/1"}) + _, err = r.DetailedForBatch(context.Background(), entity.Batch{ID: "1"}) require.ErrorIs(t, err, sentinel) } diff --git a/submitqueue/core/changeset/resolver_test.go b/submitqueue/core/changeset/resolver_test.go index b2d167019..fd5583289 100644 --- a/submitqueue/core/changeset/resolver_test.go +++ b/submitqueue/core/changeset/resolver_test.go @@ -60,7 +60,7 @@ func TestResolverChanges(t *testing.T) { reqs.EXPECT().Get(gomock.Any(), "r2").Return(req("r2", "u2"), nil) reqs.EXPECT().Get(gomock.Any(), "r3").Return(req("r3", "u3"), nil) - got, err := r.ChangesForBatch(context.Background(), entity.Batch{ID: "q/batch/2", Contains: []string{"r2", "r3"}}) + got, err := r.ChangesForBatch(context.Background(), entity.Batch{ID: "2", Contains: []string{"r2", "r3"}}) require.NoError(t, err) // request order within the batch is preserved. assert.Equal(t, []change.Change{{URIs: []string{"u2"}}, {URIs: []string{"u3"}}}, got) @@ -70,7 +70,7 @@ func TestResolverChangesEmpty(t *testing.T) { ctrl := gomock.NewController(t) r := newTestResolver(ctrl, storagemock.NewMockRequestStore(ctrl), storagemock.NewMockChangeStore(ctrl)) - got, err := r.ChangesForBatch(context.Background(), entity.Batch{ID: "q/batch/1"}) + got, err := r.ChangesForBatch(context.Background(), entity.Batch{ID: "1"}) require.NoError(t, err) assert.Empty(t, got) } @@ -83,7 +83,7 @@ func TestResolverChangesRequestError(t *testing.T) { sentinel := errors.New("not found") reqs.EXPECT().Get(gomock.Any(), "r1").Return(entity.Request{}, sentinel) - _, err := r.ChangesForBatch(context.Background(), entity.Batch{ID: "q/batch/1", Contains: []string{"r1"}}) + _, err := r.ChangesForBatch(context.Background(), entity.Batch{ID: "1", Contains: []string{"r1"}}) require.ErrorIs(t, err, sentinel) } @@ -93,7 +93,7 @@ func TestResolverDetailed(t *testing.T) { changes := storagemock.NewMockChangeStore(ctrl) r := newTestResolver(ctrl, reqs, changes) - batch := entity.Batch{ID: "q/batch/1", Queue: "q", Contains: []string{"r1", "r2"}} + batch := entity.Batch{ID: "1", Queue: "q", Contains: []string{"r1", "r2"}} reqs.EXPECT().Get(gomock.Any(), "r1").Return(req("r1", "u1"), nil) reqs.EXPECT().Get(gomock.Any(), "r2").Return(req("r2", "u2"), nil) @@ -112,7 +112,7 @@ func TestResolverDetailed(t *testing.T) { got, err := r.DetailedForBatch(context.Background(), batch) require.NoError(t, err) assert.Equal(t, entity.BatchChanges{ - BatchID: "q/batch/1", + BatchID: "1", Queue: "q", Changes: []entity.ChangeInfo{{URI: "u1", Details: d1}, {URI: "u2", Details: d2}}, }, got) @@ -128,6 +128,6 @@ func TestResolverDetailedChangeStoreError(t *testing.T) { reqs.EXPECT().Get(gomock.Any(), "r1").Return(req("r1", "u1"), nil) changes.EXPECT().GetByURI(gomock.Any(), "u1").Return(nil, sentinel) - _, err := r.DetailedForBatch(context.Background(), entity.Batch{ID: "q/batch/1", Queue: "q", Contains: []string{"r1"}}) + _, err := r.DetailedForBatch(context.Background(), entity.Batch{ID: "1", Queue: "q", Contains: []string{"r1"}}) require.ErrorIs(t, err, sentinel) } diff --git a/submitqueue/core/messagequeue/messagequeue_test.go b/submitqueue/core/messagequeue/messagequeue_test.go index 5661adad9..79624f227 100644 --- a/submitqueue/core/messagequeue/messagequeue_test.go +++ b/submitqueue/core/messagequeue/messagequeue_test.go @@ -28,7 +28,7 @@ import ( func TestStartRoundTrip(t *testing.T) { req := StartFromLandRequest(entity.LandRequest{ - ID: "q/1", + ID: "1", Queue: "q", Change: change.Change{URIs: []string{"github://github.example.com/org/repo/pull/1/0123456789abcdef0123456789abcdef01234567"}}, LandStrategy: mergestrategy.MergeStrategySquashRebase, @@ -44,12 +44,12 @@ func TestStartRoundTrip(t *testing.T) { } func TestCancelRoundTrip(t *testing.T) { - msg := CancelFromEntity(entity.CancelRequest{ID: "q/7", Queue: "q", Reason: "user"}) + msg := CancelFromEntity(entity.CancelRequest{ID: "7", Queue: "q", Reason: "user"}) data, err := Marshal(msg) require.NoError(t, err) got := &Cancel{} require.NoError(t, Unmarshal(data, got)) - assert.Equal(t, entity.CancelRequest{ID: "q/7", Queue: "q", Reason: "user"}, CancelToEntity(got)) + assert.Equal(t, entity.CancelRequest{ID: "7", Queue: "q", Reason: "user"}, CancelToEntity(got)) } func TestIDMessageRoundTrip(t *testing.T) { @@ -58,14 +58,14 @@ func TestIDMessageRoundTrip(t *testing.T) { msg proto.Message into proto.Message }{ - {name: "validate", msg: &Validate{Id: "q/1", Queue: "q"}, into: &Validate{}}, - {name: "batch", msg: &Batch{Id: "q/1", Queue: "q"}, into: &Batch{}}, - {name: "dependency-analysis", msg: &DependencyAnalysis{Id: "q/batch/1", Queue: "q"}, into: &DependencyAnalysis{}}, - {name: "speculate", msg: &Speculate{Id: "q/batch/1", Queue: "q"}, into: &Speculate{}}, - {name: "build", msg: &Build{Id: "q/batch/1", Queue: "q"}, into: &Build{}}, + {name: "validate", msg: &Validate{Id: "1", Queue: "q"}, into: &Validate{}}, + {name: "batch", msg: &Batch{Id: "1", Queue: "q"}, into: &Batch{}}, + {name: "dependency-analysis", msg: &DependencyAnalysis{Id: "1", Queue: "q"}, into: &DependencyAnalysis{}}, + {name: "speculate", msg: &Speculate{Id: "1", Queue: "q"}, into: &Speculate{}}, + {name: "build", msg: &Build{Id: "1", Queue: "q"}, into: &Build{}}, {name: "buildsignal", msg: &BuildSignal{Id: "build-1", Queue: "q"}, into: &BuildSignal{}}, - {name: "merge", msg: &Merge{Id: "q/batch/1", Queue: "q"}, into: &Merge{}}, - {name: "conclude", msg: &Conclude{Id: "q/batch/1", Queue: "q"}, into: &Conclude{}}, + {name: "merge", msg: &Merge{Id: "1", Queue: "q"}, into: &Merge{}}, + {name: "conclude", msg: &Conclude{Id: "1", Queue: "q"}, into: &Conclude{}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -78,7 +78,7 @@ func TestIDMessageRoundTrip(t *testing.T) { } func TestLogRoundTrip(t *testing.T) { - entry := entity.NewRequestStatusLog("q", "q/1", entity.RequestStatusStarted, 1, "boom", map[string]string{"k": "v"}) + entry := entity.NewRequestStatusLog("q", "1", entity.RequestStatusStarted, 1, "boom", map[string]string{"k": "v"}) entry.TimestampMs = 1700000000000 data, err := Marshal(LogFromEntity(entry)) @@ -89,14 +89,14 @@ func TestLogRoundTrip(t *testing.T) { } func TestLogToEntityDefaults(t *testing.T) { - got := LogToEntity(&Log{RequestId: "q/1", Queue: "q"}) + got := LogToEntity(&Log{RequestId: "1", Queue: "q"}) assert.Equal(t, entity.RequestLogTypeStatus, got.Type) assert.NotNil(t, got.Metadata) assert.Empty(t, got.Metadata) } func TestLogEventRoundTrip(t *testing.T) { - entry := entity.NewRequestEventLog("q", "q/1", entity.RequestEventBuilding, map[string]string{"build_id": "b/7"}) + entry := entity.NewRequestEventLog("q", "1", entity.RequestEventBuilding, map[string]string{"build_id": "b/7"}) entry.TimestampMs = 1700000000000 data, err := Marshal(LogFromEntity(entry)) @@ -130,7 +130,7 @@ func TestLandStrategyMapping(t *testing.T) { } func TestStartNilChange(t *testing.T) { - got := LandRequestFromStart(&Start{Id: "q/1", Queue: "q"}) + got := LandRequestFromStart(&Start{Id: "1", Queue: "q"}) assert.Nil(t, got.Change.URIs) assert.Equal(t, mergestrategy.MergeStrategyUnknown, got.LandStrategy) } @@ -139,7 +139,7 @@ func TestStartNilChange(t *testing.T) { // and int64 as a JSON string. func TestWireFormat(t *testing.T) { start, err := Marshal(StartFromLandRequest(entity.LandRequest{ - ID: "q/1", + ID: "1", Queue: "q", Change: change.Change{URIs: []string{"u"}}, LandStrategy: mergestrategy.MergeStrategyRebase, @@ -150,7 +150,7 @@ func TestWireFormat(t *testing.T) { assert.Contains(t, string(start), `"land_strategy"`) assert.Contains(t, string(start), `"REBASE"`) - logBytes, err := Marshal(LogFromEntity(entity.RequestLog{RequestID: "q/1", TimestampMs: 42})) + logBytes, err := Marshal(LogFromEntity(entity.RequestLog{RequestID: "1", TimestampMs: 42})) require.NoError(t, err) assert.Contains(t, string(logBytes), `"timestamp_ms":"42"`) } @@ -198,45 +198,45 @@ func TestMarshalIDRoundTripPerTopic(t *testing.T) { _, err = UnmarshalRequestLog([]byte(`{`)) require.Error(t, err) - land, err := UnmarshalLandRequest([]byte(`{"id":"q/1","queue":"q"}`)) + land, err := UnmarshalLandRequest([]byte(`{"id":"1","queue":"q"}`)) require.NoError(t, err) - assert.Equal(t, "q/1", land.ID) - cancel, err := UnmarshalCancelRequest([]byte(`{"id":"q/7","queue":"q","reason":"x"}`)) + assert.Equal(t, "1", land.ID) + cancel, err := UnmarshalCancelRequest([]byte(`{"id":"7","queue":"q","reason":"x"}`)) require.NoError(t, err) assert.Equal(t, "x", cancel.Reason) - logEntry, err := UnmarshalRequestLog([]byte(`{"request_id":"q/1","queue":"q"}`)) + logEntry, err := UnmarshalRequestLog([]byte(`{"request_id":"1","queue":"q"}`)) require.NoError(t, err) assert.Equal(t, entity.RequestLogTypeStatus, logEntry.Type) } func TestMarshalIDRejectsUnknownTopic(t *testing.T) { - _, err := MarshalID(TopicKeyStart, "q/1", "q") + _, err := MarshalID(TopicKeyStart, "1", "q") require.Error(t, err) } func TestUnmarshalIDAndTypedIDs(t *testing.T) { - data, err := MarshalID(TopicKeyBuild, "q/batch/1", "q") + data, err := MarshalID(TopicKeyBuild, "1", "q") require.NoError(t, err) id, queue, err := UnmarshalID(TopicKeyBuild, data) require.NoError(t, err) - assert.Equal(t, "q/batch/1", id) + assert.Equal(t, "1", id) assert.Equal(t, "q", queue) bid, err := UnmarshalBatchID(TopicKeyBuild, data) require.NoError(t, err) - assert.Equal(t, entity.BatchID{ID: "q/batch/1", Queue: "q"}, bid) + assert.Equal(t, entity.BatchID{ID: "1", Queue: "q"}, bid) _, _, err = UnmarshalID(TopicKeyBuild, []byte(`{`)) require.Error(t, err) } func TestUnmarshalIDRejectsNonIDTopic(t *testing.T) { - _, _, err := UnmarshalID(TopicKeyStart, []byte(`{"id":"q/1","queue":"q"}`)) + _, _, err := UnmarshalID(TopicKeyStart, []byte(`{"id":"1","queue":"q"}`)) require.Error(t, err) - _, err = UnmarshalRequestID(TopicKeySpeculate, []byte(`{"id":"q/1","queue":"q"}`)) + _, err = UnmarshalRequestID(TopicKeySpeculate, []byte(`{"id":"1","queue":"q"}`)) require.Error(t, err) - _, err = UnmarshalBatchID(TopicKeyValidate, []byte(`{"id":"q/1","queue":"q"}`)) + _, err = UnmarshalBatchID(TopicKeyValidate, []byte(`{"id":"1","queue":"q"}`)) require.Error(t, err) _, err = UnmarshalBuildID(TopicKeyBuild, []byte(`{"id":"b","queue":"q"}`)) require.Error(t, err) diff --git a/submitqueue/core/messagequeue/proto/batch.proto b/submitqueue/core/messagequeue/proto/batch.proto index 4a8e465e0..04c7a7f4a 100644 --- a/submitqueue/core/messagequeue/proto/batch.proto +++ b/submitqueue/core/messagequeue/proto/batch.proto @@ -27,7 +27,7 @@ option java_package = "com.uber.submitqueue.submitqueue.messagequeue"; message Batch { option (uber.base.messagequeue.topic_keys) = "batch"; - // id is the request id to enroll in a batch. Format: "/". + // id is the request id to enroll in a batch. Canonical positive decimal string; queue is carried separately. string id = 1; // queue is the name of the queue processing this work, carried so the // consumer can route by queue without loading state first. Empty on diff --git a/submitqueue/core/messagequeue/proto/cancel.proto b/submitqueue/core/messagequeue/proto/cancel.proto index d264396b3..4b9cfb5de 100644 --- a/submitqueue/core/messagequeue/proto/cancel.proto +++ b/submitqueue/core/messagequeue/proto/cancel.proto @@ -27,7 +27,7 @@ option java_package = "com.uber.submitqueue.submitqueue.messagequeue"; message Cancel { option (uber.base.messagequeue.topic_keys) = "cancel"; - // id is the request id to cancel. Format: "/". + // id is the request id to cancel. Canonical positive decimal string; queue is carried separately. string id = 1; // queue is the name of the queue processing this work, carried so the // consumer can route by queue without loading state first. Empty on diff --git a/submitqueue/core/messagequeue/proto/conclude.proto b/submitqueue/core/messagequeue/proto/conclude.proto index 6c0273e95..dc6ba5ed0 100644 --- a/submitqueue/core/messagequeue/proto/conclude.proto +++ b/submitqueue/core/messagequeue/proto/conclude.proto @@ -27,7 +27,7 @@ option java_package = "com.uber.submitqueue.submitqueue.messagequeue"; message Conclude { option (uber.base.messagequeue.topic_keys) = "conclude"; - // id is the batch id to conclude. Format: "/batch/". + // id is the batch id to conclude. Canonical positive decimal string; queue and kind are carried separately. string id = 1; // queue is the name of the queue processing this work, carried so the // consumer can route by queue without loading state first. Empty on diff --git a/submitqueue/core/messagequeue/proto/dependencyanalysis.proto b/submitqueue/core/messagequeue/proto/dependencyanalysis.proto index 0ffc142bb..081364ce6 100644 --- a/submitqueue/core/messagequeue/proto/dependencyanalysis.proto +++ b/submitqueue/core/messagequeue/proto/dependencyanalysis.proto @@ -27,7 +27,7 @@ option java_package = "com.uber.submitqueue.submitqueue.messagequeue"; message DependencyAnalysis { option (uber.base.messagequeue.topic_keys) = "dependency-analysis"; - // id is the batch id to analyze. Format: "/batch/". + // id is the batch id to analyze. Canonical positive decimal string; queue and kind are carried separately. string id = 1; // queue is the name of the queue processing this work, carried so the // consumer can route by queue without loading state first. Empty on diff --git a/submitqueue/core/messagequeue/proto/speculate.proto b/submitqueue/core/messagequeue/proto/speculate.proto index d3345ac44..9175ad18e 100644 --- a/submitqueue/core/messagequeue/proto/speculate.proto +++ b/submitqueue/core/messagequeue/proto/speculate.proto @@ -27,7 +27,7 @@ option java_package = "com.uber.submitqueue.submitqueue.messagequeue"; message Speculate { option (uber.base.messagequeue.topic_keys) = "speculate"; - // id is the batch id to speculate. Format: "/batch/". + // id is the batch id to speculate. Canonical positive decimal string; queue and kind are carried separately. string id = 1; // queue is the name of the queue processing this work, carried so the // consumer can route by queue without loading state first. Empty on diff --git a/submitqueue/core/messagequeue/proto/start.proto b/submitqueue/core/messagequeue/proto/start.proto index ea2189bc8..343f44cde 100644 --- a/submitqueue/core/messagequeue/proto/start.proto +++ b/submitqueue/core/messagequeue/proto/start.proto @@ -31,7 +31,7 @@ option java_package = "com.uber.submitqueue.submitqueue.messagequeue"; message Start { option (uber.base.messagequeue.topic_keys) = "start"; - // id is the minted request id. Format: "/". + // id is the minted request id. Canonical positive decimal string; queue is carried separately. string id = 1; // queue is the name of the queue processing the land request. string queue = 2; diff --git a/submitqueue/core/messagequeue/proto/submitqueuebuild.proto b/submitqueue/core/messagequeue/proto/submitqueuebuild.proto index f84d8cdb0..583505aa8 100644 --- a/submitqueue/core/messagequeue/proto/submitqueuebuild.proto +++ b/submitqueue/core/messagequeue/proto/submitqueuebuild.proto @@ -27,7 +27,7 @@ option java_package = "com.uber.submitqueue.submitqueue.messagequeue"; message Build { option (uber.base.messagequeue.topic_keys) = "build"; - // id is the batch id whose speculated head should be built. Format: "/batch/". + // id is the batch id whose speculated head should be built. Canonical positive decimal string; queue and kind are carried separately. string id = 1; // queue is the name of the queue processing this work, carried so the // consumer can route by queue without loading state first. Empty on diff --git a/submitqueue/core/messagequeue/proto/submitqueuemerge.proto b/submitqueue/core/messagequeue/proto/submitqueuemerge.proto index 88e9f1909..bd07a147f 100644 --- a/submitqueue/core/messagequeue/proto/submitqueuemerge.proto +++ b/submitqueue/core/messagequeue/proto/submitqueuemerge.proto @@ -27,7 +27,7 @@ option java_package = "com.uber.submitqueue.submitqueue.messagequeue"; message Merge { option (uber.base.messagequeue.topic_keys) = "submitqueue-land"; - // id is the batch id to land. Format: "/batch/". + // id is the batch id to land. Canonical positive decimal string; queue and kind are carried separately. string id = 1; // queue is the name of the queue processing this work, carried so the // consumer can route by queue without loading state first. Empty on diff --git a/submitqueue/core/messagequeue/proto/validate.proto b/submitqueue/core/messagequeue/proto/validate.proto index ec22dc229..d86000607 100644 --- a/submitqueue/core/messagequeue/proto/validate.proto +++ b/submitqueue/core/messagequeue/proto/validate.proto @@ -27,7 +27,7 @@ option java_package = "com.uber.submitqueue.submitqueue.messagequeue"; message Validate { option (uber.base.messagequeue.topic_keys) = "validate"; - // id is the request id to validate. Format: "/". + // id is the request id to validate. Canonical positive decimal string; queue is carried separately. string id = 1; // queue is the name of the queue processing this work, carried so the // consumer can route by queue without loading state first. Empty on diff --git a/submitqueue/core/messagequeue/protopb/batch.pb.go b/submitqueue/core/messagequeue/protopb/batch.pb.go index a0cd2cd04..947afdcdc 100644 --- a/submitqueue/core/messagequeue/protopb/batch.pb.go +++ b/submitqueue/core/messagequeue/protopb/batch.pb.go @@ -40,7 +40,7 @@ const ( // Batch is the payload mergeconflictsignal publishes to the batch stage: only the request id travels. batch reloads the Request from storage. type Batch struct { state protoimpl.MessageState `protogen:"open.v1"` - // id is the request id to enroll in a batch. Format: "/". + // id is the request id to enroll in a batch. Canonical positive decimal string; queue is carried separately. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // queue is the name of the queue processing this work, carried so the // consumer can route by queue without loading state first. Empty on diff --git a/submitqueue/core/messagequeue/protopb/cancel.pb.go b/submitqueue/core/messagequeue/protopb/cancel.pb.go index 2ea16bc01..0f752ecd8 100644 --- a/submitqueue/core/messagequeue/protopb/cancel.pb.go +++ b/submitqueue/core/messagequeue/protopb/cancel.pb.go @@ -40,7 +40,7 @@ const ( // Cancel is the payload the gateway publishes to the cancel stage. type Cancel struct { state protoimpl.MessageState `protogen:"open.v1"` - // id is the request id to cancel. Format: "/". + // id is the request id to cancel. Canonical positive decimal string; queue is carried separately. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // queue is the name of the queue processing this work, carried so the // consumer can route by queue without loading state first. Empty on diff --git a/submitqueue/core/messagequeue/protopb/conclude.pb.go b/submitqueue/core/messagequeue/protopb/conclude.pb.go index 543b8e0a3..0bb9a42a4 100644 --- a/submitqueue/core/messagequeue/protopb/conclude.pb.go +++ b/submitqueue/core/messagequeue/protopb/conclude.pb.go @@ -40,7 +40,7 @@ const ( // Conclude is the payload published to the conclude stage: the id is a batch id. conclude reloads the Batch from storage. A failed batch's reason travels in message metadata, not this payload. type Conclude struct { state protoimpl.MessageState `protogen:"open.v1"` - // id is the batch id to conclude. Format: "/batch/". + // id is the batch id to conclude. Canonical positive decimal string; queue and kind are carried separately. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // queue is the name of the queue processing this work, carried so the // consumer can route by queue without loading state first. Empty on diff --git a/submitqueue/core/messagequeue/protopb/dependencyanalysis.pb.go b/submitqueue/core/messagequeue/protopb/dependencyanalysis.pb.go index 687490932..f94dafca9 100644 --- a/submitqueue/core/messagequeue/protopb/dependencyanalysis.pb.go +++ b/submitqueue/core/messagequeue/protopb/dependencyanalysis.pb.go @@ -40,7 +40,7 @@ const ( // DependencyAnalysis is the payload batch publishes to the dependency-analysis stage: only the batch id travels. The consumer reloads the Batch from storage. type DependencyAnalysis struct { state protoimpl.MessageState `protogen:"open.v1"` - // id is the batch id to analyze. Format: "/batch/". + // id is the batch id to analyze. Canonical positive decimal string; queue and kind are carried separately. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // queue is the name of the queue processing this work, carried so the // consumer can route by queue without loading state first. Empty on diff --git a/submitqueue/core/messagequeue/protopb/speculate.pb.go b/submitqueue/core/messagequeue/protopb/speculate.pb.go index 53c6407ef..1817e682b 100644 --- a/submitqueue/core/messagequeue/protopb/speculate.pb.go +++ b/submitqueue/core/messagequeue/protopb/speculate.pb.go @@ -40,7 +40,7 @@ const ( // Speculate is the payload published to the speculate stage: only the batch id travels. speculate reloads the Batch from storage. type Speculate struct { state protoimpl.MessageState `protogen:"open.v1"` - // id is the batch id to speculate. Format: "/batch/". + // id is the batch id to speculate. Canonical positive decimal string; queue and kind are carried separately. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // queue is the name of the queue processing this work, carried so the // consumer can route by queue without loading state first. Empty on diff --git a/submitqueue/core/messagequeue/protopb/start.pb.go b/submitqueue/core/messagequeue/protopb/start.pb.go index 343d9a6ac..11cf339b0 100644 --- a/submitqueue/core/messagequeue/protopb/start.pb.go +++ b/submitqueue/core/messagequeue/protopb/start.pb.go @@ -44,7 +44,7 @@ const ( // from these fields; producer and consumer do not share a store at this seam. type Start struct { state protoimpl.MessageState `protogen:"open.v1"` - // id is the minted request id. Format: "/". + // id is the minted request id. Canonical positive decimal string; queue is carried separately. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // queue is the name of the queue processing the land request. Queue string `protobuf:"bytes,2,opt,name=queue,proto3" json:"queue,omitempty"` diff --git a/submitqueue/core/messagequeue/protopb/submitqueuebuild.pb.go b/submitqueue/core/messagequeue/protopb/submitqueuebuild.pb.go index c6ba889b3..dd77d9628 100644 --- a/submitqueue/core/messagequeue/protopb/submitqueuebuild.pb.go +++ b/submitqueue/core/messagequeue/protopb/submitqueuebuild.pb.go @@ -40,7 +40,7 @@ const ( // Build is the payload speculate publishes to the build stage: the id is a batch id. build reloads the Batch from storage. type Build struct { state protoimpl.MessageState `protogen:"open.v1"` - // id is the batch id whose speculated head should be built. Format: "/batch/". + // id is the batch id whose speculated head should be built. Canonical positive decimal string; queue and kind are carried separately. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // queue is the name of the queue processing this work, carried so the // consumer can route by queue without loading state first. Empty on diff --git a/submitqueue/core/messagequeue/protopb/submitqueuemerge.pb.go b/submitqueue/core/messagequeue/protopb/submitqueuemerge.pb.go index 8b6119e4b..7c4b3cc7d 100644 --- a/submitqueue/core/messagequeue/protopb/submitqueuemerge.pb.go +++ b/submitqueue/core/messagequeue/protopb/submitqueuemerge.pb.go @@ -40,7 +40,7 @@ const ( // Merge is the payload speculate publishes to the internal land stage: the id is a batch id. land reloads the Batch from storage before handing work to Runway. type Merge struct { state protoimpl.MessageState `protogen:"open.v1"` - // id is the batch id to land. Format: "/batch/". + // id is the batch id to land. Canonical positive decimal string; queue and kind are carried separately. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // queue is the name of the queue processing this work, carried so the // consumer can route by queue without loading state first. Empty on diff --git a/submitqueue/core/messagequeue/protopb/validate.pb.go b/submitqueue/core/messagequeue/protopb/validate.pb.go index d3ec8ec1b..ca014b47d 100644 --- a/submitqueue/core/messagequeue/protopb/validate.pb.go +++ b/submitqueue/core/messagequeue/protopb/validate.pb.go @@ -40,7 +40,7 @@ const ( // Validate is the payload start publishes to the validate stage: only the request id travels. validate reloads the Request from storage. type Validate struct { state protoimpl.MessageState `protogen:"open.v1"` - // id is the request id to validate. Format: "/". + // id is the request id to validate. Canonical positive decimal string; queue is carried separately. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // queue is the name of the queue processing this work, carried so the // consumer can route by queue without loading state first. Empty on diff --git a/submitqueue/entity/batch.go b/submitqueue/entity/batch.go index 9a53eee59..3f9242be7 100644 --- a/submitqueue/entity/batch.go +++ b/submitqueue/entity/batch.go @@ -133,7 +133,7 @@ func DependencyBatchStates() []BatchState { // Batch represents a group of requests to land on the source control repository's target branch. type Batch struct { - // ID is the globally unique identifier for the batch. Format: "/batch/". + // ID is the canonical decimal batch identifier, unique within Queue. ID string // Queue is the name of the queue processing the land request. Queue name is defined in the configuration and should be unique within the system. @@ -142,7 +142,7 @@ type Batch struct { // Contains is a list of land request IDs that are part of this batch. // Request IDs will always be part of the same queue. // - // For e.g. - [queueA/1, queueA/2, queueA/3]. + // For example: [1, 2, 3] within Queue. // Contains []string @@ -156,13 +156,13 @@ type Batch struct { // //This field is ok to be updated whether the state of the dependency graph changes. Update should use Version property for optimistic locking. // - // Example: consider batches - queueA/batch/1, queueA/batch/2, queueA/batch/3 - // such that - queueA/batch/2 and queueA/batch/3 have overlapping targets with requests in queueA/batch/1, but queueA/batch/2 and queueA/batch/3 do not have overlapping targets with each other. + // Example: consider batches 1, 2, and 3 in queueA such that batches 2 and 3 + // have overlapping targets with requests in batch 1, but not with each other. // // In this case, the Dependencies field for - - // - queueA/batch/1 will be empty - // - queueA/batch/2 will contain queueA/batch/1 - // - queueA/batch/3 will contain queueA/batch/1 + // - batch 1 will be empty + // - batch 2 will contain batch 1 + // - batch 3 will contain batch 1 // // The list is empty while the batch is Creating and final from Created onwards. Dependencies []string @@ -189,7 +189,7 @@ func BatchFromBytes(data []byte) (Batch, error) { // BatchID is a lightweight entity for publishing and consuming just the batch identifier via the queue. type BatchID struct { - // ID is the globally unique identifier for the batch. + // ID is the queue-scoped identifier for the batch. ID string `json:"id"` // Queue is the name of the queue processing the batch. Empty on payloads written before the field existed. Queue string `json:"queue"` diff --git a/submitqueue/entity/batch_changes.go b/submitqueue/entity/batch_changes.go index ece494adb..ff2353157 100644 --- a/submitqueue/entity/batch_changes.go +++ b/submitqueue/entity/batch_changes.go @@ -22,7 +22,7 @@ package entity // giving the scorer the whole batch's change facts in one value without coupling // it to storage. type BatchChanges struct { - // BatchID is the batch being scored. Format: "/batch/". + // BatchID is the canonical decimal identifier of the batch being scored. BatchID string // Queue is the queue the batch belongs to. Queue string diff --git a/submitqueue/entity/batch_dependent.go b/submitqueue/entity/batch_dependent.go index fb237f6d8..f71ae74eb 100644 --- a/submitqueue/entity/batch_dependent.go +++ b/submitqueue/entity/batch_dependent.go @@ -19,15 +19,15 @@ package entity // when a batch completes or fails — rather than scanning all batches to find which ones reference a given dependency, // the system can look up dependents directly. // -// Example: consider batches queueA/batch/1, queueA/batch/2, queueA/batch/3 -// where batch/2 and batch/3 both depend on batch/1 (i.e. batch/1 is in their Dependencies list). +// Example: consider batches 1, 2, and 3 in queueA +// where batches 2 and 3 both depend on batch 1 (i.e. 1 is in their Dependencies list). // // The BatchDependent records would be: -// - BatchID=queueA/batch/1 → Dependents=[queueA/batch/2, queueA/batch/3] -// - BatchID=queueA/batch/2 → Dependents=[] (nothing depends on it) -// - BatchID=queueA/batch/3 → Dependents=[] (nothing depends on it) +// - BatchID=1 → Dependents=[2, 3] +// - BatchID=2 → Dependents=[] (nothing depends on it) +// - BatchID=3 → Dependents=[] (nothing depends on it) type BatchDependent struct { - // BatchID is the globally unique identifier of the upstream batch. Format: "/batch/". + // BatchID is the canonical decimal identifier of the upstream batch. BatchID string // Dependents is a list of batch IDs that depend on this batch (i.e. batches whose Dependencies list contains BatchID). diff --git a/submitqueue/entity/batch_test.go b/submitqueue/entity/batch_test.go index 0d1496d14..f774a1750 100644 --- a/submitqueue/entity/batch_test.go +++ b/submitqueue/entity/batch_test.go @@ -82,9 +82,9 @@ func TestBatch_SerializationRoundTrip(t *testing.T) { { name: "batch with single request", batch: Batch{ - ID: "queueA/batch/1", + ID: "1", Queue: "queueA", - Contains: []string{"queueA/1"}, + Contains: []string{"1"}, State: BatchStateCreated, Version: 1, }, @@ -92,9 +92,9 @@ func TestBatch_SerializationRoundTrip(t *testing.T) { { name: "batch with multiple requests", batch: Batch{ - ID: "queueB/batch/42", + ID: "42", Queue: "queueB", - Contains: []string{"queueB/10", "queueB/11", "queueB/12"}, + Contains: []string{"10", "11", "12"}, State: BatchStateSpeculating, Version: 3, }, @@ -102,12 +102,12 @@ func TestBatch_SerializationRoundTrip(t *testing.T) { { name: "batch with dependencies", batch: Batch{ - ID: "queueA/batch/3", + ID: "3", Queue: "queueA", - Contains: []string{"queueA/5"}, + Contains: []string{"5"}, Dependencies: []string{ - "queueA/batch/1", - "queueA/batch/2", + "1", + "2", }, State: BatchStateCreated, Version: 1, @@ -116,9 +116,9 @@ func TestBatch_SerializationRoundTrip(t *testing.T) { { name: "batch in terminal state", batch: Batch{ - ID: "queueC/batch/99", + ID: "99", Queue: "queueC", - Contains: []string{"queueC/50"}, + Contains: []string{"50"}, State: BatchStateSucceeded, Version: 5, }, diff --git a/submitqueue/entity/cancel.go b/submitqueue/entity/cancel.go index e7ccd762d..ec599895b 100644 --- a/submitqueue/entity/cancel.go +++ b/submitqueue/entity/cancel.go @@ -17,7 +17,7 @@ package entity // CancelRequest represents a cancellation request sent over the queue from the gateway to the orchestrator. // It identifies the request to cancel by its ID and carries an optional human-readable reason for observability. type CancelRequest struct { - // ID is the globally unique identifier of the request to cancel. Format: "/". + // ID is the canonical decimal request identifier, resolved within Queue. ID string `json:"id"` // Queue is the name of the queue processing the request to cancel. Empty on payloads written before the field existed. Queue string `json:"queue"` diff --git a/submitqueue/entity/change_record.go b/submitqueue/entity/change_record.go index d78c53a56..7c6bafc9d 100644 --- a/submitqueue/entity/change_record.go +++ b/submitqueue/entity/change_record.go @@ -24,7 +24,7 @@ type ChangeRecord struct { URI string `json:"uri"` // RequestID is the owning land request that claimed this URI. - // Format matches entity.Request.ID: "/". + // Format matches entity.Request.ID: a canonical positive decimal string. // // RequestID participates in the change-store primary key so that concurrent claims // by different requests on the same URI coexist as distinct rows. Same-request diff --git a/submitqueue/entity/land.go b/submitqueue/entity/land.go index f579de1c5..ea1b4937a 100644 --- a/submitqueue/entity/land.go +++ b/submitqueue/entity/land.go @@ -23,7 +23,7 @@ import ( // to the orchestrator. It contains only the validated inputs and generated ID — the orchestrator // is responsible for constructing the full Request entity with state machine fields. type LandRequest struct { - // ID is the globally unique identifier for the land request. Format: "/". + // ID is the canonical decimal request identifier, unique within Queue. ID string `json:"id"` // Queue is the name of the queue processing the land request. Queue string `json:"queue"` @@ -38,7 +38,6 @@ type LandRequest struct { // controller assigned to the request so the transport layer can echo it back to // the caller. type LandResult struct { - // ID is the globally unique identifier assigned to the accepted land request. - // Format: "/". + // ID is the canonical decimal request identifier assigned within the request queue. ID string } diff --git a/submitqueue/entity/queue_batch_state.go b/submitqueue/entity/queue_batch_state.go index 1aa7ab1c4..181db1548 100644 --- a/submitqueue/entity/queue_batch_state.go +++ b/submitqueue/entity/queue_batch_state.go @@ -35,6 +35,6 @@ type QueueBatchState struct { // the batch's authoritative state lives on the Batch entity and may differ transiently. State BatchState - // BatchID is the globally unique identifier of the batch. Format: "/batch/". + // BatchID is the canonical decimal batch identifier, resolved within Queue. BatchID string } diff --git a/submitqueue/entity/request.go b/submitqueue/entity/request.go index 5182c8f83..6bc4b72ab 100644 --- a/submitqueue/entity/request.go +++ b/submitqueue/entity/request.go @@ -75,7 +75,7 @@ type Request struct { // Immutable fields, fixed at request entity creation // **************** - // ID is the globally unique identifier for the land request. Format: "/". + // ID is the canonical decimal request identifier, unique within Queue. ID string `json:"id"` // Queue is the name of the queue processing the land request. Queue name is defined in the configuration and should be unique within the system. Queue string `json:"queue"` @@ -110,7 +110,7 @@ func RequestFromBytes(data []byte) (Request, error) { // RequestID is a lightweight entity for publishing and consuming just the request identifier via the queue. type RequestID struct { - // ID is the globally unique identifier for the land request. + // ID is the queue-scoped identifier for the land request. ID string `json:"id"` // Queue is the name of the queue processing the land request. Empty on payloads written before the field existed. Queue string `json:"queue"` diff --git a/submitqueue/entity/request_batch.go b/submitqueue/entity/request_batch.go index af3c4829a..bc866ca53 100644 --- a/submitqueue/entity/request_batch.go +++ b/submitqueue/entity/request_batch.go @@ -16,9 +16,9 @@ package entity // RequestBatch is an immutable association between a request and one batch attempt containing it. type RequestBatch struct { - // RequestID is the globally unique request identifier. + // RequestID is the queue-scoped identifier of the request. RequestID string - // BatchID is the globally unique identifier of the batch containing the request. + // BatchID is the queue-scoped identifier of the batch containing the request. BatchID string // Version is the version of the association. Immutable associations start at version 1. Version int32 diff --git a/submitqueue/entity/request_history.go b/submitqueue/entity/request_history.go index 9c450a9a5..e526620bc 100644 --- a/submitqueue/entity/request_history.go +++ b/submitqueue/entity/request_history.go @@ -16,7 +16,7 @@ package entity // GetRequestHistoryByIDRequest identifies one retained request history by request ID. type GetRequestHistoryByIDRequest struct { - // ID is the globally unique identifier of the request. + // ID is the queue-scoped identifier of the request. ID string // Queue is the name of the queue processing the request. It scopes the lookup: // a request is only resolvable within its own queue. @@ -34,7 +34,7 @@ type GetRequestHistoryByChangeURIRequest struct { // RequestHistory groups retained events for one request. type RequestHistory struct { - // RequestID is the globally unique identifier of the request. + // RequestID is the queue-scoped identifier of the request. RequestID string // Events are retained request-log events ordered chronologically. Events []RequestLog diff --git a/submitqueue/entity/request_log_test.go b/submitqueue/entity/request_log_test.go index e6e33f4fe..2f70ae369 100644 --- a/submitqueue/entity/request_log_test.go +++ b/submitqueue/entity/request_log_test.go @@ -21,7 +21,7 @@ import ( ) func TestNewRequestLog_NilMetadata(t *testing.T) { - log := NewRequestStatusLog("queue1", "queue1/100", RequestStatusStarted, 0, "", nil) + log := NewRequestStatusLog("queue1", "100", RequestStatusStarted, 0, "", nil) assert.NotNil(t, log.Metadata) assert.Empty(t, log.Metadata) @@ -32,7 +32,7 @@ func TestNewRequestLog_NilMetadata(t *testing.T) { // meant to build them — so this pins what each one sets and leaves unset. func TestRequestLogConstructors(t *testing.T) { t.Run("status entry carries a status and no event", func(t *testing.T) { - log := NewRequestStatusLog("q", "q/1", RequestStatusSpeculating, 4, "boom", map[string]string{"k": "v"}) + log := NewRequestStatusLog("q", "1", RequestStatusSpeculating, 4, "boom", map[string]string{"k": "v"}) assert.Equal(t, RequestLogTypeStatus, log.Type) assert.Equal(t, RequestStatusSpeculating, log.Status) @@ -43,7 +43,7 @@ func TestRequestLogConstructors(t *testing.T) { }) t.Run("event entry carries an event and no status", func(t *testing.T) { - log := NewRequestEventLog("q", "q/1", RequestEventBuilding, map[string]string{"build_id": "b/7"}) + log := NewRequestEventLog("q", "1", RequestEventBuilding, map[string]string{"build_id": "b/7"}) assert.Equal(t, RequestLogTypeEvent, log.Type) assert.Equal(t, RequestEventBuilding, log.Event) diff --git a/submitqueue/entity/request_summary.go b/submitqueue/entity/request_summary.go index 7d098620d..f45051f17 100644 --- a/submitqueue/entity/request_summary.go +++ b/submitqueue/entity/request_summary.go @@ -16,7 +16,7 @@ package entity // GetRequestSummaryByIDRequest identifies one request summary by sqid. type GetRequestSummaryByIDRequest struct { - // ID is the globally unique identifier of the request. Format: "/". + // ID is the canonical decimal request identifier, resolved within Queue. ID string // Queue is the name of the queue processing the request. It scopes the lookup: // a request is only resolvable within its own queue. @@ -35,7 +35,7 @@ type GetRequestSummaryByChangeURIRequest struct { // RequestSummary is the gateway-owned materialized current view of a request. // RequestID is exposed as sqid by the gateway API. type RequestSummary struct { - // RequestID is the globally unique request identifier. + // RequestID is the canonical decimal request identifier, unique within Queue. RequestID string // Queue is the queue supplied at receipt. Queue string @@ -59,7 +59,7 @@ type RequestSummary struct { // RequestQueueSummary is the queue-ordered projection returned by List. type RequestQueueSummary struct { - // RequestID is the globally unique request identifier. + // RequestID is the canonical decimal request identifier, unique within Queue. RequestID string // Queue is the queue supplied at receipt. Queue string @@ -86,6 +86,6 @@ type RequestURI struct { Queue string // ReceivedAtMs is the immutable receipt timestamp in Unix milliseconds. ReceivedAtMs int64 - // RequestID is the globally unique request identifier. + // RequestID is the canonical decimal request identifier, unique within Queue. RequestID string } diff --git a/submitqueue/entity/request_test.go b/submitqueue/entity/request_test.go index 4415fb204..8c4c33441 100644 --- a/submitqueue/entity/request_test.go +++ b/submitqueue/entity/request_test.go @@ -25,7 +25,7 @@ import ( func TestRequest_ToBytes(t *testing.T) { req := Request{ - ID: "test-queue/123", + ID: "123", Queue: "test-queue", Change: change.Change{URIs: []string{ "github://github.example.com/uber/submitqueue/pull/456/abcdef0123456789abcdef0123456789abcdef01", @@ -42,7 +42,7 @@ func TestRequest_ToBytes(t *testing.T) { // Verify JSON contains expected fields jsonStr := string(data) - assert.Contains(t, jsonStr, "test-queue/123") + assert.Contains(t, jsonStr, "123") assert.Contains(t, jsonStr, "github://github.example.com/uber/submitqueue/pull/456/abcdef0123456789abcdef0123456789abcdef01") assert.Contains(t, jsonStr, "rebase") assert.Contains(t, jsonStr, "started") @@ -50,7 +50,7 @@ func TestRequest_ToBytes(t *testing.T) { func TestRequestFromBytes(t *testing.T) { original := Request{ - ID: "my-queue/999", + ID: "999", Queue: "my-queue", Change: change.Change{URIs: []string{"code.uber.internal.com/D111"}}, LandStrategy: mergestrategy.MergeStrategyMerge, @@ -146,7 +146,7 @@ func TestRequest_SerializationRoundTrip(t *testing.T) { { name: "github stacked diff", req: Request{ - ID: "queue1/100", + ID: "100", Queue: "queue1", Change: change.Change{URIs: []string{ "github://github.example.com/uber/repo-a/pull/101/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", @@ -161,7 +161,7 @@ func TestRequest_SerializationRoundTrip(t *testing.T) { { name: "phabricator revision", req: Request{ - ID: "queue2/200", + ID: "200", Queue: "queue2", Change: change.Change{URIs: []string{"code.uber.internal.com/D12345"}}, LandStrategy: mergestrategy.MergeStrategyRebase, @@ -172,7 +172,7 @@ func TestRequest_SerializationRoundTrip(t *testing.T) { { name: "github enterprise request", req: Request{ - ID: "queue3/300", + ID: "300", Queue: "queue3", Change: change.Change{URIs: []string{"github.uber.com/internal/service/999/deadbeef12"}}, LandStrategy: mergestrategy.MergeStrategyMerge, diff --git a/submitqueue/entity/speculation_test.go b/submitqueue/entity/speculation_test.go index b5946e01e..6fe80f669 100644 --- a/submitqueue/entity/speculation_test.go +++ b/submitqueue/entity/speculation_test.go @@ -22,19 +22,19 @@ import ( func TestSpeculationPath_ID_Deterministic(t *testing.T) { path := SpeculationPath{ - Head: "queueA/batch/3", + Head: "3", Dependencies: []PathDependency{ - {Batch: "queueA/batch/1", Assumption: DependencyAssumptionSucceeds}, - {Batch: "queueA/batch/2", Assumption: DependencyAssumptionFails}, + {Batch: "1", Assumption: DependencyAssumptionSucceeds}, + {Batch: "2", Assumption: DependencyAssumptionFails}, }, } // Same content produces the same ID across separate value instances. same := SpeculationPath{ - Head: "queueA/batch/3", + Head: "3", Dependencies: []PathDependency{ - {Batch: "queueA/batch/1", Assumption: DependencyAssumptionSucceeds}, - {Batch: "queueA/batch/2", Assumption: DependencyAssumptionFails}, + {Batch: "1", Assumption: DependencyAssumptionSucceeds}, + {Batch: "2", Assumption: DependencyAssumptionFails}, }, } @@ -44,10 +44,10 @@ func TestSpeculationPath_ID_Deterministic(t *testing.T) { func TestSpeculationPath_ID_Sensitivity(t *testing.T) { base := SpeculationPath{ - Head: "queueA/batch/3", + Head: "3", Dependencies: []PathDependency{ - {Batch: "queueA/batch/1", Assumption: DependencyAssumptionSucceeds}, - {Batch: "queueA/batch/2", Assumption: DependencyAssumptionSucceeds}, + {Batch: "1", Assumption: DependencyAssumptionSucceeds}, + {Batch: "2", Assumption: DependencyAssumptionSucceeds}, }, } @@ -58,7 +58,7 @@ func TestSpeculationPath_ID_Sensitivity(t *testing.T) { { name: "different head", path: SpeculationPath{ - Head: "queueA/batch/4", + Head: "4", Dependencies: base.Dependencies, }, }, @@ -67,8 +67,8 @@ func TestSpeculationPath_ID_Sensitivity(t *testing.T) { path: SpeculationPath{ Head: base.Head, Dependencies: []PathDependency{ - {Batch: "queueA/batch/1", Assumption: DependencyAssumptionSucceeds}, - {Batch: "queueA/batch/2", Assumption: DependencyAssumptionFails}, + {Batch: "1", Assumption: DependencyAssumptionSucceeds}, + {Batch: "2", Assumption: DependencyAssumptionFails}, }, }, }, @@ -77,8 +77,8 @@ func TestSpeculationPath_ID_Sensitivity(t *testing.T) { path: SpeculationPath{ Head: base.Head, Dependencies: []PathDependency{ - {Batch: "queueA/batch/1", Assumption: DependencyAssumptionSucceeds}, - {Batch: "queueA/batch/9", Assumption: DependencyAssumptionSucceeds}, + {Batch: "1", Assumption: DependencyAssumptionSucceeds}, + {Batch: "9", Assumption: DependencyAssumptionSucceeds}, }, }, }, @@ -87,8 +87,8 @@ func TestSpeculationPath_ID_Sensitivity(t *testing.T) { path: SpeculationPath{ Head: base.Head, Dependencies: []PathDependency{ - {Batch: "queueA/batch/2", Assumption: DependencyAssumptionSucceeds}, - {Batch: "queueA/batch/1", Assumption: DependencyAssumptionSucceeds}, + {Batch: "2", Assumption: DependencyAssumptionSucceeds}, + {Batch: "1", Assumption: DependencyAssumptionSucceeds}, }, }, }, @@ -131,8 +131,8 @@ func TestSpeculationPathStatus_IsTerminal(t *testing.T) { func TestSpeculationPathEntry_IDDerivesFromPath(t *testing.T) { path := SpeculationPath{ - Head: "queueA/batch/2", - Dependencies: []PathDependency{{Batch: "queueA/batch/1", Assumption: DependencyAssumptionSucceeds}}, + Head: "2", + Dependencies: []PathDependency{{Batch: "1", Assumption: DependencyAssumptionSucceeds}}, } // A well-formed entry keys itself by its path's content hash. @@ -142,7 +142,7 @@ func TestSpeculationPathEntry_IDDerivesFromPath(t *testing.T) { // A different path (same head, different assumption) is a different entry. other := SpeculationPath{ Head: path.Head, - Dependencies: []PathDependency{{Batch: "queueA/batch/1", Assumption: DependencyAssumptionFails}}, + Dependencies: []PathDependency{{Batch: "1", Assumption: DependencyAssumptionFails}}, } assert.NotEqual(t, entry.ID, other.ID()) } diff --git a/submitqueue/extension/changeprovider/routing/provider_test.go b/submitqueue/extension/changeprovider/routing/provider_test.go index 8caa47a9d..eb56e6f31 100644 --- a/submitqueue/extension/changeprovider/routing/provider_test.go +++ b/submitqueue/extension/changeprovider/routing/provider_test.go @@ -39,7 +39,7 @@ const ( func newRequest(uris ...string) entity.Request { return entity.Request{ - ID: "test-queue/1", + ID: "1", Queue: "test-queue", Change: change.Change{ URIs: uris, diff --git a/submitqueue/extension/conflict/all/all_test.go b/submitqueue/extension/conflict/all/all_test.go index 73d8cfb8f..88f8de300 100644 --- a/submitqueue/extension/conflict/all/all_test.go +++ b/submitqueue/extension/conflict/all/all_test.go @@ -25,7 +25,7 @@ import ( ) func TestAnalyze(t *testing.T) { - batch := entity.Batch{ID: "queueA/batch/10"} + batch := entity.Batch{ID: "10"} tests := []struct { name string @@ -45,14 +45,14 @@ func TestAnalyze(t *testing.T) { { name: "every in-flight batch is reported in input order", inFlight: []entity.Batch{ - {ID: "queueA/batch/1"}, - {ID: "queueA/batch/2"}, - {ID: "queueA/batch/3"}, + {ID: "1"}, + {ID: "2"}, + {ID: "3"}, }, want: []entity.Conflict{ - {BatchID: "queueA/batch/1", Type: entity.ConflictTypeConservative}, - {BatchID: "queueA/batch/2", Type: entity.ConflictTypeConservative}, - {BatchID: "queueA/batch/3", Type: entity.ConflictTypeConservative}, + {BatchID: "1", Type: entity.ConflictTypeConservative}, + {BatchID: "2", Type: entity.ConflictTypeConservative}, + {BatchID: "3", Type: entity.ConflictTypeConservative}, }, }, } diff --git a/submitqueue/extension/conflict/fake/fake_test.go b/submitqueue/extension/conflict/fake/fake_test.go index fab7a7054..ace90d6fa 100644 --- a/submitqueue/extension/conflict/fake/fake_test.go +++ b/submitqueue/extension/conflict/fake/fake_test.go @@ -37,35 +37,35 @@ func TestAnalyze_DelegatesWhenNoFailOn(t *testing.T) { // Delegate to "all": one conflict per in-flight batch. nil failOn -> passthrough. a := New(testCfg, all.New(testCfg), nil) got, err := a.Analyze(context.Background(), - entity.Batch{ID: "q/batch/1"}, - []entity.Batch{{ID: "q/batch/2"}, {ID: "q/batch/3"}}) + entity.Batch{ID: "1"}, + []entity.Batch{{ID: "2"}, {ID: "3"}}) require.NoError(t, err) assert.Len(t, got, 2) } func TestAnalyze_DelegatesWhenFailOnFalse(t *testing.T) { a := New(testCfg, none.New(testCfg), func(entity.Batch, []entity.Batch) bool { return false }) - got, err := a.Analyze(context.Background(), entity.Batch{ID: "q/batch/1"}, nil) + got, err := a.Analyze(context.Background(), entity.Batch{ID: "1"}, nil) require.NoError(t, err) assert.Empty(t, got) } func TestAnalyze_FailAlways(t *testing.T) { a := New(testCfg, none.New(testCfg), FailAlways) - _, err := a.Analyze(context.Background(), entity.Batch{ID: "q/batch/1"}, nil) + _, err := a.Analyze(context.Background(), entity.Batch{ID: "1"}, nil) require.Error(t, err) } func TestAnalyze_FailOnPredicate(t *testing.T) { // Inject an error only for a specific batch ID. a := New(testCfg, none.New(testCfg), func(b entity.Batch, _ []entity.Batch) bool { - return b.ID == "q/batch/bad" + return b.ID == "1" }) - _, err := a.Analyze(context.Background(), entity.Batch{ID: "q/batch/bad"}, nil) + _, err := a.Analyze(context.Background(), entity.Batch{ID: "1"}, nil) require.Error(t, err) - got, err := a.Analyze(context.Background(), entity.Batch{ID: "q/batch/ok"}, nil) + got, err := a.Analyze(context.Background(), entity.Batch{ID: "2"}, nil) require.NoError(t, err) assert.Empty(t, got) } diff --git a/submitqueue/extension/conflict/none/none_test.go b/submitqueue/extension/conflict/none/none_test.go index d776fe8a1..e82874110 100644 --- a/submitqueue/extension/conflict/none/none_test.go +++ b/submitqueue/extension/conflict/none/none_test.go @@ -25,7 +25,7 @@ import ( ) func TestAnalyze(t *testing.T) { - batch := entity.Batch{ID: "queueA/batch/10"} + batch := entity.Batch{ID: "10"} tests := []struct { name string @@ -36,9 +36,9 @@ func TestAnalyze(t *testing.T) { { name: "many in-flight batches", inFlight: []entity.Batch{ - {ID: "queueA/batch/1"}, - {ID: "queueA/batch/2"}, - {ID: "queueA/batch/3"}, + {ID: "1"}, + {ID: "2"}, + {ID: "3"}, }, }, } diff --git a/submitqueue/extension/speculation/allocator/sticky/sticky_test.go b/submitqueue/extension/speculation/allocator/sticky/sticky_test.go index 24d3519a7..412e8b8ac 100644 --- a/submitqueue/extension/speculation/allocator/sticky/sticky_test.go +++ b/submitqueue/extension/speculation/allocator/sticky/sticky_test.go @@ -82,27 +82,27 @@ func TestSticky_Allocate(t *testing.T) { name: "fills free budget in order", budget: 2, items: []entity.CandidatePath{ - candidate("q/2", entity.DependencyAssumptionSucceeds, "q/1"), - candidate("q/2", entity.DependencyAssumptionFails, "q/1"), - candidate("q/3", entity.DependencyAssumptionSucceeds, "q/1"), + candidate("2", entity.DependencyAssumptionSucceeds, "1"), + candidate("2", entity.DependencyAssumptionFails, "1"), + candidate("3", entity.DependencyAssumptionSucceeds, "1"), }, wantIdx: []int{0, 1}, }, { name: "keeps in-flight path funded and fills the free slot", budget: 2, - pathSets: []entity.SpeculationPathSet{fundedSet("q/2", entity.DependencyAssumptionSucceeds, "q/1", entity.SpeculationPathStatusBuilding)}, + pathSets: []entity.SpeculationPathSet{fundedSet("2", entity.DependencyAssumptionSucceeds, "1", entity.SpeculationPathStatusBuilding)}, items: []entity.CandidatePath{ - candidate("q/2", entity.DependencyAssumptionSucceeds, "q/1"), // already funded -> skipped - candidate("q/2", entity.DependencyAssumptionFails, "q/1"), // fills the one free slot + candidate("2", entity.DependencyAssumptionSucceeds, "1"), // already funded -> skipped + candidate("2", entity.DependencyAssumptionFails, "1"), // fills the one free slot }, wantIdx: []int{1}, }, { name: "terminal entry does not charge budget", budget: 1, - pathSets: []entity.SpeculationPathSet{fundedSet("q/2", entity.DependencyAssumptionSucceeds, "q/1", entity.SpeculationPathStatusPassed)}, - items: []entity.CandidatePath{candidate("q/3", entity.DependencyAssumptionFails, "q/1")}, + pathSets: []entity.SpeculationPathSet{fundedSet("2", entity.DependencyAssumptionSucceeds, "1", entity.SpeculationPathStatusPassed)}, + items: []entity.CandidatePath{candidate("3", entity.DependencyAssumptionFails, "1")}, wantIdx: []int{0}, }, { @@ -112,38 +112,38 @@ func TestSticky_Allocate(t *testing.T) { // then goes to the next eligible candidate. name: "skips a passed top candidate without spending the slot", budget: 1, - pathSets: []entity.SpeculationPathSet{fundedSet("q/2", entity.DependencyAssumptionSucceeds, "q/1", entity.SpeculationPathStatusPassed)}, + pathSets: []entity.SpeculationPathSet{fundedSet("2", entity.DependencyAssumptionSucceeds, "1", entity.SpeculationPathStatusPassed)}, items: []entity.CandidatePath{ - candidate("q/2", entity.DependencyAssumptionSucceeds, "q/1"), // the passed path itself - candidate("q/2", entity.DependencyAssumptionFails, "q/1"), + candidate("2", entity.DependencyAssumptionSucceeds, "1"), // the passed path itself + candidate("2", entity.DependencyAssumptionFails, "1"), }, wantIdx: []int{1}, }, { name: "skips a failed top candidate without spending the slot", budget: 1, - pathSets: []entity.SpeculationPathSet{fundedSet("q/2", entity.DependencyAssumptionSucceeds, "q/1", entity.SpeculationPathStatusFailed)}, + pathSets: []entity.SpeculationPathSet{fundedSet("2", entity.DependencyAssumptionSucceeds, "1", entity.SpeculationPathStatusFailed)}, items: []entity.CandidatePath{ - candidate("q/2", entity.DependencyAssumptionSucceeds, "q/1"), // the failed path itself - candidate("q/2", entity.DependencyAssumptionFails, "q/1"), + candidate("2", entity.DependencyAssumptionSucceeds, "1"), // the failed path itself + candidate("2", entity.DependencyAssumptionFails, "1"), }, wantIdx: []int{1}, }, { name: "skips a cancelled top candidate without spending the slot", budget: 1, - pathSets: []entity.SpeculationPathSet{fundedSet("q/2", entity.DependencyAssumptionSucceeds, "q/1", entity.SpeculationPathStatusCancelled)}, + pathSets: []entity.SpeculationPathSet{fundedSet("2", entity.DependencyAssumptionSucceeds, "1", entity.SpeculationPathStatusCancelled)}, items: []entity.CandidatePath{ - candidate("q/2", entity.DependencyAssumptionSucceeds, "q/1"), // the cancelled path itself - candidate("q/2", entity.DependencyAssumptionFails, "q/1"), + candidate("2", entity.DependencyAssumptionSucceeds, "1"), // the cancelled path itself + candidate("2", entity.DependencyAssumptionFails, "1"), }, wantIdx: []int{1}, }, { name: "pending entry charges budget", budget: 1, - pathSets: []entity.SpeculationPathSet{fundedSet("q/2", entity.DependencyAssumptionSucceeds, "q/1", entity.SpeculationPathStatusPending)}, - items: []entity.CandidatePath{candidate("q/3", entity.DependencyAssumptionSucceeds, "q/1")}, + pathSets: []entity.SpeculationPathSet{fundedSet("2", entity.DependencyAssumptionSucceeds, "1", entity.SpeculationPathStatusPending)}, + items: []entity.CandidatePath{candidate("3", entity.DependencyAssumptionSucceeds, "1")}, wantIdx: nil, }, { @@ -152,8 +152,8 @@ func TestSticky_Allocate(t *testing.T) { // oversubscribe the hard cap on concurrent builds. name: "cancelling entry charges budget", budget: 1, - pathSets: []entity.SpeculationPathSet{fundedSet("q/2", entity.DependencyAssumptionSucceeds, "q/1", entity.SpeculationPathStatusCancelling)}, - items: []entity.CandidatePath{candidate("q/3", entity.DependencyAssumptionSucceeds, "q/1")}, + pathSets: []entity.SpeculationPathSet{fundedSet("2", entity.DependencyAssumptionSucceeds, "1", entity.SpeculationPathStatusCancelling)}, + items: []entity.CandidatePath{candidate("3", entity.DependencyAssumptionSucceeds, "1")}, wantIdx: nil, }, { @@ -161,16 +161,16 @@ func TestSticky_Allocate(t *testing.T) { // could be wrong, so sticky fails fast rather than guessing. name: "entry with no status fails fast", budget: 1, - pathSets: []entity.SpeculationPathSet{fundedSet("q/2", entity.DependencyAssumptionSucceeds, "q/1", entity.SpeculationPathStatusUnknown)}, - items: []entity.CandidatePath{candidate("q/3", entity.DependencyAssumptionSucceeds, "q/1")}, + pathSets: []entity.SpeculationPathSet{fundedSet("2", entity.DependencyAssumptionSucceeds, "1", entity.SpeculationPathStatusUnknown)}, + items: []entity.CandidatePath{candidate("3", entity.DependencyAssumptionSucceeds, "1")}, wantErr: true, }, { name: "deduplicates a repeated candidate", budget: 5, items: []entity.CandidatePath{ - candidate("q/2", entity.DependencyAssumptionSucceeds, "q/1"), - candidate("q/2", entity.DependencyAssumptionSucceeds, "q/1"), + candidate("2", entity.DependencyAssumptionSucceeds, "1"), + candidate("2", entity.DependencyAssumptionSucceeds, "1"), }, wantIdx: []int{0}, }, @@ -205,8 +205,8 @@ func TestSticky_HonorsCancelledContext(t *testing.T) { // The run's output is all-or-nothing: a partial action list would fund an // arbitrary prefix of the ranking, so cancellation yields no actions at all. items := []entity.CandidatePath{ - candidate("q/2", entity.DependencyAssumptionSucceeds, "q/1"), - candidate("q/2", entity.DependencyAssumptionFails, "q/1"), + candidate("2", entity.DependencyAssumptionSucceeds, "1"), + candidate("2", entity.DependencyAssumptionFails, "1"), } t.Run("cancelled before the first pull", func(t *testing.T) { diff --git a/submitqueue/extension/speculation/scorer/evidence/evidence_test.go b/submitqueue/extension/speculation/scorer/evidence/evidence_test.go index c6882e600..23ee7c13e 100644 --- a/submitqueue/extension/speculation/scorer/evidence/evidence_test.go +++ b/submitqueue/extension/speculation/scorer/evidence/evidence_test.go @@ -47,14 +47,14 @@ func (errorScorer) Score(_ context.Context, _ entity.Batch, _ entity.Speculation // pathSet builds a set whose entries carry the given statuses, every path // assuming all of its dependencies succeed. func pathSet(statuses ...entity.SpeculationPathStatus) entity.SpeculationPathSet { - set := entity.SpeculationPathSet{Queue: "q", Head: "q/batch/1"} + set := entity.SpeculationPathSet{Queue: "q", Head: "1"} for i, status := range statuses { set.Paths = append(set.Paths, entity.SpeculationPathEntry{ ID: fmt.Sprintf("path-%d", i), Status: status, Path: entity.SpeculationPath{ - Head: "q/batch/1", - Dependencies: []entity.PathDependency{{Batch: "q/batch/0", Assumption: entity.DependencyAssumptionSucceeds}}, + Head: "1", + Dependencies: []entity.PathDependency{{Batch: "0", Assumption: entity.DependencyAssumptionSucceeds}}, }, }) } diff --git a/submitqueue/extension/speculation/scorer/fake/fake_test.go b/submitqueue/extension/speculation/scorer/fake/fake_test.go index f018cf184..0e96af2b3 100644 --- a/submitqueue/extension/speculation/scorer/fake/fake_test.go +++ b/submitqueue/extension/speculation/scorer/fake/fake_test.go @@ -31,7 +31,7 @@ import ( // testCfg is the per-queue identity used by every case in this file. var testCfg = scorer.Config{QueueName: "test-queue"} -const batchID = "q/batch/1" +const batchID = "1" func TestNew_ImplementsInterface(t *testing.T) { var _ scorer.Scorer = New(testCfg, nil, nil) diff --git a/submitqueue/extension/speculation/speculator/standard/standard_test.go b/submitqueue/extension/speculation/speculator/standard/standard_test.go index a25a4e1c2..7eca64e84 100644 --- a/submitqueue/extension/speculation/speculator/standard/standard_test.go +++ b/submitqueue/extension/speculation/speculator/standard/standard_test.go @@ -54,8 +54,8 @@ func (c constScorer) Score(context.Context, entity.Batch, entity.SpeculationPath func TestComposed_EndToEnd_NaivePair(t *testing.T) { batches := []entity.Batch{ - {ID: "q/1", State: entity.BatchStateSpeculating}, - {ID: "q/2", State: entity.BatchStateSpeculating, Dependencies: []string{"q/1"}}, + {ID: "1", State: entity.BatchStateSpeculating}, + {ID: "2", State: entity.BatchStateSpeculating, Dependencies: []string{"1"}}, } // bestfirst generator + sticky allocator with a 2-build budget. @@ -69,22 +69,22 @@ func TestComposed_EndToEnd_NaivePair(t *testing.T) { var found bool for _, s := range got { assert.Equal(t, entity.PathActionBuild, s.Action) - if s.Path.Head == "q/2" { + if s.Path.Head == "2" { q2, found = s, true } } // q/2's funded path is the optimistic one: it assumes q/1 succeeds. require.True(t, found, "q/2 should have been funded") - assert.Equal(t, entity.DependencyAssumptionSucceeds, assumptionFor(q2.Path, "q/1")) + assert.Equal(t, entity.DependencyAssumptionSucceeds, assumptionFor(q2.Path, "1")) } func TestComposed_WiresGeneratorIntoAllocator(t *testing.T) { ctrl := gomock.NewController(t) - batches := []entity.Batch{{ID: "q/1", State: entity.BatchStateSpeculating}} - pathSets := []entity.SpeculationPathSet{{Head: "q/1"}} + batches := []entity.Batch{{ID: "1", State: entity.BatchStateSpeculating}} + pathSets := []entity.SpeculationPathSet{{Head: "1"}} want := []entity.Speculation{{ - Path: entity.SpeculationPath{Head: "q/1"}, + Path: entity.SpeculationPath{Head: "1"}, Action: entity.PathActionBuild, }} @@ -119,8 +119,8 @@ func TestComposed_PropagatesContextCancellation(t *testing.T) { // the two parts. This pins that the seam actually propagates: a cancelled // run yields the context error and no actions. batches := []entity.Batch{ - {ID: "q/1", State: entity.BatchStateSpeculating}, - {ID: "q/2", State: entity.BatchStateSpeculating, Dependencies: []string{"q/1"}}, + {ID: "1", State: entity.BatchStateSpeculating}, + {ID: "2", State: entity.BatchStateSpeculating, Dependencies: []string{"1"}}, } ctx, cancel := context.WithCancel(context.Background()) cancel() diff --git a/submitqueue/gateway/controller/BUILD.bazel b/submitqueue/gateway/controller/BUILD.bazel index ad80124e8..d26b44d02 100644 --- a/submitqueue/gateway/controller/BUILD.bazel +++ b/submitqueue/gateway/controller/BUILD.bazel @@ -16,6 +16,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//api/submitqueue/gateway/protopb:go_default_library", + "//platform/base/id:go_default_library", "//platform/consumer:go_default_library", "//platform/errs:go_default_library", "//platform/extension/counter:go_default_library", diff --git a/submitqueue/gateway/controller/cancel.go b/submitqueue/gateway/controller/cancel.go index f436b09c2..9dc6401e4 100644 --- a/submitqueue/gateway/controller/cancel.go +++ b/submitqueue/gateway/controller/cancel.go @@ -78,8 +78,8 @@ func (c *cancelController) Cancel(ctx context.Context, req entity.CancelRequest) op := metrics.Begin(c.metricsScope, opName, metrics.StorageLatencyBuckets) defer func() { op.Complete(retErr) }() - if req.ID == "" { - return fmt.Errorf("requires the request to have a sqid specified: %w", ErrInvalidRequest) + if err := validateResourceID("sqid", req.ID); err != nil { + return fmt.Errorf("Cancel invalid sqid: %w", err) } if err := validateQueueIdentifier(req.Queue); err != nil { return fmt.Errorf("Cancel invalid queue: %w", err) diff --git a/submitqueue/gateway/controller/cancel_test.go b/submitqueue/gateway/controller/cancel_test.go index 2935549ca..1fbb2f00a 100644 --- a/submitqueue/gateway/controller/cancel_test.go +++ b/submitqueue/gateway/controller/cancel_test.go @@ -83,7 +83,7 @@ func testCancelRequest(queue string, sqid string, reason string) entity.CancelRe func TestNewCancelController(t *testing.T) { ctrl := gomock.NewController(t) - controller := newTestCancelController(ctrl, tally.NoopScope, newCancelStorageFixture(ctrl, "test-queue/42"), newCancelTestRegistryWithNoopPublisher(t, ctrl)) + controller := newTestCancelController(ctrl, tally.NoopScope, newCancelStorageFixture(ctrl, "42"), newCancelTestRegistryWithNoopPublisher(t, ctrl)) require.NotNil(t, controller) } @@ -91,10 +91,10 @@ func TestCancel_HappyPath(t *testing.T) { ctrl := gomock.NewController(t) scope := tally.NewTestScope("gateway", nil) - controller := newTestCancelController(ctrl, scope, newCancelStorageFixture(ctrl, "test-queue/42"), newCancelTestRegistryWithNoopPublisher(t, ctrl)) + controller := newTestCancelController(ctrl, scope, newCancelStorageFixture(ctrl, "42"), newCancelTestRegistryWithNoopPublisher(t, ctrl)) ctx := context.Background() - err := controller.Cancel(ctx, testCancelRequest("test-queue", "test-queue/42", "user changed their mind")) + err := controller.Cancel(ctx, testCancelRequest("test-queue", "42", "user changed their mind")) require.NoError(t, err) @@ -122,7 +122,7 @@ func TestCancel_HappyPath(t *testing.T) { func TestCancel_ReturnsErrorOnEmptySqid(t *testing.T) { ctrl := gomock.NewController(t) - controller := newTestCancelController(ctrl, tally.NoopScope, newCancelStorageFixture(ctrl, "test-queue/42"), newCancelTestRegistryWithNoopPublisher(t, ctrl)) + controller := newTestCancelController(ctrl, tally.NoopScope, newCancelStorageFixture(ctrl, "42"), newCancelTestRegistryWithNoopPublisher(t, ctrl)) ctx := context.Background() err := controller.Cancel(ctx, testCancelRequest("test-queue", "", "anything")) @@ -131,6 +131,16 @@ func TestCancel_ReturnsErrorOnEmptySqid(t *testing.T) { assert.True(t, IsInvalidRequest(err)) } +func TestCancel_ReturnsErrorOnCompositeSqid(t *testing.T) { + ctrl := gomock.NewController(t) + + controller := newTestCancelController(ctrl, tally.NoopScope, newCancelStorageFixture(ctrl, "42"), newCancelTestRegistryWithNoopPublisher(t, ctrl)) + err := controller.Cancel(context.Background(), testCancelRequest("test-queue", "test-queue/42", "anything")) + + require.Error(t, err) + assert.True(t, IsInvalidRequest(err)) +} + func TestCancel_PublishesToQueue(t *testing.T) { var publishedTopic string var publishedMessage entityqueue.Message @@ -146,20 +156,20 @@ func TestCancel_PublishesToQueue(t *testing.T) { }, ) - controller := newTestCancelController(ctrl, tally.NoopScope, newCancelStorageFixture(ctrl, "my-queue/7"), registry) + controller := newTestCancelController(ctrl, tally.NoopScope, newCancelStorageFixture(ctrl, "7"), registry) ctx := context.Background() - err := controller.Cancel(ctx, testCancelRequest("my-queue", "my-queue/7", "obsolete change")) + err := controller.Cancel(ctx, testCancelRequest("my-queue", "7", "obsolete change")) require.NoError(t, err) assert.Equal(t, "cancel", publishedTopic) - assert.Equal(t, "my-queue/7", publishedMessage.ID) + assert.Equal(t, "7", publishedMessage.ID) assert.Equal(t, "my-queue", publishedMessage.Tenant) - assert.Equal(t, "my-queue/7", publishedMessage.PartitionKey) + assert.Equal(t, "7", publishedMessage.PartitionKey) deserialized, err := sqmq.UnmarshalCancelRequest(publishedMessage.Payload) require.NoError(t, err) - assert.Equal(t, "my-queue/7", deserialized.ID) + assert.Equal(t, "7", deserialized.ID) assert.Equal(t, "obsolete change", deserialized.Reason) } @@ -168,7 +178,7 @@ func TestCancel_PublishesToQueue(t *testing.T) { // before the cancel topic publish so observers see intent the moment Cancel returns. func TestCancel_InsertsCancellingLog(t *testing.T) { ctrl := gomock.NewController(t) - fixture := newCancelStorageFixture(ctrl, "my-queue/42") + fixture := newCancelStorageFixture(ctrl, "42") registry, publisher := newCancelTestRegistry(t, ctrl) insertedBeforePublish := false @@ -183,14 +193,14 @@ func TestCancel_InsertsCancellingLog(t *testing.T) { controller := newTestCancelController(ctrl, tally.NoopScope, fixture, registry) - err := controller.Cancel(context.Background(), testCancelRequest("my-queue", "my-queue/42", "obsolete change")) + err := controller.Cancel(context.Background(), testCancelRequest("my-queue", "42", "obsolete change")) require.NoError(t, err) fixture.mu.Lock() require.Len(t, fixture.logs, 1) insertedLog := fixture.logs[0] fixture.mu.Unlock() - assert.Equal(t, "my-queue/42", insertedLog.RequestID) + assert.Equal(t, "42", insertedLog.RequestID) assert.Equal(t, entity.RequestStatusCancelling, insertedLog.Status) assert.Equal(t, "obsolete change", insertedLog.Metadata["reason"]) assert.True(t, insertedBeforePublish, "log entry must be inserted before publish to the cancel topic") @@ -200,14 +210,14 @@ func TestCancel_InsertsCancellingLog(t *testing.T) { // short-circuits the RPC with an error and the cancel topic is never published to. func TestCancel_LogInsertFailure(t *testing.T) { ctrl := gomock.NewController(t) - fixture := newCancelStorageFixture(ctrl, "q/1") + fixture := newCancelStorageFixture(ctrl, "1") fixture.setLogInsertError(fmt.Errorf("db unavailable")) registry, publisher := newCancelTestRegistry(t, ctrl) _ = publisher controller := newTestCancelController(ctrl, tally.NoopScope, fixture, registry) - err := controller.Cancel(context.Background(), testCancelRequest("q", "q/1", "")) + err := controller.Cancel(context.Background(), testCancelRequest("q", "1", "")) require.Error(t, err) } @@ -217,10 +227,10 @@ func TestCancel_ReturnsErrorOnPublishFailure(t *testing.T) { registry, publisher := newCancelTestRegistry(t, ctrl) publisher.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).Return(fmt.Errorf("queue unavailable")) - controller := newTestCancelController(ctrl, tally.NoopScope, newCancelStorageFixture(ctrl, "test-queue/1"), registry) + controller := newTestCancelController(ctrl, tally.NoopScope, newCancelStorageFixture(ctrl, "1"), registry) ctx := context.Background() - err := controller.Cancel(ctx, testCancelRequest("test-queue", "test-queue/1", "")) + err := controller.Cancel(ctx, testCancelRequest("test-queue", "1", "")) require.Error(t, err) } @@ -233,7 +243,7 @@ func TestCancel_UnknownSqidIsUserError(t *testing.T) { _ = publisher controller := newTestCancelController(ctrl, tally.NoopScope, fixture, registry) - err := controller.Cancel(context.Background(), testCancelRequest("ghost", "ghost/1", "")) + err := controller.Cancel(context.Background(), testCancelRequest("ghost", "1", "")) require.Error(t, err) assert.True(t, IsRequestNotFound(err)) assert.True(t, errs.IsUserError(err)) @@ -241,7 +251,7 @@ func TestCancel_UnknownSqidIsUserError(t *testing.T) { var typed *RequestNotFoundError require.ErrorAs(t, err, &typed) - assert.Equal(t, "ghost/1", typed.Sqid) + assert.Equal(t, "1", typed.Sqid) } // TestCancel_RequestSummaryLookupFailure asserts that an infrastructure failure on @@ -251,13 +261,13 @@ func TestCancel_RequestSummaryLookupFailure(t *testing.T) { ctrl := gomock.NewController(t) summaryStore := storagemock.NewMockRequestSummaryStore(ctrl) - summaryStore.EXPECT().Get(gomock.Any(), "q/1").Return(entity.RequestSummary{}, fmt.Errorf("summary backend down")) + summaryStore.EXPECT().Get(gomock.Any(), "1").Return(entity.RequestSummary{}, fmt.Errorf("summary backend down")) registry, publisher := newCancelTestRegistry(t, ctrl) _ = publisher controller := NewCancelController(zap.NewNop().Sugar(), tally.NoopScope, factoryForStorage(ctrl, storageWithSummaryStore(ctrl, summaryStore)), newControllerStorageFixture(ctrl).newMaterializer(ctrl), registry) - err := controller.Cancel(context.Background(), testCancelRequest("q", "q/1", "")) + err := controller.Cancel(context.Background(), testCancelRequest("q", "1", "")) require.Error(t, err) assert.False(t, errs.IsUserError(err)) assert.False(t, IsRequestNotFound(err)) diff --git a/submitqueue/gateway/controller/land.go b/submitqueue/gateway/controller/land.go index 5a863d02c..4819b0893 100644 --- a/submitqueue/gateway/controller/land.go +++ b/submitqueue/gateway/controller/land.go @@ -21,6 +21,7 @@ import ( "time" "github.com/uber-go/tally" + "github.com/uber/submitqueue/platform/base/id" "github.com/uber/submitqueue/platform/consumer" "github.com/uber/submitqueue/platform/errs" "github.com/uber/submitqueue/platform/extension/counter" @@ -62,9 +63,7 @@ func IsUnrecognizedQueue(err error) bool { return errors.As(err, &target) } -// counterDomainRequest names the per-queue sequence that mints request IDs. The -// sqid is built independently as "/", so the domain is a -// sequence name only and never appears in the ID. +// counterDomainRequest names the per-queue sequence that mints request IDs. const counterDomainRequest = "request" // LandController handles land business logic for the gateway @@ -122,7 +121,7 @@ func (c *landController) Land(ctx context.Context, req entity.LandRequest) (resu return entity.LandResult{}, fmt.Errorf("failed to look up queue %q: %w", queue, err) } - // Generate a globally unique request ID for the land request. + // Generate a queue-scoped request ID for the land request. // The inbound entity arrives with an empty ID; the controller owns minting it. stores, err := c.stores.For(storage.Config{QueueName: queue}) if err != nil { @@ -136,8 +135,8 @@ func (c *landController) Land(ctx context.Context, req entity.LandRequest) (resu if err != nil { return entity.LandResult{}, fmt.Errorf("failed to generate request ID for queue=%s: %w", queue, err) } - req.ID = fmt.Sprintf("%s/%d", queue, seq) - if err := validateStoredIdentifier("generated sqid", req.ID); err != nil { + req.ID, err = id.FromCounter(seq) + if err != nil { return entity.LandResult{}, fmt.Errorf("generated invalid request ID for queue=%s: %w", queue, err) } diff --git a/submitqueue/gateway/controller/land_test.go b/submitqueue/gateway/controller/land_test.go index 2cf19006f..481ae8b4a 100644 --- a/submitqueue/gateway/controller/land_test.go +++ b/submitqueue/gateway/controller/land_test.go @@ -128,7 +128,7 @@ func TestLand_ReturnsSqid(t *testing.T) { result, err := controller.Land(ctx, testLandRequest("test-queue")) require.NoError(t, err) - assert.Equal(t, "test-queue/1", result.ID) + assert.Equal(t, "1", result.ID) } func TestLand_ReturnsErrorOnCounterFailure(t *testing.T) { @@ -144,9 +144,21 @@ func TestLand_ReturnsErrorOnCounterFailure(t *testing.T) { require.Error(t, err) } +func TestLand_ReturnsErrorOnNonPositiveCounterValue(t *testing.T) { + ctrl := gomock.NewController(t) + + cnt := countermock.NewMockCounter(ctrl) + cnt.EXPECT().Next(gomock.Any(), gomock.Any()).Return(int64(0), nil) + controller := newNoopLandController(t, ctrl, cnt) + + _, err := controller.Land(context.Background(), testLandRequest("test-queue")) + + require.Error(t, err) +} + // TestLand_ResolvesCounterForRequestQueue pins that the queue reaches the counter -// through the factory binding rather than through the domain string: the domain is a -// bare sequence name, and the sqid is still "/". +// through the factory binding rather than through the domain name: the value is a +// bare sequence name, and the sqid is the decimal counter value. func TestLand_ResolvesCounterForRequestQueue(t *testing.T) { var capturedDomain, capturedQueue string @@ -176,7 +188,7 @@ func TestLand_ResolvesCounterForRequestQueue(t *testing.T) { require.NoError(t, err) assert.Equal(t, "request", capturedDomain, "the domain is a sequence name, not a queue-qualified key") assert.Equal(t, "my-queue", capturedQueue, "the queue reaches the counter through the factory binding") - assert.Equal(t, "my-queue/1", result.ID) + assert.Equal(t, "1", result.ID) } func TestLand_ReturnsErrorOnEmptyQueue(t *testing.T) { @@ -223,7 +235,7 @@ func TestLand_ValidatesQueueLengthBeforeAllocatingSqid(t *testing.T) { return } require.NoError(t, err) - assert.Equal(t, tt.queue+"/1", result.ID) + assert.Equal(t, "1", result.ID) }) } } @@ -377,7 +389,7 @@ func TestLand_PublishesToQueue(t *testing.T) { return nil }, ), - summaryStore.EXPECT().Get(gomock.Any(), "test-queue/123").DoAndReturn( + summaryStore.EXPECT().Get(gomock.Any(), "123").DoAndReturn( func(context.Context, string) (entity.RequestSummary, error) { return receiptSummary, nil }, @@ -389,7 +401,7 @@ func TestLand_PublishesToQueue(t *testing.T) { return nil }, ), - queueStore.EXPECT().Get(gomock.Any(), gomock.Any(), "test-queue/123").DoAndReturn( + queueStore.EXPECT().Get(gomock.Any(), gomock.Any(), "123").DoAndReturn( func(context.Context, int64, string) (entity.RequestQueueSummary, error) { return entity.RequestQueueSummary{}, basestorage.ErrNotFound }, @@ -419,10 +431,10 @@ func TestLand_PublishesToQueue(t *testing.T) { result, err := controller.Land(ctx, req) require.NoError(t, err) - assert.Equal(t, "test-queue/123", result.ID) + assert.Equal(t, "123", result.ID) assert.Equal(t, entity.RequestSummary{ - RequestID: "test-queue/123", + RequestID: "123", Queue: "test-queue", ChangeURIs: []string{"github://github.example.com/uber/backend/pull/456/fedcba9876543210fedcba9876543210fedcba98"}, ReceivedAtMs: receiptSummary.ReceivedAtMs, @@ -433,7 +445,7 @@ func TestLand_PublishesToQueue(t *testing.T) { }, receiptSummary) assert.Positive(t, receiptSummary.ReceivedAtMs) assert.Equal(t, entity.RequestLog{ - RequestID: "test-queue/123", + RequestID: "123", Queue: "test-queue", TimestampMs: receiptSummary.ReceivedAtMs, Type: entity.RequestLogTypeStatus, @@ -446,10 +458,10 @@ func TestLand_PublishesToQueue(t *testing.T) { ChangeURI: "github://github.example.com/uber/backend/pull/456/fedcba9876543210fedcba9876543210fedcba98", Queue: "test-queue", ReceivedAtMs: receiptSummary.ReceivedAtMs, - RequestID: "test-queue/123", + RequestID: "123", }, persistedMapping) assert.Equal(t, entity.RequestQueueSummary{ - RequestID: "test-queue/123", + RequestID: "123", Queue: "test-queue", ChangeURIs: []string{"github://github.example.com/uber/backend/pull/456/fedcba9876543210fedcba9876543210fedcba98"}, ReceivedAtMs: receiptSummary.ReceivedAtMs, @@ -460,14 +472,14 @@ func TestLand_PublishesToQueue(t *testing.T) { // Verify message was published to the topic registered under TopicKeyStart assert.Equal(t, "start", publishedTopic) - assert.Equal(t, "test-queue/123", publishedMessage.ID) + assert.Equal(t, "123", publishedMessage.ID) assert.Equal(t, "test-queue", publishedMessage.Tenant) assert.Equal(t, "test-queue", publishedMessage.PartitionKey) // Verify payload can be deserialized deserializedReq, err := sqmq.UnmarshalLandRequest(publishedMessage.Payload) require.NoError(t, err) - assert.Equal(t, "test-queue/123", deserializedReq.ID) + assert.Equal(t, "123", deserializedReq.ID) assert.Equal(t, "test-queue", deserializedReq.Queue) assert.Equal(t, []string{"github://github.example.com/uber/backend/pull/456/fedcba9876543210fedcba9876543210fedcba98"}, deserializedReq.Change.URIs) assert.Equal(t, mergestrategy.MergeStrategyRebase, deserializedReq.LandStrategy) @@ -515,5 +527,5 @@ func TestLand_ReturnsSqidWhenAcceptedLogFailsAfterPublish(t *testing.T) { result, err := controller.Land(context.Background(), testLandRequest("test-queue")) require.NoError(t, err) - assert.Equal(t, "test-queue/999", result.ID) + assert.Equal(t, "999", result.ID) } diff --git a/submitqueue/gateway/controller/list_test.go b/submitqueue/gateway/controller/list_test.go index 5fd31d949..b7c79ae27 100644 --- a/submitqueue/gateway/controller/list_test.go +++ b/submitqueue/gateway/controller/list_test.go @@ -50,9 +50,9 @@ func TestList_ReturnsPageAndCursor(t *testing.T) { store.EXPECT().List(gomock.Any(), basestorage.RequestQueueSummaryQuery{ ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, Limit: 3, }).Return([]entity.RequestQueueSummary{ - {RequestID: "q/3", Queue: "q", ChangeURIs: []string{}, ReceivedAtMs: 190, Status: entity.RequestStatusAccepted, Metadata: map[string]string{}}, - {RequestID: "q/2", Queue: "q", ChangeURIs: []string{}, ReceivedAtMs: 180, Status: entity.RequestStatusLanded, Metadata: map[string]string{}}, - {RequestID: "q/1", Queue: "q", ChangeURIs: []string{}, ReceivedAtMs: 170, Status: entity.RequestStatusError, Metadata: map[string]string{}}, + {RequestID: "3", Queue: "q", ChangeURIs: []string{}, ReceivedAtMs: 190, Status: entity.RequestStatusAccepted, Metadata: map[string]string{}}, + {RequestID: "2", Queue: "q", ChangeURIs: []string{}, ReceivedAtMs: 180, Status: entity.RequestStatusLanded, Metadata: map[string]string{}}, + {RequestID: "1", Queue: "q", ChangeURIs: []string{}, ReceivedAtMs: 170, Status: entity.RequestStatusError, Metadata: map[string]string{}}, }, nil) controller := newConfiguredListController(ctrl, store) @@ -60,22 +60,22 @@ func TestList_ReturnsPageAndCursor(t *testing.T) { require.NoError(t, err) require.Len(t, result.Requests, 2) - assert.Equal(t, []string{"q/3", "q/2"}, []string{result.Requests[0].RequestID, result.Requests[1].RequestID}) + assert.Equal(t, []string{"3", "2"}, []string{result.Requests[0].RequestID, result.Requests[1].RequestID}) assert.Equal(t, int64(190), result.Requests[0].ReceivedAtMs) require.NotEmpty(t, result.NextPageToken) token, err := decodeListPageToken(result.NextPageToken) require.NoError(t, err) assert.Equal(t, int64(180), token.LastReceivedAtMs) - assert.Equal(t, "q/2", token.LastRequestID) + assert.Equal(t, "2", token.LastRequestID) } func TestList_UsesCursor(t *testing.T) { ctrl := gomock.NewController(t) store := storagemock.NewMockRequestQueueSummaryStore(ctrl) - token := encodeListPageToken(listPageToken{Queue: "q", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, LastReceivedAtMs: 180, LastRequestID: "q/2"}) + token := encodeListPageToken(listPageToken{Queue: "q", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, LastReceivedAtMs: 180, LastRequestID: "2"}) store.EXPECT().List(gomock.Any(), basestorage.RequestQueueSummaryQuery{ ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, Limit: 51, - HasCursor: true, Cursor: basestorage.RequestQueueSummaryCursor{ReceivedAtMs: 180, RequestID: "q/2"}, + HasCursor: true, Cursor: basestorage.RequestQueueSummaryCursor{ReceivedAtMs: 180, RequestID: "2"}, }).Return([]entity.RequestQueueSummary{}, nil) controller := newConfiguredListController(ctrl, store) @@ -87,7 +87,7 @@ func TestList_UsesCursor(t *testing.T) { } func TestList_Errors(t *testing.T) { - validToken := encodeListPageToken(listPageToken{Queue: "other", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, LastReceivedAtMs: 150, LastRequestID: "other/1"}) + validToken := encodeListPageToken(listPageToken{Queue: "other", ReceivedAtOrAfterMs: 100, ReceivedBeforeMs: 200, LastReceivedAtMs: 150, LastRequestID: "1"}) invalidFieldsToken := base64.RawURLEncoding.EncodeToString([]byte("queue=q&received_at_or_after_ms=100&received_before_ms=200&last_received_at_ms=150")) invalidNumberToken := base64.RawURLEncoding.EncodeToString([]byte("queue=q&received_at_or_after_ms=x&received_before_ms=200&last_received_at_ms=150&last_request_id=q%2F1")) backendErr := fmt.Errorf("store down") diff --git a/submitqueue/gateway/controller/log/log_test.go b/submitqueue/gateway/controller/log/log_test.go index 49b3fcb3b..41207b809 100644 --- a/submitqueue/gateway/controller/log/log_test.go +++ b/submitqueue/gateway/controller/log/log_test.go @@ -49,7 +49,7 @@ func TestController_Process(t *testing.T) { }{ { name: "success", - logEntry: newRequestLog("test-queue/1", entity.RequestStatusStarted, 1, "", nil), + logEntry: newRequestLog("1", entity.RequestStatusStarted, 1, "", nil), setupStore: func(ctrl *gomock.Controller) *requestcore.Materializer { return newLogControllerStore(ctrl, nil, nil, nil, nil) }, @@ -62,7 +62,7 @@ func TestController_Process(t *testing.T) { }, { name: "audit insert failure", - logEntry: newRequestLog("test-queue/2", entity.RequestStatusError, 3, "merge conflict", nil), + logEntry: newRequestLog("2", entity.RequestStatusError, 3, "merge conflict", nil), setupStore: func(ctrl *gomock.Controller) *requestcore.Materializer { return newLogControllerStore(ctrl, fmt.Errorf("audit down"), nil, nil, nil) }, @@ -70,7 +70,7 @@ func TestController_Process(t *testing.T) { }, { name: "summary read failure", - logEntry: newRequestLog("test-queue/2", entity.RequestStatusError, 3, "merge conflict", nil), + logEntry: newRequestLog("2", entity.RequestStatusError, 3, "merge conflict", nil), setupStore: func(ctrl *gomock.Controller) *requestcore.Materializer { return newLogControllerStore(ctrl, nil, fmt.Errorf("summary down"), nil, nil) }, @@ -78,7 +78,7 @@ func TestController_Process(t *testing.T) { }, { name: "summary update failure", - logEntry: newRequestLog("test-queue/2", entity.RequestStatusError, 3, "merge conflict", nil), + logEntry: newRequestLog("2", entity.RequestStatusError, 3, "merge conflict", nil), setupStore: func(ctrl *gomock.Controller) *requestcore.Materializer { return newLogControllerStore(ctrl, nil, nil, fmt.Errorf("summary update down"), nil) }, @@ -86,7 +86,7 @@ func TestController_Process(t *testing.T) { }, { name: "queue projection failure", - logEntry: newRequestLog("test-queue/2", entity.RequestStatusError, 3, "merge conflict", nil), + logEntry: newRequestLog("2", entity.RequestStatusError, 3, "merge conflict", nil), setupStore: func(ctrl *gomock.Controller) *requestcore.Materializer { return newLogControllerStore(ctrl, nil, nil, nil, fmt.Errorf("queue update down")) }, @@ -104,7 +104,7 @@ func TestController_Process(t *testing.T) { require.NoError(t, err) } controller := NewController(zaptest.NewLogger(t).Sugar(), tally.NoopScope, tt.setupStore(ctrl), topickey.TopicKeyLog, "gateway-log") - msg := entityqueue.NewMessage("test-queue/1", payload, "test-queue", nil) + msg := entityqueue.NewMessage("1", payload, "test-queue", nil) if tt.logEntry != nil { msg.Tenant = tt.logEntry.Queue } @@ -124,7 +124,7 @@ func TestController_Process(t *testing.T) { func TestController_Process_RejectsTenantPayloadQueueMismatch(t *testing.T) { ctrl := gomock.NewController(t) - logEntry := newRequestLog("test-queue/1", entity.RequestStatusStarted, 1, "", nil) + logEntry := newRequestLog("1", entity.RequestStatusStarted, 1, "", nil) payload, err := sqmq.Marshal(sqmq.LogFromEntity(*logEntry)) require.NoError(t, err) controller := NewController(zaptest.NewLogger(t).Sugar(), tally.NoopScope, newUnusedMaterializer(ctrl), topickey.TopicKeyLog, "gateway-log") @@ -154,7 +154,7 @@ func newLogControllerStore(ctrl *gomock.Controller, insertErr, getErr, updateErr return materializer } summaryStore.EXPECT().Get(gomock.Any(), gomock.Any()).Return(entity.RequestSummary{ - RequestID: "test-queue/2", Queue: "test-queue", ChangeURIs: []string{}, ReceivedAtMs: 1, + RequestID: "2", Queue: "test-queue", ChangeURIs: []string{}, ReceivedAtMs: 1, Status: entity.RequestStatusAccepted, StatusTimestampMs: 1, Version: 1, Metadata: map[string]string{}, }, getErr) if getErr != nil { @@ -164,8 +164,8 @@ func newLogControllerStore(ctrl *gomock.Controller, insertErr, getErr, updateErr if updateErr != nil { return materializer } - queueStore.EXPECT().Get(gomock.Any(), int64(1), "test-queue/2").Return(entity.RequestQueueSummary{ - RequestID: "test-queue/2", Queue: "test-queue", ChangeURIs: []string{}, ReceivedAtMs: 1, + queueStore.EXPECT().Get(gomock.Any(), int64(1), "2").Return(entity.RequestQueueSummary{ + RequestID: "2", Queue: "test-queue", ChangeURIs: []string{}, ReceivedAtMs: 1, Status: entity.RequestStatusAccepted, Version: 1, Metadata: map[string]string{}, }, nil) queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(queueErr) diff --git a/submitqueue/gateway/controller/read_errors.go b/submitqueue/gateway/controller/read_errors.go index 861f3c92d..15b3999f3 100644 --- a/submitqueue/gateway/controller/read_errors.go +++ b/submitqueue/gateway/controller/read_errors.go @@ -17,6 +17,8 @@ package controller import ( "errors" "fmt" + + "github.com/uber/submitqueue/platform/base/id" ) const ( @@ -98,6 +100,13 @@ func validateStoredIdentifier(name, value string) error { return nil } +func validateResourceID(name, value string) error { + if err := id.Validate(value); err != nil { + return fmt.Errorf("invalid %s %q: %v: %w", name, value, err, ErrInvalidRequest) + } + return nil +} + func validateChangeURIs(changeURIs []string) error { if len(changeURIs) == 0 { return fmt.Errorf("at least one change URI is required: %w", ErrInvalidRequest) diff --git a/submitqueue/gateway/controller/request_history.go b/submitqueue/gateway/controller/request_history.go index 26bb6d47b..efb4151d4 100644 --- a/submitqueue/gateway/controller/request_history.go +++ b/submitqueue/gateway/controller/request_history.go @@ -18,10 +18,9 @@ import ( "context" "fmt" "sort" - "strconv" - "strings" "github.com/uber-go/tally" + "github.com/uber/submitqueue/platform/base/id" "github.com/uber/submitqueue/platform/errs" "github.com/uber/submitqueue/platform/metrics" "github.com/uber/submitqueue/submitqueue/entity" @@ -58,7 +57,7 @@ func (c *requestHistoryController) GetRequestHistoryByID(ctx context.Context, re op := metrics.Begin(c.metricsScope, "get_by_id", metrics.StorageLatencyBuckets) defer func() { op.Complete(retErr) }() - if err := validateStoredIdentifier("sqid", req.ID); err != nil { + if err := validateResourceID("sqid", req.ID); err != nil { return nil, fmt.Errorf("GetRequestHistoryByID invalid request: %w", err) } if err := validateQueueIdentifier(req.Queue); err != nil { @@ -114,8 +113,11 @@ func (c *requestHistoryController) GetRequestHistoryByChangeURI(ctx context.Cont return nil, errs.NewUserError(&TooManyChangeRequestsError{ChangeURI: req.ChangeURI, Limit: maxChangeRequestResults}) } - histories := make([]requestHistoryWithCounter, 0, len(mappings)) + histories := make([]entity.RequestHistory, 0, len(mappings)) for _, mapping := range mappings { + if err := id.Validate(mapping.RequestID); err != nil { + return nil, &InternalConsistencyError{Message: fmt.Sprintf("invalid mapped sqid %q: %v", mapping.RequestID, err)} + } logs, err := logStore.List(ctx, mapping.RequestID) if err != nil { if basestorage.IsNotFound(err) { @@ -123,16 +125,9 @@ func (c *requestHistoryController) GetRequestHistoryByChangeURI(ctx context.Cont } return nil, fmt.Errorf("GetRequestHistoryByChangeURI failed to list request logs change_uri=%s sqid=%s: %w", req.ChangeURI, mapping.RequestID, err) } - counter, err := sqidCounter(mapping.RequestID) - if err != nil { - return nil, &InternalConsistencyError{Message: fmt.Sprintf("invalid mapped sqid %q: %v", mapping.RequestID, err)} - } - histories = append(histories, requestHistoryWithCounter{ - counter: counter, - history: entity.RequestHistory{ - RequestID: mapping.RequestID, - Events: append([]entity.RequestLog{}, logs...), - }, + histories = append(histories, entity.RequestHistory{ + RequestID: mapping.RequestID, + Events: append([]entity.RequestLog{}, logs...), }) } if len(histories) == 0 { @@ -140,36 +135,14 @@ func (c *requestHistoryController) GetRequestHistoryByChangeURI(ctx context.Cont } sort.Slice(histories, func(i, j int) bool { - if histories[i].counter != histories[j].counter { - return histories[i].counter < histories[j].counter - } - return histories[i].history.RequestID < histories[j].history.RequestID + comparison, _ := id.Compare(histories[i].RequestID, histories[j].RequestID) + return comparison < 0 }) - result = make([]entity.RequestHistory, len(histories)) - for i, history := range histories { - result[i] = history.history - } + result = histories c.logger.Debugw("request histories retrieved", "change_uri", req.ChangeURI, "request_count", len(result), ) return result, nil } - -type requestHistoryWithCounter struct { - counter int64 - history entity.RequestHistory -} - -func sqidCounter(sqid string) (int64, error) { - separator := strings.LastIndexByte(sqid, '/') - if separator < 0 || separator == len(sqid)-1 { - return 0, fmt.Errorf("missing numeric counter") - } - counter, err := strconv.ParseInt(sqid[separator+1:], 10, 64) - if err != nil || counter <= 0 { - return 0, fmt.Errorf("invalid numeric counter") - } - return counter, nil -} diff --git a/submitqueue/gateway/controller/request_history_test.go b/submitqueue/gateway/controller/request_history_test.go index 04f4c95d9..4f252ac1c 100644 --- a/submitqueue/gateway/controller/request_history_test.go +++ b/submitqueue/gateway/controller/request_history_test.go @@ -34,14 +34,14 @@ func TestGetRequestHistoryByID(t *testing.T) { ctrl := gomock.NewController(t) logStore := storagemock.NewMockRequestLogStore(ctrl) uriStore := storagemock.NewMockRequestURIStore(ctrl) - logStore.EXPECT().List(gomock.Any(), "queue/1").Return([]entity.RequestLog{ - {RequestID: "queue/1", TimestampMs: 10, Status: entity.RequestStatusAccepted, Metadata: map[string]string{}}, - {RequestID: "queue/1", TimestampMs: 20, Status: entity.RequestStatusStarted, LastError: "retry", Metadata: map[string]string{"attempt": "1"}}, - {RequestID: "queue/1", TimestampMs: 20, Status: entity.RequestStatusStarted, LastError: "retry", Metadata: map[string]string{"attempt": "1"}}, + logStore.EXPECT().List(gomock.Any(), "1").Return([]entity.RequestLog{ + {RequestID: "1", TimestampMs: 10, Status: entity.RequestStatusAccepted, Metadata: map[string]string{}}, + {RequestID: "1", TimestampMs: 20, Status: entity.RequestStatusStarted, LastError: "retry", Metadata: map[string]string{"attempt": "1"}}, + {RequestID: "1", TimestampMs: 20, Status: entity.RequestStatusStarted, LastError: "retry", Metadata: map[string]string{"attempt": "1"}}, }, nil) controller := NewRequestHistoryController(zap.NewNop().Sugar(), tally.NoopScope, readModelFactory(ctrl, nil, logStore, uriStore)) - events, err := controller.GetRequestHistoryByID(context.Background(), entity.GetRequestHistoryByIDRequest{ID: "queue/1", Queue: "queue"}) + events, err := controller.GetRequestHistoryByID(context.Background(), entity.GetRequestHistoryByIDRequest{ID: "1", Queue: "queue"}) require.NoError(t, err) require.Len(t, events, 3) @@ -56,24 +56,24 @@ func TestGetRequestHistoryByChangeURI(t *testing.T) { logStore := storagemock.NewMockRequestLogStore(ctrl) uriStore := storagemock.NewMockRequestURIStore(ctrl) uriStore.EXPECT().ListByURI(gomock.Any(), "uri", 101).Return([]entity.RequestURI{ - {ChangeURI: "uri", RequestID: "queue/10"}, - {ChangeURI: "uri", RequestID: "b/2"}, - {ChangeURI: "uri", RequestID: "missing/3"}, - {ChangeURI: "uri", RequestID: "queue/1"}, - {ChangeURI: "uri", RequestID: "a/2"}, + {ChangeURI: "uri", RequestID: "10"}, + {ChangeURI: "uri", RequestID: "2"}, + {ChangeURI: "uri", RequestID: "3"}, + {ChangeURI: "uri", RequestID: "1"}, + {ChangeURI: "uri", RequestID: "4"}, }, nil) - logStore.EXPECT().List(gomock.Any(), "queue/10").Return([]entity.RequestLog{{RequestID: "queue/10", TimestampMs: 10, Status: entity.RequestStatusLanded}}, nil) - logStore.EXPECT().List(gomock.Any(), "b/2").Return([]entity.RequestLog{{RequestID: "b/2", TimestampMs: 2, Status: entity.RequestStatusStarted}}, nil) - logStore.EXPECT().List(gomock.Any(), "missing/3").Return(nil, basestorage.ErrNotFound) - logStore.EXPECT().List(gomock.Any(), "queue/1").Return([]entity.RequestLog{{RequestID: "queue/1", TimestampMs: 1, Status: entity.RequestStatusAccepted}}, nil) - logStore.EXPECT().List(gomock.Any(), "a/2").Return([]entity.RequestLog{{RequestID: "a/2", TimestampMs: 2, Status: entity.RequestStatusError}}, nil) + logStore.EXPECT().List(gomock.Any(), "10").Return([]entity.RequestLog{{RequestID: "10", TimestampMs: 10, Status: entity.RequestStatusLanded}}, nil) + logStore.EXPECT().List(gomock.Any(), "2").Return([]entity.RequestLog{{RequestID: "2", TimestampMs: 2, Status: entity.RequestStatusStarted}}, nil) + logStore.EXPECT().List(gomock.Any(), "3").Return(nil, basestorage.ErrNotFound) + logStore.EXPECT().List(gomock.Any(), "1").Return([]entity.RequestLog{{RequestID: "1", TimestampMs: 1, Status: entity.RequestStatusAccepted}}, nil) + logStore.EXPECT().List(gomock.Any(), "4").Return([]entity.RequestLog{{RequestID: "4", TimestampMs: 4, Status: entity.RequestStatusError}}, nil) controller := NewRequestHistoryController(zap.NewNop().Sugar(), tally.NoopScope, readModelFactory(ctrl, nil, logStore, uriStore)) histories, err := controller.GetRequestHistoryByChangeURI(context.Background(), entity.GetRequestHistoryByChangeURIRequest{ChangeURI: "uri", Queue: "queue"}) require.NoError(t, err) require.Len(t, histories, 4) - assert.Equal(t, []string{"queue/1", "a/2", "b/2", "queue/10"}, []string{ + assert.Equal(t, []string{"1", "2", "4", "10"}, []string{ histories[0].RequestID, histories[1].RequestID, histories[2].RequestID, @@ -102,6 +102,15 @@ func TestHistoryErrors(t *testing.T) { wantInvalid: true, wantUser: true, }, + { + name: "composite sqid", + call: func(c RequestHistoryController) error { + _, err := c.GetRequestHistoryByID(context.Background(), entity.GetRequestHistoryByIDRequest{ID: "queue/1", Queue: "queue"}) + return err + }, + wantInvalid: true, + wantUser: true, + }, { name: "empty change URI", call: func(c RequestHistoryController) error { @@ -114,10 +123,10 @@ func TestHistoryErrors(t *testing.T) { { name: "sqid not found", setup: func(logStore *storagemock.MockRequestLogStore, _ *storagemock.MockRequestURIStore) { - logStore.EXPECT().List(gomock.Any(), "missing/1").Return(nil, basestorage.ErrNotFound) + logStore.EXPECT().List(gomock.Any(), "1").Return(nil, basestorage.ErrNotFound) }, call: func(c RequestHistoryController) error { - _, err := c.GetRequestHistoryByID(context.Background(), entity.GetRequestHistoryByIDRequest{ID: "missing/1", Queue: "missing"}) + _, err := c.GetRequestHistoryByID(context.Background(), entity.GetRequestHistoryByIDRequest{ID: "1", Queue: "missing"}) return err }, wantNotFound: true, @@ -126,10 +135,10 @@ func TestHistoryErrors(t *testing.T) { { name: "sqid storage failure", setup: func(logStore *storagemock.MockRequestLogStore, _ *storagemock.MockRequestURIStore) { - logStore.EXPECT().List(gomock.Any(), "queue/1").Return(nil, backendErr) + logStore.EXPECT().List(gomock.Any(), "1").Return(nil, backendErr) }, call: func(c RequestHistoryController) error { - _, err := c.GetRequestHistoryByID(context.Background(), entity.GetRequestHistoryByIDRequest{ID: "queue/1", Queue: "queue"}) + _, err := c.GetRequestHistoryByID(context.Background(), entity.GetRequestHistoryByIDRequest{ID: "1", Queue: "queue"}) return err }, }, @@ -160,8 +169,8 @@ func TestHistoryErrors(t *testing.T) { { name: "all mapped logs absent", setup: func(logStore *storagemock.MockRequestLogStore, uriStore *storagemock.MockRequestURIStore) { - uriStore.EXPECT().ListByURI(gomock.Any(), "uri", 101).Return([]entity.RequestURI{{RequestID: "queue/1"}}, nil) - logStore.EXPECT().List(gomock.Any(), "queue/1").Return(nil, basestorage.ErrNotFound) + uriStore.EXPECT().ListByURI(gomock.Any(), "uri", 101).Return([]entity.RequestURI{{RequestID: "1"}}, nil) + logStore.EXPECT().List(gomock.Any(), "1").Return(nil, basestorage.ErrNotFound) }, call: func(c RequestHistoryController) error { _, err := c.GetRequestHistoryByChangeURI(context.Background(), entity.GetRequestHistoryByChangeURIRequest{ChangeURI: "uri", Queue: "queue"}) @@ -172,9 +181,8 @@ func TestHistoryErrors(t *testing.T) { }, { name: "mapped sqid malformed", - setup: func(logStore *storagemock.MockRequestLogStore, uriStore *storagemock.MockRequestURIStore) { + setup: func(_ *storagemock.MockRequestLogStore, uriStore *storagemock.MockRequestURIStore) { uriStore.EXPECT().ListByURI(gomock.Any(), "uri", 101).Return([]entity.RequestURI{{RequestID: "malformed"}}, nil) - logStore.EXPECT().List(gomock.Any(), "malformed").Return([]entity.RequestLog{{RequestID: "malformed"}}, nil) }, call: func(c RequestHistoryController) error { _, err := c.GetRequestHistoryByChangeURI(context.Background(), entity.GetRequestHistoryByChangeURIRequest{ChangeURI: "uri", Queue: "queue"}) diff --git a/submitqueue/gateway/controller/request_summary.go b/submitqueue/gateway/controller/request_summary.go index 2d63f41d9..2fccb421c 100644 --- a/submitqueue/gateway/controller/request_summary.go +++ b/submitqueue/gateway/controller/request_summary.go @@ -55,7 +55,7 @@ func (c *requestSummaryController) GetRequestSummaryByID(ctx context.Context, re op := metrics.Begin(c.metricsScope, "get_by_id", metrics.StorageLatencyBuckets) defer func() { op.Complete(retErr) }() - if err := validateStoredIdentifier("sqid", req.ID); err != nil { + if err := validateResourceID("sqid", req.ID); err != nil { return entity.RequestSummary{}, fmt.Errorf("GetRequestSummaryByID invalid request: %w", err) } if err := validateQueueIdentifier(req.Queue); err != nil { diff --git a/submitqueue/gateway/controller/request_summary_test.go b/submitqueue/gateway/controller/request_summary_test.go index d96355968..68356956b 100644 --- a/submitqueue/gateway/controller/request_summary_test.go +++ b/submitqueue/gateway/controller/request_summary_test.go @@ -34,8 +34,8 @@ func TestGetRequestSummaryByID(t *testing.T) { ctrl := gomock.NewController(t) summaryStore := storagemock.NewMockRequestSummaryStore(ctrl) uriStore := storagemock.NewMockRequestURIStore(ctrl) - summaryStore.EXPECT().Get(gomock.Any(), "test-queue/1").Return(entity.RequestSummary{ - RequestID: "test-queue/1", + summaryStore.EXPECT().Get(gomock.Any(), "1").Return(entity.RequestSummary{ + RequestID: "1", Queue: "test-queue", ChangeURIs: []string{"github://uber/repo/pull/1/abc"}, ReceivedAtMs: 100, @@ -45,10 +45,10 @@ func TestGetRequestSummaryByID(t *testing.T) { }, nil) controller := NewRequestSummaryController(zap.NewNop().Sugar(), tally.NoopScope, readModelFactory(ctrl, summaryStore, nil, uriStore)) - summary, err := controller.GetRequestSummaryByID(context.Background(), entity.GetRequestSummaryByIDRequest{ID: "test-queue/1", Queue: "test-queue"}) + summary, err := controller.GetRequestSummaryByID(context.Background(), entity.GetRequestSummaryByIDRequest{ID: "1", Queue: "test-queue"}) require.NoError(t, err) - assert.Equal(t, "test-queue/1", summary.RequestID) + assert.Equal(t, "1", summary.RequestID) assert.Equal(t, "test-queue", summary.Queue) assert.Equal(t, []string{"github://uber/repo/pull/1/abc"}, summary.ChangeURIs) assert.Equal(t, int64(100), summary.ReceivedAtMs) @@ -62,18 +62,18 @@ func TestGetRequestSummaryByChangeURI(t *testing.T) { summaryStore := storagemock.NewMockRequestSummaryStore(ctrl) uriStore := storagemock.NewMockRequestURIStore(ctrl) uriStore.EXPECT().ListByURI(gomock.Any(), "uri", 101).Return([]entity.RequestURI{ - {ChangeURI: "uri", ReceivedAtMs: 200, RequestID: "queue/2"}, - {ChangeURI: "uri", ReceivedAtMs: 100, RequestID: "queue/1"}, + {ChangeURI: "uri", ReceivedAtMs: 200, RequestID: "2"}, + {ChangeURI: "uri", ReceivedAtMs: 100, RequestID: "1"}, }, nil) - summaryStore.EXPECT().Get(gomock.Any(), "queue/2").Return(entity.RequestSummary{RequestID: "queue/2", ReceivedAtMs: 200, Status: entity.RequestStatusLanded, ChangeURIs: []string{}}, nil) - summaryStore.EXPECT().Get(gomock.Any(), "queue/1").Return(entity.RequestSummary{RequestID: "queue/1", ReceivedAtMs: 100, Status: entity.RequestStatusError, ChangeURIs: []string{}}, nil) + summaryStore.EXPECT().Get(gomock.Any(), "2").Return(entity.RequestSummary{RequestID: "2", ReceivedAtMs: 200, Status: entity.RequestStatusLanded, ChangeURIs: []string{}}, nil) + summaryStore.EXPECT().Get(gomock.Any(), "1").Return(entity.RequestSummary{RequestID: "1", ReceivedAtMs: 100, Status: entity.RequestStatusError, ChangeURIs: []string{}}, nil) controller := NewRequestSummaryController(zap.NewNop().Sugar(), tally.NoopScope, readModelFactory(ctrl, summaryStore, nil, uriStore)) summaries, err := controller.GetRequestSummaryByChangeURI(context.Background(), entity.GetRequestSummaryByChangeURIRequest{ChangeURI: "uri", Queue: "queue"}) require.NoError(t, err) require.Len(t, summaries, 2) - assert.Equal(t, []string{"queue/2", "queue/1"}, []string{summaries[0].RequestID, summaries[1].RequestID}) + assert.Equal(t, []string{"2", "1"}, []string{summaries[0].RequestID, summaries[1].RequestID}) } func TestStatusErrors(t *testing.T) { @@ -97,6 +97,15 @@ func TestStatusErrors(t *testing.T) { wantInvalid: true, wantUser: true, }, + { + name: "prefixed sqid", + call: func(c RequestSummaryController) error { + _, err := c.GetRequestSummaryByID(context.Background(), entity.GetRequestSummaryByIDRequest{ID: "request.1", Queue: "queue"}) + return err + }, + wantInvalid: true, + wantUser: true, + }, { name: "empty change URI", call: func(c RequestSummaryController) error { @@ -109,10 +118,10 @@ func TestStatusErrors(t *testing.T) { { name: "sqid not found", setup: func(summaryStore *storagemock.MockRequestSummaryStore, _ *storagemock.MockRequestURIStore) { - summaryStore.EXPECT().Get(gomock.Any(), "missing/1").Return(entity.RequestSummary{}, basestorage.ErrNotFound) + summaryStore.EXPECT().Get(gomock.Any(), "1").Return(entity.RequestSummary{}, basestorage.ErrNotFound) }, call: func(c RequestSummaryController) error { - _, err := c.GetRequestSummaryByID(context.Background(), entity.GetRequestSummaryByIDRequest{ID: "missing/1", Queue: "missing"}) + _, err := c.GetRequestSummaryByID(context.Background(), entity.GetRequestSummaryByIDRequest{ID: "1", Queue: "missing"}) return err }, wantNotFound: true, @@ -121,13 +130,13 @@ func TestStatusErrors(t *testing.T) { { name: "sqid still accepting", setup: func(summaryStore *storagemock.MockRequestSummaryStore, _ *storagemock.MockRequestURIStore) { - summaryStore.EXPECT().Get(gomock.Any(), "queue/1").Return(entity.RequestSummary{ - RequestID: "queue/1", + summaryStore.EXPECT().Get(gomock.Any(), "1").Return(entity.RequestSummary{ + RequestID: "1", Status: entity.RequestStatusAccepting, }, nil) }, call: func(c RequestSummaryController) error { - _, err := c.GetRequestSummaryByID(context.Background(), entity.GetRequestSummaryByIDRequest{ID: "queue/1", Queue: "queue"}) + _, err := c.GetRequestSummaryByID(context.Background(), entity.GetRequestSummaryByIDRequest{ID: "1", Queue: "queue"}) return err }, wantNotFound: true, @@ -136,10 +145,10 @@ func TestStatusErrors(t *testing.T) { { name: "sqid storage failure", setup: func(summaryStore *storagemock.MockRequestSummaryStore, _ *storagemock.MockRequestURIStore) { - summaryStore.EXPECT().Get(gomock.Any(), "queue/1").Return(entity.RequestSummary{}, backendErr) + summaryStore.EXPECT().Get(gomock.Any(), "1").Return(entity.RequestSummary{}, backendErr) }, call: func(c RequestSummaryController) error { - _, err := c.GetRequestSummaryByID(context.Background(), entity.GetRequestSummaryByIDRequest{ID: "queue/1", Queue: "queue"}) + _, err := c.GetRequestSummaryByID(context.Background(), entity.GetRequestSummaryByIDRequest{ID: "1", Queue: "queue"}) return err }, }, @@ -170,8 +179,8 @@ func TestStatusErrors(t *testing.T) { { name: "mapped summary missing", setup: func(summaryStore *storagemock.MockRequestSummaryStore, uriStore *storagemock.MockRequestURIStore) { - uriStore.EXPECT().ListByURI(gomock.Any(), "uri", 101).Return([]entity.RequestURI{{RequestID: "missing/1"}}, nil) - summaryStore.EXPECT().Get(gomock.Any(), "missing/1").Return(entity.RequestSummary{}, basestorage.ErrNotFound) + uriStore.EXPECT().ListByURI(gomock.Any(), "uri", 101).Return([]entity.RequestURI{{RequestID: "1"}}, nil) + summaryStore.EXPECT().Get(gomock.Any(), "1").Return(entity.RequestSummary{}, basestorage.ErrNotFound) }, call: func(c RequestSummaryController) error { _, err := c.GetRequestSummaryByChangeURI(context.Background(), entity.GetRequestSummaryByChangeURIRequest{ChangeURI: "uri", Queue: "queue"}) diff --git a/submitqueue/gateway/core/request/materializer_test.go b/submitqueue/gateway/core/request/materializer_test.go index bf37bdc49..9704edd46 100644 --- a/submitqueue/gateway/core/request/materializer_test.go +++ b/submitqueue/gateway/core/request/materializer_test.go @@ -30,18 +30,18 @@ import ( func TestMaterializer_PersistLog(t *testing.T) { base := testRequestSummary() - log := entity.RequestLog{RequestID: "q/1", TimestampMs: 20, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusLanded, RequestVersion: 2, Metadata: map[string]string{}} + log := entity.RequestLog{RequestID: "1", TimestampMs: 20, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusLanded, RequestVersion: 2, Metadata: map[string]string{}} t.Run("winning log updates both projections", func(t *testing.T) { ctrl := gomock.NewController(t) m, summaryStore, queueStore, _, logStore := materializerStores(ctrl) logStore.EXPECT().Insert(gomock.Any(), log).Return(nil) - summaryStore.EXPECT().Get(gomock.Any(), "q/1").Return(base, nil) + summaryStore.EXPECT().Get(gomock.Any(), "1").Return(base, nil) summaryStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).DoAndReturn(func(_ context.Context, updated entity.RequestSummary, _, _ int32) error { assert.Equal(t, entity.RequestStatusLanded, updated.Status) assert.Equal(t, int32(2), updated.RequestVersion) return nil }) - queueStore.EXPECT().Get(gomock.Any(), int64(10), "q/1").Return(queueSummaryFromSummary(base), nil) + queueStore.EXPECT().Get(gomock.Any(), int64(10), "1").Return(queueSummaryFromSummary(base), nil) queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(nil) require.NoError(t, m.PersistLog(context.Background(), log)) }) @@ -52,14 +52,14 @@ func TestMaterializer_PersistLog(t *testing.T) { current := base current.Status = entity.RequestStatusLanded current.RequestVersion = 0 - incoming := entity.RequestLog{RequestID: "q/1", TimestampMs: 20, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusSpeculating, RequestVersion: 0, Metadata: map[string]string{}} + incoming := entity.RequestLog{RequestID: "1", TimestampMs: 20, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusSpeculating, RequestVersion: 0, Metadata: map[string]string{}} logStore.EXPECT().Insert(gomock.Any(), incoming).Return(nil) - summaryStore.EXPECT().Get(gomock.Any(), "q/1").Return(current, nil) + summaryStore.EXPECT().Get(gomock.Any(), "1").Return(current, nil) summaryStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).DoAndReturn(func(_ context.Context, updated entity.RequestSummary, _, _ int32) error { assert.Equal(t, entity.RequestStatusSpeculating, updated.Status) return nil }) - queueStore.EXPECT().Get(gomock.Any(), int64(10), "q/1").Return(queueSummaryFromSummary(current), nil) + queueStore.EXPECT().Get(gomock.Any(), int64(10), "1").Return(queueSummaryFromSummary(current), nil) queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(nil) require.NoError(t, m.PersistLog(context.Background(), incoming)) }) @@ -74,17 +74,17 @@ func TestMaterializer_PersistLog(t *testing.T) { current.StatusTimestampMs = 10 // Newer than the summary, so only the entry's type keeps it out. event := entity.RequestLog{ - RequestID: "q/1", TimestampMs: 20, + RequestID: "1", TimestampMs: 20, Type: entity.RequestLogTypeEvent, Event: buildEvent, Metadata: map[string]string{"build_id": "b/7"}, } // Inserted: the entry is the request's history. logStore.EXPECT().Insert(gomock.Any(), event).Return(nil) - summaryStore.EXPECT().Get(gomock.Any(), "q/1").Return(current, nil) + summaryStore.EXPECT().Get(gomock.Any(), "1").Return(current, nil) // No summaryStore.Update expectation — gomock fails the test if // the event moves the request's current status. - queueStore.EXPECT().Get(gomock.Any(), int64(10), "q/1").Return(queueSummaryFromSummary(current), nil) + queueStore.EXPECT().Get(gomock.Any(), int64(10), "1").Return(queueSummaryFromSummary(current), nil) require.NoError(t, m.PersistLog(context.Background(), event)) }) @@ -95,15 +95,15 @@ func TestMaterializer_PersistLog(t *testing.T) { ctrl := gomock.NewController(t) m, summaryStore, queueStore, _, logStore := materializerStores(ctrl) logStore.EXPECT().Insert(gomock.Any(), log).Return(nil) - summaryStore.EXPECT().Get(gomock.Any(), "q/1").Return(base, nil) + summaryStore.EXPECT().Get(gomock.Any(), "1").Return(base, nil) summaryStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(storage.ErrVersionMismatch) advanced := base advanced.Status = entity.RequestStatusLanded advanced.RequestVersion = 2 advanced.StatusTimestampMs = 20 advanced.Version = 2 - summaryStore.EXPECT().Get(gomock.Any(), "q/1").Return(advanced, nil) - queueStore.EXPECT().Get(gomock.Any(), int64(10), "q/1").Return(queueSummaryFromSummary(base), nil) + summaryStore.EXPECT().Get(gomock.Any(), "1").Return(advanced, nil) + queueStore.EXPECT().Get(gomock.Any(), int64(10), "1").Return(queueSummaryFromSummary(base), nil) queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(nil) require.NoError(t, m.PersistLog(context.Background(), log)) }) @@ -117,8 +117,8 @@ func TestMaterializer_PersistLog(t *testing.T) { advanced.RequestVersion = 2 advanced.StatusTimestampMs = 20 advanced.Version = 2 - summaryStore.EXPECT().Get(gomock.Any(), "q/1").Return(advanced, nil) - queueStore.EXPECT().Get(gomock.Any(), int64(10), "q/1").Return(queueSummaryFromSummary(base), nil) + summaryStore.EXPECT().Get(gomock.Any(), "1").Return(advanced, nil) + queueStore.EXPECT().Get(gomock.Any(), int64(10), "1").Return(queueSummaryFromSummary(base), nil) queueStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(nil) require.NoError(t, m.PersistLog(context.Background(), log)) }) @@ -127,16 +127,16 @@ func TestMaterializer_PersistLog(t *testing.T) { ctrl := gomock.NewController(t) m, summaryStore, queueStore, uriStore, logStore := materializerStores(ctrl) logStore.EXPECT().Insert(gomock.Any(), log).Return(nil) - summaryStore.EXPECT().Get(gomock.Any(), "q/1").Return(base, nil) + summaryStore.EXPECT().Get(gomock.Any(), "1").Return(base, nil) summaryStore.EXPECT().Update(gomock.Any(), gomock.Any(), int32(1), int32(2)).Return(nil) activated := base activated.Status = entity.RequestStatusLanded activated.RequestVersion = 2 activated.StatusTimestampMs = 20 activated.Version = 2 - queueStore.EXPECT().Get(gomock.Any(), int64(10), "q/1").Return(entity.RequestQueueSummary{}, storage.ErrNotFound) - uriStore.EXPECT().Create(gomock.Any(), entity.RequestURI{ChangeURI: "uri/1", Queue: "q", ReceivedAtMs: 10, RequestID: "q/1"}).Return(nil) - uriStore.EXPECT().Create(gomock.Any(), entity.RequestURI{ChangeURI: "uri/2", Queue: "q", ReceivedAtMs: 10, RequestID: "q/1"}).Return(nil) + queueStore.EXPECT().Get(gomock.Any(), int64(10), "1").Return(entity.RequestQueueSummary{}, storage.ErrNotFound) + uriStore.EXPECT().Create(gomock.Any(), entity.RequestURI{ChangeURI: "uri/1", Queue: "q", ReceivedAtMs: 10, RequestID: "1"}).Return(nil) + uriStore.EXPECT().Create(gomock.Any(), entity.RequestURI{ChangeURI: "uri/2", Queue: "q", ReceivedAtMs: 10, RequestID: "1"}).Return(nil) queueStore.EXPECT().Create(gomock.Any(), queueSummaryFromSummary(activated)).Return(nil) require.NoError(t, m.PersistLog(context.Background(), log)) }) @@ -151,9 +151,9 @@ func TestMaterializer_PersistLog(t *testing.T) { advanced.StatusTimestampMs = 20 advanced.Version = 2 logStore.EXPECT().Insert(gomock.Any(), log).Return(nil).Times(2) - summaryStore.EXPECT().Get(gomock.Any(), "q/1").Return(advanced, nil).Times(2) - queueStore.EXPECT().Get(gomock.Any(), int64(10), "q/1").Return(entity.RequestQueueSummary{}, errors.New("queue store down")) - queueStore.EXPECT().Get(gomock.Any(), int64(10), "q/1").Return(queueSummaryFromSummary(advanced), nil) + summaryStore.EXPECT().Get(gomock.Any(), "1").Return(advanced, nil).Times(2) + queueStore.EXPECT().Get(gomock.Any(), int64(10), "1").Return(entity.RequestQueueSummary{}, errors.New("queue store down")) + queueStore.EXPECT().Get(gomock.Any(), int64(10), "1").Return(queueSummaryFromSummary(advanced), nil) require.Error(t, materializer.PersistLog(context.Background(), log)) require.NoError(t, materializer.PersistLog(context.Background(), log)) @@ -163,7 +163,7 @@ func TestMaterializer_PersistLog(t *testing.T) { ctrl := gomock.NewController(t) m, summaryStore, _, _, logStore := materializerStores(ctrl) logStore.EXPECT().Insert(gomock.Any(), log).Return(nil) - summaryStore.EXPECT().Get(gomock.Any(), "q/1").Return(entity.RequestSummary{}, storage.ErrNotFound) + summaryStore.EXPECT().Get(gomock.Any(), "1").Return(entity.RequestSummary{}, storage.ErrNotFound) require.Error(t, m.PersistLog(context.Background(), log)) }) @@ -176,10 +176,10 @@ func TestMaterializer_PersistLog(t *testing.T) { advanced.RequestVersion = 2 advanced.StatusTimestampMs = 20 advanced.Version = 2 - summaryStore.EXPECT().Get(gomock.Any(), "q/1").Return(advanced, nil) + summaryStore.EXPECT().Get(gomock.Any(), "1").Return(advanced, nil) queueAhead := queueSummaryFromSummary(advanced) queueAhead.Version = 3 - queueStore.EXPECT().Get(gomock.Any(), int64(10), "q/1").Return(queueAhead, nil) + queueStore.EXPECT().Get(gomock.Any(), int64(10), "1").Return(queueAhead, nil) require.NoError(t, m.PersistLog(context.Background(), log)) }) } @@ -329,7 +329,7 @@ func materializerStores(ctrl *gomock.Controller) (*Materializer, *storagemock.Mo func testRequestSummary() entity.RequestSummary { return entity.RequestSummary{ - RequestID: "q/1", Queue: "q", ChangeURIs: []string{"uri/1", "uri/2"}, ReceivedAtMs: 10, + RequestID: "1", Queue: "q", ChangeURIs: []string{"uri/1", "uri/2"}, ReceivedAtMs: 10, Status: entity.RequestStatusAccepting, StatusTimestampMs: 10, Version: 1, Metadata: map[string]string{}, } } diff --git a/submitqueue/gateway/core/request/request_test.go b/submitqueue/gateway/core/request/request_test.go index fd14736ca..d166ecb98 100644 --- a/submitqueue/gateway/core/request/request_test.go +++ b/submitqueue/gateway/core/request/request_test.go @@ -37,58 +37,58 @@ func TestGetCurrentStateFromRequestLog(t *testing.T) { { name: "single record", logs: []entity.RequestLog{ - {RequestID: "q/1", TimestampMs: 1000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusStarted, RequestVersion: 1, LastError: "", Metadata: map[string]string{}}, + {RequestID: "1", TimestampMs: 1000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusStarted, RequestVersion: 1, LastError: "", Metadata: map[string]string{}}, }, expected: CurrentState{Status: entity.RequestStatusStarted, LastError: "", Metadata: map[string]string{}}, }, { name: "terminal status wins over later non-terminal", logs: []entity.RequestLog{ - {RequestID: "q/1", TimestampMs: 1000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusStarted, RequestVersion: 1, LastError: "", Metadata: map[string]string{}}, - {RequestID: "q/1", TimestampMs: 2000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusLanded, RequestVersion: 3, LastError: "", Metadata: map[string]string{"batch": "b1"}}, - {RequestID: "q/1", TimestampMs: 3000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusSpeculating, RequestVersion: 0, LastError: "", Metadata: map[string]string{}}, + {RequestID: "1", TimestampMs: 1000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusStarted, RequestVersion: 1, LastError: "", Metadata: map[string]string{}}, + {RequestID: "1", TimestampMs: 2000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusLanded, RequestVersion: 3, LastError: "", Metadata: map[string]string{"batch": "b1"}}, + {RequestID: "1", TimestampMs: 3000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusSpeculating, RequestVersion: 0, LastError: "", Metadata: map[string]string{}}, }, expected: CurrentState{Status: entity.RequestStatusLanded, LastError: "", Metadata: map[string]string{"batch": "b1"}}, }, { name: "terminal error status with last error", logs: []entity.RequestLog{ - {RequestID: "q/1", TimestampMs: 1000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusStarted, RequestVersion: 1, LastError: "", Metadata: map[string]string{}}, - {RequestID: "q/1", TimestampMs: 2000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusError, RequestVersion: 4, LastError: "merge conflict", Metadata: map[string]string{"step": "land"}}, + {RequestID: "1", TimestampMs: 1000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusStarted, RequestVersion: 1, LastError: "", Metadata: map[string]string{}}, + {RequestID: "1", TimestampMs: 2000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusError, RequestVersion: 4, LastError: "merge conflict", Metadata: map[string]string{"step": "land"}}, }, expected: CurrentState{Status: entity.RequestStatusError, LastError: "merge conflict", Metadata: map[string]string{"step": "land"}}, }, { name: "multiple terminal records picks highest version", logs: []entity.RequestLog{ - {RequestID: "q/1", TimestampMs: 1000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusError, RequestVersion: 2, LastError: "timeout", Metadata: map[string]string{}}, - {RequestID: "q/1", TimestampMs: 2000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusLanded, RequestVersion: 5, LastError: "", Metadata: map[string]string{"final": "true"}}, - {RequestID: "q/1", TimestampMs: 3000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusError, RequestVersion: 3, LastError: "conflict", Metadata: map[string]string{}}, + {RequestID: "1", TimestampMs: 1000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusError, RequestVersion: 2, LastError: "timeout", Metadata: map[string]string{}}, + {RequestID: "1", TimestampMs: 2000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusLanded, RequestVersion: 5, LastError: "", Metadata: map[string]string{"final": "true"}}, + {RequestID: "1", TimestampMs: 3000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusError, RequestVersion: 3, LastError: "conflict", Metadata: map[string]string{}}, }, expected: CurrentState{Status: entity.RequestStatusLanded, LastError: "", Metadata: map[string]string{"final": "true"}}, }, { name: "same version terminal records uses timestamp tiebreaker", logs: []entity.RequestLog{ - {RequestID: "q/1", TimestampMs: 1000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusError, RequestVersion: 3, LastError: "first", Metadata: map[string]string{}}, - {RequestID: "q/1", TimestampMs: 2000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusError, RequestVersion: 3, LastError: "second", Metadata: map[string]string{}}, + {RequestID: "1", TimestampMs: 1000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusError, RequestVersion: 3, LastError: "first", Metadata: map[string]string{}}, + {RequestID: "1", TimestampMs: 2000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusError, RequestVersion: 3, LastError: "second", Metadata: map[string]string{}}, }, expected: CurrentState{Status: entity.RequestStatusError, LastError: "second", Metadata: map[string]string{}}, }, { name: "terminal status without version is not terminal", logs: []entity.RequestLog{ - {RequestID: "q/1", TimestampMs: 1000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusLanded, RequestVersion: 0, LastError: "", Metadata: map[string]string{}}, - {RequestID: "q/1", TimestampMs: 2000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusSpeculating, RequestVersion: 0, LastError: "", Metadata: map[string]string{"source": "gw"}}, + {RequestID: "1", TimestampMs: 1000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusLanded, RequestVersion: 0, LastError: "", Metadata: map[string]string{}}, + {RequestID: "1", TimestampMs: 2000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusSpeculating, RequestVersion: 0, LastError: "", Metadata: map[string]string{"source": "gw"}}, }, expected: CurrentState{Status: entity.RequestStatusSpeculating, LastError: "", Metadata: map[string]string{"source": "gw"}}, }, { name: "no terminal records falls back to latest timestamp", logs: []entity.RequestLog{ - {RequestID: "q/1", TimestampMs: 1000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusStarted, RequestVersion: 1, LastError: "", Metadata: map[string]string{}}, - {RequestID: "q/1", TimestampMs: 3000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusValidated, RequestVersion: 2, LastError: "", Metadata: map[string]string{}}, - {RequestID: "q/1", TimestampMs: 2000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusSpeculating, RequestVersion: 0, LastError: "", Metadata: map[string]string{}}, + {RequestID: "1", TimestampMs: 1000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusStarted, RequestVersion: 1, LastError: "", Metadata: map[string]string{}}, + {RequestID: "1", TimestampMs: 3000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusValidated, RequestVersion: 2, LastError: "", Metadata: map[string]string{}}, + {RequestID: "1", TimestampMs: 2000, Type: entity.RequestLogTypeStatus, Status: entity.RequestStatusSpeculating, RequestVersion: 0, LastError: "", Metadata: map[string]string{}}, }, expected: CurrentState{Status: entity.RequestStatusValidated, LastError: "", Metadata: map[string]string{}}, }, @@ -98,9 +98,9 @@ func TestGetCurrentStateFromRequestLog(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctrl := gomock.NewController(t) mockStore := storagemock.NewMockRequestLogStore(ctrl) - mockStore.EXPECT().List(gomock.Any(), "q/1").Return(tt.logs, nil) + mockStore.EXPECT().List(gomock.Any(), "1").Return(tt.logs, nil) - result, err := GetCurrentStateFromRequestLog(context.Background(), mockStore, "q/1") + result, err := GetCurrentStateFromRequestLog(context.Background(), mockStore, "1") require.NoError(t, err) assert.Equal(t, tt.expected, result) }) @@ -110,9 +110,9 @@ func TestGetCurrentStateFromRequestLog(t *testing.T) { func TestGetCurrentStateFromRequestLog_NoRecords(t *testing.T) { ctrl := gomock.NewController(t) mockStore := storagemock.NewMockRequestLogStore(ctrl) - mockStore.EXPECT().List(gomock.Any(), "q/1").Return(nil, storage.ErrNotFound) + mockStore.EXPECT().List(gomock.Any(), "1").Return(nil, storage.ErrNotFound) - _, err := GetCurrentStateFromRequestLog(context.Background(), mockStore, "q/1") + _, err := GetCurrentStateFromRequestLog(context.Background(), mockStore, "1") assert.Error(t, err) assert.True(t, storage.IsNotFound(err)) } @@ -120,8 +120,8 @@ func TestGetCurrentStateFromRequestLog_NoRecords(t *testing.T) { func TestGetCurrentStateFromRequestLog_StoreError(t *testing.T) { ctrl := gomock.NewController(t) mockStore := storagemock.NewMockRequestLogStore(ctrl) - mockStore.EXPECT().List(gomock.Any(), "q/1").Return(nil, fmt.Errorf("db connection lost")) + mockStore.EXPECT().List(gomock.Any(), "1").Return(nil, fmt.Errorf("db connection lost")) - _, err := GetCurrentStateFromRequestLog(context.Background(), mockStore, "q/1") + _, err := GetCurrentStateFromRequestLog(context.Background(), mockStore, "1") assert.Error(t, err) } diff --git a/submitqueue/gateway/extension/storage/mysql/request_log_store_test.go b/submitqueue/gateway/extension/storage/mysql/request_log_store_test.go index 8e4a04a4c..e0a7fbbeb 100644 --- a/submitqueue/gateway/extension/storage/mysql/request_log_store_test.go +++ b/submitqueue/gateway/extension/storage/mysql/request_log_store_test.go @@ -44,7 +44,7 @@ func setupRequestLogStoreTest(t *testing.T) (*sql.DB, sqlmock.Sqlmock, storage.R func TestRequestLogStore_Insert(t *testing.T) { log := entity.RequestLog{ - RequestID: "monorepo/1", + RequestID: "1", Queue: testLogQueue, TimestampMs: 1000, Type: entity.RequestLogTypeStatus, @@ -98,7 +98,7 @@ func TestRequestLogStore_Insert(t *testing.T) { func TestRequestLogStore_List(t *testing.T) { log := entity.RequestLog{ - RequestID: "monorepo/1", + RequestID: "1", Queue: testLogQueue, TimestampMs: 1000, Type: entity.RequestLogTypeStatus, diff --git a/submitqueue/gateway/extension/storage/mysql/request_queue_summary_store_test.go b/submitqueue/gateway/extension/storage/mysql/request_queue_summary_store_test.go index 56e5e522c..804c39e27 100644 --- a/submitqueue/gateway/extension/storage/mysql/request_queue_summary_store_test.go +++ b/submitqueue/gateway/extension/storage/mysql/request_queue_summary_store_test.go @@ -42,7 +42,7 @@ func setupRequestQueueSummaryStoreTest(t *testing.T) (*sql.DB, sqlmock.Sqlmock, func TestRequestQueueSummaryStore_Create(t *testing.T) { summary := entity.RequestQueueSummary{ - RequestID: "monorepo/1", + RequestID: "1", Queue: "monorepo", ChangeURIs: []string{"github://github.example.com/uber/submitqueue/pull/123/deadbeef"}, ReceivedAtMs: 1000, @@ -113,7 +113,7 @@ func TestRequestQueueSummaryStore_Create(t *testing.T) { func TestRequestQueueSummaryStore_Get(t *testing.T) { want := entity.RequestQueueSummary{ - RequestID: "monorepo/1", + RequestID: "1", Queue: "monorepo", ChangeURIs: []string{"github://github.example.com/uber/submitqueue/pull/123/deadbeef"}, ReceivedAtMs: 1000, @@ -183,7 +183,7 @@ func TestRequestQueueSummaryStore_Get(t *testing.T) { func TestRequestQueueSummaryStore_Update(t *testing.T) { summary := entity.RequestQueueSummary{ - RequestID: "monorepo/1", + RequestID: "1", Queue: "monorepo", ChangeURIs: []string{"github://github.example.com/uber/submitqueue/pull/456/cafebabe"}, ReceivedAtMs: 1000, @@ -307,7 +307,7 @@ func TestRequestQueueSummaryStore_Update(t *testing.T) { func TestRequestQueueSummaryStore_List(t *testing.T) { summary := entity.RequestQueueSummary{ - RequestID: "monorepo/1", + RequestID: "1", Queue: "monorepo", ReceivedAtMs: 1000, Status: entity.RequestStatusStarted, @@ -355,13 +355,13 @@ func TestRequestQueueSummaryStore_List(t *testing.T) { HasCursor: true, Cursor: storage.RequestQueueSummaryCursor{ ReceivedAtMs: 1500, - RequestID: "monorepo/2", + RequestID: "2", }, }, setup: func(mock sqlmock.Sqlmock) { rows := sqlmock.NewRows([]string{"queue", "received_at_ms", "request_id", "change_uris", "status", "version", "last_error", "metadata"}) mock.ExpectQuery("SELECT queue, received_at_ms, request_id, change_uris, status"). - WithArgs("monorepo", int64(0), int64(2000), int64(1500), int64(1500), "monorepo/2", 10). + WithArgs("monorepo", int64(0), int64(2000), int64(1500), int64(1500), "2", 10). WillReturnRows(rows) }, want: []entity.RequestQueueSummary{}, diff --git a/submitqueue/gateway/extension/storage/mysql/request_summary_store_test.go b/submitqueue/gateway/extension/storage/mysql/request_summary_store_test.go index 689d6baaf..719791d6f 100644 --- a/submitqueue/gateway/extension/storage/mysql/request_summary_store_test.go +++ b/submitqueue/gateway/extension/storage/mysql/request_summary_store_test.go @@ -45,7 +45,7 @@ func setupRequestSummaryStoreTest(t *testing.T) (*sql.DB, sqlmock.Sqlmock, stora func TestRequestSummaryStore_Create(t *testing.T) { summary := entity.RequestSummary{ - RequestID: "monorepo/1", + RequestID: "1", Queue: testSummaryQueue, ChangeURIs: []string{"github://github.example.com/uber/submitqueue/pull/123/deadbeef"}, ReceivedAtMs: 1000, @@ -118,7 +118,7 @@ func TestRequestSummaryStore_Create(t *testing.T) { func TestRequestSummaryStore_Get(t *testing.T) { want := entity.RequestSummary{ - RequestID: "monorepo/1", + RequestID: "1", Queue: testSummaryQueue, ChangeURIs: []string{"github://github.example.com/uber/submitqueue/pull/123/deadbeef"}, ReceivedAtMs: 1000, @@ -201,7 +201,7 @@ func TestRequestSummaryStore_Get(t *testing.T) { func TestRequestSummaryStore_Update(t *testing.T) { summary := entity.RequestSummary{ - RequestID: "monorepo/1", + RequestID: "1", Queue: testSummaryQueue, ChangeURIs: []string{"github://github.example.com/uber/submitqueue/pull/456/cafebabe"}, ReceivedAtMs: 1500, diff --git a/submitqueue/gateway/extension/storage/mysql/request_uri_store_test.go b/submitqueue/gateway/extension/storage/mysql/request_uri_store_test.go index 5a4ad0c37..e6b50f830 100644 --- a/submitqueue/gateway/extension/storage/mysql/request_uri_store_test.go +++ b/submitqueue/gateway/extension/storage/mysql/request_uri_store_test.go @@ -48,7 +48,7 @@ func TestRequestURIStore_Create(t *testing.T) { ChangeURI: "github://github.example.com/uber/submitqueue/pull/123/deadbeef", Queue: testURIQueue, ReceivedAtMs: 1000, - RequestID: "monorepo/1", + RequestID: "1", } tests := []struct { @@ -112,7 +112,7 @@ func TestRequestURIStore_ListByURI(t *testing.T) { ChangeURI: "github://github.example.com/uber/submitqueue/pull/123/deadbeef", Queue: testURIQueue, ReceivedAtMs: 1000, - RequestID: "monorepo/1", + RequestID: "1", } tests := []struct { diff --git a/submitqueue/orchestrator/controller/batch/BUILD.bazel b/submitqueue/orchestrator/controller/batch/BUILD.bazel index a3decc51f..c68fca45b 100644 --- a/submitqueue/orchestrator/controller/batch/BUILD.bazel +++ b/submitqueue/orchestrator/controller/batch/BUILD.bazel @@ -6,6 +6,7 @@ go_library( importpath = "github.com/uber/submitqueue/submitqueue/orchestrator/controller/batch", visibility = ["//visibility:public"], deps = [ + "//platform/base/id:go_default_library", "//platform/base/messagequeue:go_default_library", "//platform/consumer:go_default_library", "//platform/extension/counter:go_default_library", diff --git a/submitqueue/orchestrator/controller/batch/batch.go b/submitqueue/orchestrator/controller/batch/batch.go index 9e1a6b31b..90846725c 100644 --- a/submitqueue/orchestrator/controller/batch/batch.go +++ b/submitqueue/orchestrator/controller/batch/batch.go @@ -21,6 +21,7 @@ import ( orchstorage "github.com/uber/submitqueue/submitqueue/orchestrator/extension/storage" "github.com/uber-go/tally" + "github.com/uber/submitqueue/platform/base/id" entityqueue "github.com/uber/submitqueue/platform/base/messagequeue" "github.com/uber/submitqueue/platform/consumer" "github.com/uber/submitqueue/platform/extension/counter" @@ -51,9 +52,7 @@ var _ consumer.Controller = (*Controller)(nil) const opName = "process" -// counterDomainBatch names the per-queue sequence that mints batch IDs. The batch -// ID is built independently as "/batch/", so the domain is a -// sequence name only and never appears in the ID. +// counterDomainBatch names the per-queue sequence that mints batch IDs. const counterDomainBatch = "batch" // NewController creates a new batch controller for the orchestrator. @@ -138,7 +137,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er // TODO: if capacity is full, wait here for other requests to accumulate to batch them together, or include a request into an existing batch if it's not too late. - // Generate a globally unique batch ID. + // Generate a queue-scoped batch ID. queueCounter, err := c.counters.For(counter.Config{QueueName: request.Queue}) if err != nil { metrics.NamedCounter(c.metricsScope, opName, "counter_errors", 1) @@ -149,12 +148,17 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) er metrics.NamedCounter(c.metricsScope, opName, "counter_errors", 1) return fmt.Errorf("failed to generate batch ID for queue=%s: %w", request.Queue, err) } + batchID, err := id.FromCounter(seq) + if err != nil { + metrics.NamedCounter(c.metricsScope, opName, "counter_errors", 1) + return fmt.Errorf("generated invalid batch ID for queue=%s: %w", request.Queue, err) + } // Dependencies stay empty here: what the batch must serialize behind is // resolved by the dependency-analysis stage, which fills them in as it // promotes the batch out of Creating. batch := entity.Batch{ - ID: fmt.Sprintf("%s/batch/%d", request.Queue, seq), + ID: batchID, Queue: request.Queue, Contains: []string{request.ID}, Dependencies: []string{}, diff --git a/submitqueue/orchestrator/controller/batch/batch_test.go b/submitqueue/orchestrator/controller/batch/batch_test.go index f52b101f5..8257c9ea2 100644 --- a/submitqueue/orchestrator/controller/batch/batch_test.go +++ b/submitqueue/orchestrator/controller/batch/batch_test.go @@ -78,7 +78,7 @@ func (f staticCounterFactory) For(counter.Config) (counter.Counter, error) { ret // testRequest returns a standard test request for batch tests. func testRequest() entity.Request { return entity.Request{ - ID: "test-queue/123", + ID: "123", Queue: "test-queue", Change: change.Change{URIs: []string{"github://github.example.com/uber/service/pull/456/abcdef0123456789abcdef0123456789abcdef01"}}, LandStrategy: mergestrategy.MergeStrategyRebase, @@ -285,6 +285,16 @@ func TestController_Process_CounterFailure(t *testing.T) { assert.Error(t, controller.Process(context.Background(), newDelivery(t, ctrl, testRequest(), ""))) } +func TestController_Process_NonPositiveCounterValue(t *testing.T) { + ctrl := gomock.NewController(t) + + cnt := countermock.NewMockCounter(ctrl) + cnt.EXPECT().Next(gomock.Any(), gomock.Any()).Return(int64(0), nil) + + controller := newTestController(t, ctrl, cnt, nil, nil) + assert.Error(t, controller.Process(context.Background(), newDelivery(t, ctrl, testRequest(), ""))) +} + // A halted request must never spawn a batch. Cancelling is non-terminal but // still halts: cancel owns the request's outcome from that point. func TestController_Process_HaltedShortCircuit(t *testing.T) { @@ -324,7 +334,7 @@ func TestController_Process_WritesBatchBeforeHandoff(t *testing.T) { request := testRequest() batch := entity.Batch{ - ID: "test-queue/batch/7", + ID: "7", Queue: request.Queue, Contains: []string{request.ID}, Dependencies: []string{}, @@ -433,5 +443,5 @@ func TestController_Process_PublishesBatchingStatus(t *testing.T) { require.Len(t, logs, 1) assert.Equal(t, request.ID, logs[0].RequestID) assert.Equal(t, entity.RequestStatusBatching, logs[0].Status) - assert.Equal(t, "test-queue/batch/1", logs[0].Metadata["batch_id"]) + assert.Equal(t, "1", logs[0].Metadata["batch_id"]) } diff --git a/submitqueue/orchestrator/controller/buildsignal/buildsignal_test.go b/submitqueue/orchestrator/controller/buildsignal/buildsignal_test.go index c2c812193..74d7fd7f7 100644 --- a/submitqueue/orchestrator/controller/buildsignal/buildsignal_test.go +++ b/submitqueue/orchestrator/controller/buildsignal/buildsignal_test.go @@ -75,7 +75,7 @@ func (f staticStorageFactory) For(orchstorage.Config) (orchstorage.Storage, erro // testRequestID is the one member of the batch under test, so the request-log // fan-out has somebody to report to. -const testRequestID = "test-queue/1" +const testRequestID = "1" func newTestHarness(t *testing.T, ctrl *gomock.Controller, batchState entity.BatchState) *testHarness { h := &testHarness{} diff --git a/submitqueue/orchestrator/controller/cancel/cancel_test.go b/submitqueue/orchestrator/controller/cancel/cancel_test.go index a7c3f7feb..0b944db64 100644 --- a/submitqueue/orchestrator/controller/cancel/cancel_test.go +++ b/submitqueue/orchestrator/controller/cancel/cancel_test.go @@ -145,7 +145,7 @@ func TestNewController(t *testing.T) { func TestProcess_RejectsTenantPayloadQueueMismatch(t *testing.T) { ctrl := gomock.NewController(t) controller := newController(t, orchstoragemock.NewMockStorage(ctrl), consumer.TopicRegistry{}) - msg := entityqueue.NewMessage("cancel-msg", cancelPayload(t, "q/1", ""), "q", nil) + msg := entityqueue.NewMessage("cancel-msg", cancelPayload(t, "1", ""), "q", nil) msg.Tenant = "other-queue" d := consumermock.NewMockDelivery(ctrl) d.EXPECT().Message().Return(msg).AnyTimes() @@ -160,8 +160,8 @@ func TestProcess_AlreadyTerminal_NoOp(t *testing.T) { _ = pub reqStore := storagemock.NewMockRequestStore(ctrl) - reqStore.EXPECT().Get(gomock.Any(), "q/1").Return(entity.Request{ - ID: "q/1", Queue: "q", State: entity.RequestStateCancelled, Version: 5, + reqStore.EXPECT().Get(gomock.Any(), "1").Return(entity.Request{ + ID: "1", Queue: "q", State: entity.RequestStateCancelled, Version: 5, }, nil) store := orchstoragemock.NewMockStorage(ctrl) @@ -169,7 +169,7 @@ func TestProcess_AlreadyTerminal_NoOp(t *testing.T) { store.EXPECT().GetRequestStore().Return(reqStore).AnyTimes() controller := newController(t, store, registry) - err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "q/1", ""), "q/1")) + err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "1", ""), "1")) require.NoError(t, err) } @@ -178,14 +178,14 @@ func TestProcess_RequestNotFound_Retryable(t *testing.T) { registry, _ := newRegistry(t, ctrl) reqStore := storagemock.NewMockRequestStore(ctrl) - reqStore.EXPECT().Get(gomock.Any(), "q/1").Return(entity.Request{}, storage.ErrNotFound) + reqStore.EXPECT().Get(gomock.Any(), "1").Return(entity.Request{}, storage.ErrNotFound) store := orchstoragemock.NewMockStorage(ctrl) store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() store.EXPECT().GetRequestStore().Return(reqStore).AnyTimes() controller := newController(t, store, registry) - err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "q/1", ""), "q/1")) + err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "1", ""), "1")) require.Error(t, err) assert.ErrorIs(t, err, storage.ErrNotFound) } @@ -202,14 +202,14 @@ func TestProcess_CancelsUnbatchedRequest(t *testing.T) { }).AnyTimes() reqStore := storagemock.NewMockRequestStore(ctrl) - started := entity.Request{ID: "q/1", Queue: "q", State: entity.RequestStateStarted, Version: 2} - cancelling := entity.Request{ID: "q/1", Queue: "q", State: entity.RequestStateCancelling, Version: 3} + started := entity.Request{ID: "1", Queue: "q", State: entity.RequestStateStarted, Version: 2} + cancelling := entity.Request{ID: "1", Queue: "q", State: entity.RequestStateCancelling, Version: 3} // Two-step transition: mark Cancelling, then — after TerminateRequest re-reads // the request — the terminal Cancelled CAS. gomock.InOrder( - reqStore.EXPECT().Get(gomock.Any(), "q/1").Return(started, nil), + reqStore.EXPECT().Get(gomock.Any(), "1").Return(started, nil), reqStore.EXPECT().Update(gomock.Any(), requestWithState(started, entity.RequestStateCancelling), int32(2), int32(3)).Return(nil), - reqStore.EXPECT().Get(gomock.Any(), "q/1").Return(cancelling, nil), + reqStore.EXPECT().Get(gomock.Any(), "1").Return(cancelling, nil), reqStore.EXPECT().Update(gomock.Any(), requestWithState(cancelling, entity.RequestStateCancelled), int32(3), int32(4)).Return(nil), ) @@ -219,10 +219,10 @@ func TestProcess_CancelsUnbatchedRequest(t *testing.T) { store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() store.EXPECT().GetRequestStore().Return(reqStore).AnyTimes() store.EXPECT().GetBatchStore().Return(batchStore).AnyTimes() - expectBatchLookup(ctrl, store, batchStore, "q/1") + expectBatchLookup(ctrl, store, batchStore, "1") controller := newController(t, store, registry) - err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "q/1", "user changed mind"), "q/1")) + err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "1", "user changed mind"), "1")) require.NoError(t, err) // Only the request log entry should have been published. @@ -239,10 +239,10 @@ func TestProcess_AlreadyCancelling_SkipsMarkCancelling(t *testing.T) { pub.EXPECT().Publish(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() reqStore := storagemock.NewMockRequestStore(ctrl) - cancelling := entity.Request{ID: "q/1", Queue: "q", State: entity.RequestStateCancelling, Version: 3} + cancelling := entity.Request{ID: "1", Queue: "q", State: entity.RequestStateCancelling, Version: 3} // Get is called twice: the initial load and TerminateRequest's re-read. Both // see Cancelling (the prior pass already recorded intent). - reqStore.EXPECT().Get(gomock.Any(), "q/1").Return(cancelling, nil).Times(2) + reqStore.EXPECT().Get(gomock.Any(), "1").Return(cancelling, nil).Times(2) // Only the terminal CAS — the mark-cancelling step is a no-op. reqStore.EXPECT().Update(gomock.Any(), requestWithState(cancelling, entity.RequestStateCancelled), int32(3), int32(4)).Return(nil) @@ -252,10 +252,10 @@ func TestProcess_AlreadyCancelling_SkipsMarkCancelling(t *testing.T) { store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() store.EXPECT().GetRequestStore().Return(reqStore).AnyTimes() store.EXPECT().GetBatchStore().Return(batchStore).AnyTimes() - expectBatchLookup(ctrl, store, batchStore, "q/1") + expectBatchLookup(ctrl, store, batchStore, "1") controller := newController(t, store, registry) - err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "q/1", ""), "q/1")) + err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "1", ""), "1")) require.NoError(t, err) } @@ -271,9 +271,9 @@ func TestProcess_MarkCancellingVersionMismatch_Retryable(t *testing.T) { reqStore := storagemock.NewMockRequestStore(ctrl) started := entity.Request{ - ID: "q/1", Queue: "q", State: entity.RequestStateStarted, Version: 2, + ID: "1", Queue: "q", State: entity.RequestStateStarted, Version: 2, } - reqStore.EXPECT().Get(gomock.Any(), "q/1").Return(started, nil) + reqStore.EXPECT().Get(gomock.Any(), "1").Return(started, nil) reqStore.EXPECT().Update(gomock.Any(), requestWithState(started, entity.RequestStateCancelling), int32(2), int32(3)). Return(storage.ErrVersionMismatch) @@ -282,7 +282,7 @@ func TestProcess_MarkCancellingVersionMismatch_Retryable(t *testing.T) { store.EXPECT().GetRequestStore().Return(reqStore).AnyTimes() controller := newController(t, store, registry) - err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "q/1", ""), "q/1")) + err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "1", ""), "1")) require.Error(t, err) assert.ErrorIs(t, err, storage.ErrVersionMismatch) assert.Equal(t, entity.RequestStateStarted, started.State) @@ -295,12 +295,12 @@ func TestProcess_UnbatchedVersionMismatch_Retryable(t *testing.T) { _ = pub reqStore := storagemock.NewMockRequestStore(ctrl) - started := entity.Request{ID: "q/1", Queue: "q", State: entity.RequestStateStarted, Version: 2} - cancelling := entity.Request{ID: "q/1", Queue: "q", State: entity.RequestStateCancelling, Version: 3} + started := entity.Request{ID: "1", Queue: "q", State: entity.RequestStateStarted, Version: 2} + cancelling := entity.Request{ID: "1", Queue: "q", State: entity.RequestStateCancelling, Version: 3} gomock.InOrder( - reqStore.EXPECT().Get(gomock.Any(), "q/1").Return(started, nil), + reqStore.EXPECT().Get(gomock.Any(), "1").Return(started, nil), reqStore.EXPECT().Update(gomock.Any(), requestWithState(started, entity.RequestStateCancelling), int32(2), int32(3)).Return(nil), - reqStore.EXPECT().Get(gomock.Any(), "q/1").Return(cancelling, nil), + reqStore.EXPECT().Get(gomock.Any(), "1").Return(cancelling, nil), reqStore.EXPECT().Update(gomock.Any(), requestWithState(cancelling, entity.RequestStateCancelled), int32(3), int32(4)). Return(storage.ErrVersionMismatch), ) @@ -311,10 +311,10 @@ func TestProcess_UnbatchedVersionMismatch_Retryable(t *testing.T) { store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() store.EXPECT().GetRequestStore().Return(reqStore).AnyTimes() store.EXPECT().GetBatchStore().Return(batchStore).AnyTimes() - expectBatchLookup(ctrl, store, batchStore, "q/1") + expectBatchLookup(ctrl, store, batchStore, "1") controller := newController(t, store, registry) - err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "q/1", ""), "q/1")) + err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "1", ""), "1")) require.Error(t, err) assert.ErrorIs(t, err, storage.ErrVersionMismatch) } @@ -324,13 +324,13 @@ func TestProcess_UnbatchedRequestDiverged_Acks(t *testing.T) { registry, pub := newRegistry(t, ctrl) _ = pub - started := entity.Request{ID: "q/1", Queue: "q", State: entity.RequestStateStarted, Version: 2} - landed := entity.Request{ID: "q/1", Queue: "q", State: entity.RequestStateLanded, Version: 4} + started := entity.Request{ID: "1", Queue: "q", State: entity.RequestStateStarted, Version: 2} + landed := entity.Request{ID: "1", Queue: "q", State: entity.RequestStateLanded, Version: 4} reqStore := storagemock.NewMockRequestStore(ctrl) gomock.InOrder( - reqStore.EXPECT().Get(gomock.Any(), "q/1").Return(started, nil), + reqStore.EXPECT().Get(gomock.Any(), "1").Return(started, nil), reqStore.EXPECT().Update(gomock.Any(), requestWithState(started, entity.RequestStateCancelling), int32(2), int32(3)).Return(nil), - reqStore.EXPECT().Get(gomock.Any(), "q/1").Return(landed, nil), + reqStore.EXPECT().Get(gomock.Any(), "1").Return(landed, nil), ) batchStore := storagemock.NewMockBatchStore(ctrl) @@ -339,10 +339,10 @@ func TestProcess_UnbatchedRequestDiverged_Acks(t *testing.T) { store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() store.EXPECT().GetRequestStore().Return(reqStore).AnyTimes() store.EXPECT().GetBatchStore().Return(batchStore).AnyTimes() - expectBatchLookup(ctrl, store, batchStore, "q/1") + expectBatchLookup(ctrl, store, batchStore, "1") controller := newController(t, store, registry) - err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "q/1", ""), "q/1")) + err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "1", ""), "1")) require.NoError(t, err) } @@ -351,12 +351,12 @@ func TestProcess_UnbatchedRequestDisappears_Retryable(t *testing.T) { registry, pub := newRegistry(t, ctrl) _ = pub - started := entity.Request{ID: "q/1", Queue: "q", State: entity.RequestStateStarted, Version: 2} + started := entity.Request{ID: "1", Queue: "q", State: entity.RequestStateStarted, Version: 2} reqStore := storagemock.NewMockRequestStore(ctrl) gomock.InOrder( - reqStore.EXPECT().Get(gomock.Any(), "q/1").Return(started, nil), + reqStore.EXPECT().Get(gomock.Any(), "1").Return(started, nil), reqStore.EXPECT().Update(gomock.Any(), requestWithState(started, entity.RequestStateCancelling), int32(2), int32(3)).Return(nil), - reqStore.EXPECT().Get(gomock.Any(), "q/1").Return(entity.Request{}, storage.ErrNotFound), + reqStore.EXPECT().Get(gomock.Any(), "1").Return(entity.Request{}, storage.ErrNotFound), ) batchStore := storagemock.NewMockBatchStore(ctrl) @@ -365,10 +365,10 @@ func TestProcess_UnbatchedRequestDisappears_Retryable(t *testing.T) { store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() store.EXPECT().GetRequestStore().Return(reqStore).AnyTimes() store.EXPECT().GetBatchStore().Return(batchStore).AnyTimes() - expectBatchLookup(ctrl, store, batchStore, "q/1") + expectBatchLookup(ctrl, store, batchStore, "1") controller := newController(t, store, registry) - err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "q/1", ""), "q/1")) + err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "1", ""), "1")) require.Error(t, err) assert.ErrorIs(t, err, storage.ErrNotFound) } @@ -403,17 +403,17 @@ func TestProcess_BatchPath_HandsOffToSpeculate(t *testing.T) { return nil }).AnyTimes() - req := entity.Request{ID: "q/1", Queue: "q", State: entity.RequestStateStarted, Version: 2} + req := entity.Request{ID: "1", Queue: "q", State: entity.RequestStateStarted, Version: 2} batch := entity.Batch{ - ID: "q/batch/1", + ID: "1", Queue: "q", - Contains: []string{"q/1", "q/2"}, + Contains: []string{"1", "2"}, State: entity.BatchStateSpeculating, Version: 3, } reqStore := storagemock.NewMockRequestStore(ctrl) - reqStore.EXPECT().Get(gomock.Any(), "q/1").Return(req, nil) + reqStore.EXPECT().Get(gomock.Any(), "1").Return(req, nil) reqStore.EXPECT().Update(gomock.Any(), requestWithState(req, entity.RequestStateCancelling), int32(2), int32(3)).Return(nil) batchStore := storagemock.NewMockBatchStore(ctrl) @@ -428,7 +428,7 @@ func TestProcess_BatchPath_HandsOffToSpeculate(t *testing.T) { // BatchDependentStore and BuildStore must NOT be touched — speculate owns those now. controller := newController(t, store, registry) - err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "q/1", "stop"), "q/1")) + err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "1", "stop"), "1")) require.NoError(t, err) require.Len(t, records, 1) @@ -443,10 +443,10 @@ func TestProcess_CancelsEveryApplicableBatch(t *testing.T) { ctrl := gomock.NewController(t) registry, publisher := newRegistry(t, ctrl) - request := entity.Request{ID: "q/1", Queue: "q", State: entity.RequestStateBatched, Version: 2} - batch1 := entity.Batch{ID: "q/batch/1", Queue: "q", Contains: []string{request.ID}, State: entity.BatchStateCreated, Version: 1} - batch2 := entity.Batch{ID: "q/batch/2", Queue: "q", Contains: []string{request.ID}, State: entity.BatchStateSpeculating, Version: 3} - terminalBatch := entity.Batch{ID: "q/batch/3", Queue: "q", Contains: []string{request.ID}, State: entity.BatchStateSucceeded, Version: 4} + request := entity.Request{ID: "1", Queue: "q", State: entity.RequestStateBatched, Version: 2} + batch1 := entity.Batch{ID: "1", Queue: "q", Contains: []string{request.ID}, State: entity.BatchStateCreated, Version: 1} + batch2 := entity.Batch{ID: "2", Queue: "q", Contains: []string{request.ID}, State: entity.BatchStateSpeculating, Version: 3} + terminalBatch := entity.Batch{ID: "3", Queue: "q", Contains: []string{request.ID}, State: entity.BatchStateSucceeded, Version: 4} requestStore := storagemock.NewMockRequestStore(ctrl) requestStore.EXPECT().Get(gomock.Any(), request.ID).Return(request, nil) @@ -486,10 +486,10 @@ func TestProcess_CancelsEveryApplicableBatch(t *testing.T) { controller := newController(t, store, registry) assert.NoError(t, controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, request.ID, ""), request.ID))) assert.Equal(t, []string{ - "update:q/batch/1", - "publish:q/batch/1", - "update:q/batch/2", - "publish:q/batch/2", + "update:1", + "publish:1", + "update:2", + "publish:2", }, operations) } @@ -497,9 +497,9 @@ func TestProcess_BatchFailureDoesNotPreventLaterCancellation(t *testing.T) { ctrl := gomock.NewController(t) registry, publisher := newRegistry(t, ctrl) - request := entity.Request{ID: "q/1", Queue: "q", State: entity.RequestStateBatched, Version: 2} - batch1 := entity.Batch{ID: "q/batch/1", Queue: "q", Contains: []string{request.ID}, State: entity.BatchStateCreated, Version: 1} - batch2 := entity.Batch{ID: "q/batch/2", Queue: "q", Contains: []string{request.ID}, State: entity.BatchStateCreated, Version: 2} + request := entity.Request{ID: "1", Queue: "q", State: entity.RequestStateBatched, Version: 2} + batch1 := entity.Batch{ID: "1", Queue: "q", Contains: []string{request.ID}, State: entity.BatchStateCreated, Version: 1} + batch2 := entity.Batch{ID: "2", Queue: "q", Contains: []string{request.ID}, State: entity.BatchStateCreated, Version: 2} storeErr := fmt.Errorf("storage failed") requestStore := storagemock.NewMockRequestStore(ctrl) @@ -526,7 +526,7 @@ func TestProcess_BatchFailureDoesNotPreventLaterCancellation(t *testing.T) { controller := newController(t, store, registry) err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, request.ID, ""), request.ID)) - assert.ErrorContains(t, err, "failed to mark batch q/batch/1 as cancelling") + assert.ErrorContains(t, err, "failed to mark batch 1 as cancelling") } func TestProcess_NonCancellableBatchSuppressesRequestCancellation(t *testing.T) { @@ -545,8 +545,8 @@ func TestProcess_NonCancellableBatchSuppressesRequestCancellation(t *testing.T) ctrl := gomock.NewController(t) registry, _ := newRegistry(t, ctrl) - request := entity.Request{ID: "q/1", Queue: "q", State: entity.RequestStateBatched, Version: 2} - batch := entity.Batch{ID: "q/batch/1", Queue: "q", Contains: []string{request.ID}, State: tt.state, Version: 4} + request := entity.Request{ID: "1", Queue: "q", State: entity.RequestStateBatched, Version: 2} + batch := entity.Batch{ID: "1", Queue: "q", Contains: []string{request.ID}, State: tt.state, Version: 4} requestStore := storagemock.NewMockRequestStore(ctrl) requestStore.EXPECT().Get(gomock.Any(), request.ID).Return(request, nil) @@ -571,7 +571,7 @@ func TestProcess_BatchedWithoutMatchCancelsRequest(t *testing.T) { registry, publisher := newRegistry(t, ctrl) publisher.EXPECT().Publish(gomock.Any(), "log", gomock.Any()).Return(nil) - request := entity.Request{ID: "q/1", Queue: "q", State: entity.RequestStateBatched, Version: 2} + request := entity.Request{ID: "1", Queue: "q", State: entity.RequestStateBatched, Version: 2} cancelling := requestWithState(request, entity.RequestStateCancelling) cancelling.Version = 3 requestStore := storagemock.NewMockRequestStore(ctrl) @@ -599,11 +599,11 @@ func TestProcess_CreatingBatchDoesNotSuppressRequestCancellation(t *testing.T) { registry, publisher := newRegistry(t, ctrl) publisher.EXPECT().Publish(gomock.Any(), "log", gomock.Any()).Return(nil) - request := entity.Request{ID: "q/1", Queue: "q", State: entity.RequestStateBatched, Version: 2} + request := entity.Request{ID: "1", Queue: "q", State: entity.RequestStateBatched, Version: 2} cancelling := requestWithState(request, entity.RequestStateCancelling) cancelling.Version = 3 batch := entity.Batch{ - ID: "q/batch/1", + ID: "1", Queue: request.Queue, Contains: []string{request.ID}, State: entity.BatchStateCreating, @@ -648,17 +648,17 @@ func TestProcess_BatchAlreadyCancelling_RepublishesToSpeculate(t *testing.T) { // Request is already in RequestStateCancelling from the prior pass; the // batch is in BatchStateCancelling from the same prior pass. - req := entity.Request{ID: "q/1", Queue: "q", State: entity.RequestStateCancelling, Version: 3} + req := entity.Request{ID: "1", Queue: "q", State: entity.RequestStateCancelling, Version: 3} batch := entity.Batch{ - ID: "q/batch/1", + ID: "1", Queue: "q", - Contains: []string{"q/1"}, + Contains: []string{"1"}, State: entity.BatchStateCancelling, Version: 4, } reqStore := storagemock.NewMockRequestStore(ctrl) - reqStore.EXPECT().Get(gomock.Any(), "q/1").Return(req, nil) + reqStore.EXPECT().Get(gomock.Any(), "1").Return(req, nil) // No request UpdateState — already in Cancelling. batchStore := storagemock.NewMockBatchStore(ctrl) @@ -671,7 +671,7 @@ func TestProcess_BatchAlreadyCancelling_RepublishesToSpeculate(t *testing.T) { expectBatchLookup(ctrl, store, batchStore, req.ID, batch) controller := newController(t, store, registry) - err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "q/1", ""), "q/1")) + err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "1", ""), "1")) require.NoError(t, err) assert.Equal(t, []string{"speculate"}, publishedTopics) @@ -685,11 +685,11 @@ func TestProcess_BatchIntentVersionMismatch_Retryable(t *testing.T) { ctrl := gomock.NewController(t) registry, _ := newRegistry(t, ctrl) - req := entity.Request{ID: "q/1", Queue: "q", State: entity.RequestStateStarted, Version: 2} - batch := entity.Batch{ID: "q/batch/1", Queue: "q", Contains: []string{"q/1"}, State: entity.BatchStateCreated, Version: 1} + req := entity.Request{ID: "1", Queue: "q", State: entity.RequestStateStarted, Version: 2} + batch := entity.Batch{ID: "1", Queue: "q", Contains: []string{"1"}, State: entity.BatchStateCreated, Version: 1} reqStore := storagemock.NewMockRequestStore(ctrl) - reqStore.EXPECT().Get(gomock.Any(), "q/1").Return(req, nil) + reqStore.EXPECT().Get(gomock.Any(), "1").Return(req, nil) reqStore.EXPECT().Update(gomock.Any(), requestWithState(req, entity.RequestStateCancelling), int32(2), int32(3)).Return(nil) batchStore := storagemock.NewMockBatchStore(ctrl) @@ -703,7 +703,7 @@ func TestProcess_BatchIntentVersionMismatch_Retryable(t *testing.T) { expectBatchLookup(ctrl, store, batchStore, req.ID, batch) controller := newController(t, store, registry) - err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "q/1", ""), "q/1")) + err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "1", ""), "1")) require.Error(t, err) assert.ErrorIs(t, err, storage.ErrVersionMismatch) } @@ -715,7 +715,7 @@ func TestProcess_DeserializeError(t *testing.T) { store := orchstoragemock.NewMockStorage(ctrl) store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() controller := newController(t, store, registry) - err := controller.Process(context.Background(), newDelivery(t, ctrl, []byte("not json"), "q/1")) + err := controller.Process(context.Background(), newDelivery(t, ctrl, []byte("not json"), "1")) require.Error(t, err) } @@ -724,21 +724,21 @@ func TestProcess_RequestStoreError(t *testing.T) { registry, _ := newRegistry(t, ctrl) reqStore := storagemock.NewMockRequestStore(ctrl) - reqStore.EXPECT().Get(gomock.Any(), "q/1").Return(entity.Request{}, fmt.Errorf("db down")) + reqStore.EXPECT().Get(gomock.Any(), "1").Return(entity.Request{}, fmt.Errorf("db down")) store := orchstoragemock.NewMockStorage(ctrl) store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() store.EXPECT().GetRequestStore().Return(reqStore).AnyTimes() controller := newController(t, store, registry) - err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "q/1", ""), "q/1")) + err := controller.Process(context.Background(), newDelivery(t, ctrl, cancelPayload(t, "1", ""), "1")) require.Error(t, err) } func TestFindBatches(t *testing.T) { - request := entity.Request{ID: "q/1", Queue: "q"} - batch1 := entity.Batch{ID: "q/batch/1", Contains: []string{request.ID}} - batch2 := entity.Batch{ID: "q/batch/2", Contains: []string{"q/other", request.ID}} + request := entity.Request{ID: "1", Queue: "q"} + batch1 := entity.Batch{ID: "1", Contains: []string{request.ID}} + batch2 := entity.Batch{ID: "2", Contains: []string{"q/other", request.ID}} storeErr := fmt.Errorf("storage failed") tests := []struct { @@ -759,10 +759,10 @@ func TestFindBatches(t *testing.T) { mockFunc: func(store *storagemock.MockRequestBatchStore, batchStore *storagemock.MockBatchStore) { store.EXPECT().GetByRequestID(gomock.Any(), request.ID).Return([]entity.RequestBatch{{ RequestID: request.ID, - BatchID: "q/batch/missing", + BatchID: "999", Version: 1, }}, nil) - batchStore.EXPECT().Get(gomock.Any(), "q/batch/missing").Return(entity.Batch{}, storage.ErrNotFound) + batchStore.EXPECT().Get(gomock.Any(), "999").Return(entity.Batch{}, storage.ErrNotFound) }, }, { diff --git a/submitqueue/orchestrator/controller/conclude/conclude_test.go b/submitqueue/orchestrator/controller/conclude/conclude_test.go index 3296e2dcb..21c5f1c18 100644 --- a/submitqueue/orchestrator/controller/conclude/conclude_test.go +++ b/submitqueue/orchestrator/controller/conclude/conclude_test.go @@ -104,7 +104,7 @@ func TestNewController(t *testing.T) { func TestController_Process_RejectsTenantPayloadQueueMismatch(t *testing.T) { ctrl := gomock.NewController(t) controller, _ := newTestController(t, ctrl, nil, false) - msg := entityqueue.NewMessage("test-queue/batch/1", batchIDPayload(t, "test-queue/batch/1"), "test-queue", nil) + msg := entityqueue.NewMessage("1", batchIDPayload(t, "1"), "test-queue", nil) msg.Tenant = "other-queue" delivery := consumermock.NewMockDelivery(ctrl) delivery.EXPECT().Message().Return(msg).AnyTimes() @@ -124,32 +124,32 @@ func TestController_Process(t *testing.T) { { name: "succeeded batch lands requests", batch: entity.Batch{ - ID: "test-queue/batch/1", + ID: "1", Queue: "test-queue", - Contains: []string{"test-queue/1", "test-queue/2"}, + Contains: []string{"1", "2"}, State: entity.BatchStateSucceeded, Version: 3, }, setupStore: func(ctrl *gomock.Controller) *orchstoragemock.MockStorage { mockBatchStore := storagemock.NewMockBatchStore(ctrl) - mockBatchStore.EXPECT().Get(gomock.Any(), "test-queue/batch/1").Return(entity.Batch{ - ID: "test-queue/batch/1", + mockBatchStore.EXPECT().Get(gomock.Any(), "1").Return(entity.Batch{ + ID: "1", Queue: "test-queue", - Contains: []string{"test-queue/1", "test-queue/2"}, + Contains: []string{"1", "2"}, State: entity.BatchStateSucceeded, Version: 3, }, nil) mockRequestStore := storagemock.NewMockRequestStore(ctrl) request1 := entity.Request{ - ID: "test-queue/1", Queue: "test-queue", Version: 2, State: entity.RequestStateProcessing, + ID: "1", Queue: "test-queue", Version: 2, State: entity.RequestStateProcessing, } - mockRequestStore.EXPECT().Get(gomock.Any(), "test-queue/1").Return(request1, nil) + mockRequestStore.EXPECT().Get(gomock.Any(), "1").Return(request1, nil) mockRequestStore.EXPECT().Update(gomock.Any(), requestWithState(request1, entity.RequestStateLanded), int32(2), int32(3)).Return(nil) request2 := entity.Request{ - ID: "test-queue/2", Queue: "test-queue", Version: 3, State: entity.RequestStateProcessing, + ID: "2", Queue: "test-queue", Version: 3, State: entity.RequestStateProcessing, } - mockRequestStore.EXPECT().Get(gomock.Any(), "test-queue/2").Return(request2, nil) + mockRequestStore.EXPECT().Get(gomock.Any(), "2").Return(request2, nil) mockRequestStore.EXPECT().Update(gomock.Any(), requestWithState(request2, entity.RequestStateLanded), int32(3), int32(4)).Return(nil) mockStorage := orchstoragemock.NewMockStorage(ctrl) @@ -162,27 +162,27 @@ func TestController_Process(t *testing.T) { { name: "failed batch errors requests", batch: entity.Batch{ - ID: "test-queue/batch/2", + ID: "2", Queue: "test-queue", - Contains: []string{"test-queue/5"}, + Contains: []string{"5"}, State: entity.BatchStateFailed, Version: 2, }, setupStore: func(ctrl *gomock.Controller) *orchstoragemock.MockStorage { mockBatchStore := storagemock.NewMockBatchStore(ctrl) - mockBatchStore.EXPECT().Get(gomock.Any(), "test-queue/batch/2").Return(entity.Batch{ - ID: "test-queue/batch/2", + mockBatchStore.EXPECT().Get(gomock.Any(), "2").Return(entity.Batch{ + ID: "2", Queue: "test-queue", - Contains: []string{"test-queue/5"}, + Contains: []string{"5"}, State: entity.BatchStateFailed, Version: 2, }, nil) mockRequestStore := storagemock.NewMockRequestStore(ctrl) request := entity.Request{ - ID: "test-queue/5", Queue: "test-queue", Version: 1, State: entity.RequestStateProcessing, + ID: "5", Queue: "test-queue", Version: 1, State: entity.RequestStateProcessing, } - mockRequestStore.EXPECT().Get(gomock.Any(), "test-queue/5").Return(request, nil) + mockRequestStore.EXPECT().Get(gomock.Any(), "5").Return(request, nil) mockRequestStore.EXPECT().Update(gomock.Any(), requestWithState(request, entity.RequestStateError), int32(1), int32(2)).Return(nil) mockStorage := orchstoragemock.NewMockStorage(ctrl) @@ -195,27 +195,27 @@ func TestController_Process(t *testing.T) { { name: "cancelled batch cancels requests", batch: entity.Batch{ - ID: "test-queue/batch/3", + ID: "3", Queue: "test-queue", - Contains: []string{"test-queue/10"}, + Contains: []string{"10"}, State: entity.BatchStateCancelled, Version: 2, }, setupStore: func(ctrl *gomock.Controller) *orchstoragemock.MockStorage { mockBatchStore := storagemock.NewMockBatchStore(ctrl) - mockBatchStore.EXPECT().Get(gomock.Any(), "test-queue/batch/3").Return(entity.Batch{ - ID: "test-queue/batch/3", + mockBatchStore.EXPECT().Get(gomock.Any(), "3").Return(entity.Batch{ + ID: "3", Queue: "test-queue", - Contains: []string{"test-queue/10"}, + Contains: []string{"10"}, State: entity.BatchStateCancelled, Version: 2, }, nil) mockRequestStore := storagemock.NewMockRequestStore(ctrl) request := entity.Request{ - ID: "test-queue/10", Queue: "test-queue", Version: 4, State: entity.RequestStateProcessing, + ID: "10", Queue: "test-queue", Version: 4, State: entity.RequestStateProcessing, } - mockRequestStore.EXPECT().Get(gomock.Any(), "test-queue/10").Return(request, nil) + mockRequestStore.EXPECT().Get(gomock.Any(), "10").Return(request, nil) mockRequestStore.EXPECT().Update(gomock.Any(), requestWithState(request, entity.RequestStateCancelled), int32(4), int32(5)).Return(nil) mockStorage := orchstoragemock.NewMockStorage(ctrl) @@ -228,18 +228,18 @@ func TestController_Process(t *testing.T) { { name: "idempotent retry: request already in target terminal state still publishes log", batch: entity.Batch{ - ID: "test-queue/batch/8", + ID: "8", Queue: "test-queue", - Contains: []string{"test-queue/20"}, + Contains: []string{"20"}, State: entity.BatchStateSucceeded, Version: 2, }, setupStore: func(ctrl *gomock.Controller) *orchstoragemock.MockStorage { mockBatchStore := storagemock.NewMockBatchStore(ctrl) - mockBatchStore.EXPECT().Get(gomock.Any(), "test-queue/batch/8").Return(entity.Batch{ - ID: "test-queue/batch/8", + mockBatchStore.EXPECT().Get(gomock.Any(), "8").Return(entity.Batch{ + ID: "8", Queue: "test-queue", - Contains: []string{"test-queue/20"}, + Contains: []string{"20"}, State: entity.BatchStateSucceeded, Version: 2, }, nil) @@ -247,8 +247,8 @@ func TestController_Process(t *testing.T) { // Request is already Landed (prior delivery wrote it). UpdateState // must NOT be called — gomock will fail the test if it is. mockRequestStore := storagemock.NewMockRequestStore(ctrl) - mockRequestStore.EXPECT().Get(gomock.Any(), "test-queue/20").Return(entity.Request{ - ID: "test-queue/20", Queue: "test-queue", Version: 7, State: entity.RequestStateLanded, + mockRequestStore.EXPECT().Get(gomock.Any(), "20").Return(entity.Request{ + ID: "20", Queue: "test-queue", Version: 7, State: entity.RequestStateLanded, }, nil) mockStorage := orchstoragemock.NewMockStorage(ctrl) @@ -261,18 +261,18 @@ func TestController_Process(t *testing.T) { { name: "divergent terminal state skips reconcile and log publish", batch: entity.Batch{ - ID: "test-queue/batch/9", + ID: "9", Queue: "test-queue", - Contains: []string{"test-queue/30"}, + Contains: []string{"30"}, State: entity.BatchStateSucceeded, Version: 2, }, setupStore: func(ctrl *gomock.Controller) *orchstoragemock.MockStorage { mockBatchStore := storagemock.NewMockBatchStore(ctrl) - mockBatchStore.EXPECT().Get(gomock.Any(), "test-queue/batch/9").Return(entity.Batch{ - ID: "test-queue/batch/9", + mockBatchStore.EXPECT().Get(gomock.Any(), "9").Return(entity.Batch{ + ID: "9", Queue: "test-queue", - Contains: []string{"test-queue/30"}, + Contains: []string{"30"}, State: entity.BatchStateSucceeded, Version: 2, }, nil) @@ -281,8 +281,8 @@ func TestController_Process(t *testing.T) { // Conclude must not write the log entry (the other writer owns it), // and must not attempt UpdateState. mockRequestStore := storagemock.NewMockRequestStore(ctrl) - mockRequestStore.EXPECT().Get(gomock.Any(), "test-queue/30").Return(entity.Request{ - ID: "test-queue/30", Queue: "test-queue", Version: 5, State: entity.RequestStateCancelled, + mockRequestStore.EXPECT().Get(gomock.Any(), "30").Return(entity.Request{ + ID: "30", Queue: "test-queue", Version: 5, State: entity.RequestStateCancelled, }, nil) mockStorage := orchstoragemock.NewMockStorage(ctrl) @@ -295,18 +295,18 @@ func TestController_Process(t *testing.T) { { name: "missing request returns error", batch: entity.Batch{ - ID: "test-queue/batch/11", + ID: "11", Queue: "test-queue", - Contains: []string{"test-queue/40"}, + Contains: []string{"40"}, State: entity.BatchStateSucceeded, Version: 2, }, setupStore: func(ctrl *gomock.Controller) *orchstoragemock.MockStorage { mockBatchStore := storagemock.NewMockBatchStore(ctrl) - mockBatchStore.EXPECT().Get(gomock.Any(), "test-queue/batch/11").Return(entity.Batch{ - ID: "test-queue/batch/11", + mockBatchStore.EXPECT().Get(gomock.Any(), "11").Return(entity.Batch{ + ID: "11", Queue: "test-queue", - Contains: []string{"test-queue/40"}, + Contains: []string{"40"}, State: entity.BatchStateSucceeded, Version: 2, }, nil) @@ -314,7 +314,7 @@ func TestController_Process(t *testing.T) { // A request referenced by the batch but missing from the store is a // hard error (nack/retry, eventually DLQ) — not a silent skip. mockRequestStore := storagemock.NewMockRequestStore(ctrl) - mockRequestStore.EXPECT().Get(gomock.Any(), "test-queue/40").Return(entity.Request{}, storage.ErrNotFound) + mockRequestStore.EXPECT().Get(gomock.Any(), "40").Return(entity.Request{}, storage.ErrNotFound) mockStorage := orchstoragemock.NewMockStorage(ctrl) mockStorage.EXPECT().GetBatchStore().Return(mockBatchStore).AnyTimes() @@ -327,18 +327,18 @@ func TestController_Process(t *testing.T) { { name: "non-terminal batch state returns error", batch: entity.Batch{ - ID: "test-queue/batch/4", + ID: "4", Queue: "test-queue", - Contains: []string{"test-queue/1"}, + Contains: []string{"1"}, State: entity.BatchStateCreated, Version: 1, }, setupStore: func(ctrl *gomock.Controller) *orchstoragemock.MockStorage { mockBatchStore := storagemock.NewMockBatchStore(ctrl) - mockBatchStore.EXPECT().Get(gomock.Any(), "test-queue/batch/4").Return(entity.Batch{ - ID: "test-queue/batch/4", + mockBatchStore.EXPECT().Get(gomock.Any(), "4").Return(entity.Batch{ + ID: "4", Queue: "test-queue", - Contains: []string{"test-queue/1"}, + Contains: []string{"1"}, State: entity.BatchStateCreated, Version: 1, }, nil) @@ -353,24 +353,24 @@ func TestController_Process(t *testing.T) { { name: "request store get failure returns error", batch: entity.Batch{ - ID: "test-queue/batch/5", + ID: "5", Queue: "test-queue", - Contains: []string{"test-queue/1"}, + Contains: []string{"1"}, State: entity.BatchStateSucceeded, Version: 2, }, setupStore: func(ctrl *gomock.Controller) *orchstoragemock.MockStorage { mockBatchStore := storagemock.NewMockBatchStore(ctrl) - mockBatchStore.EXPECT().Get(gomock.Any(), "test-queue/batch/5").Return(entity.Batch{ - ID: "test-queue/batch/5", + mockBatchStore.EXPECT().Get(gomock.Any(), "5").Return(entity.Batch{ + ID: "5", Queue: "test-queue", - Contains: []string{"test-queue/1"}, + Contains: []string{"1"}, State: entity.BatchStateSucceeded, Version: 2, }, nil) mockRequestStore := storagemock.NewMockRequestStore(ctrl) - mockRequestStore.EXPECT().Get(gomock.Any(), "test-queue/1").Return(entity.Request{}, fmt.Errorf("db connection lost")) + mockRequestStore.EXPECT().Get(gomock.Any(), "1").Return(entity.Request{}, fmt.Errorf("db connection lost")) mockStorage := orchstoragemock.NewMockStorage(ctrl) mockStorage.EXPECT().GetBatchStore().Return(mockBatchStore).AnyTimes() @@ -383,27 +383,27 @@ func TestController_Process(t *testing.T) { { name: "request store version mismatch is retryable", batch: entity.Batch{ - ID: "test-queue/batch/6", + ID: "6", Queue: "test-queue", - Contains: []string{"test-queue/1"}, + Contains: []string{"1"}, State: entity.BatchStateSucceeded, Version: 2, }, setupStore: func(ctrl *gomock.Controller) *orchstoragemock.MockStorage { mockBatchStore := storagemock.NewMockBatchStore(ctrl) - mockBatchStore.EXPECT().Get(gomock.Any(), "test-queue/batch/6").Return(entity.Batch{ - ID: "test-queue/batch/6", + mockBatchStore.EXPECT().Get(gomock.Any(), "6").Return(entity.Batch{ + ID: "6", Queue: "test-queue", - Contains: []string{"test-queue/1"}, + Contains: []string{"1"}, State: entity.BatchStateSucceeded, Version: 2, }, nil) mockRequestStore := storagemock.NewMockRequestStore(ctrl) request := entity.Request{ - ID: "test-queue/1", Queue: "test-queue", Version: 2, State: entity.RequestStateProcessing, + ID: "1", Queue: "test-queue", Version: 2, State: entity.RequestStateProcessing, } - mockRequestStore.EXPECT().Get(gomock.Any(), "test-queue/1").Return(request, nil) + mockRequestStore.EXPECT().Get(gomock.Any(), "1").Return(request, nil) mockRequestStore.EXPECT().Update(gomock.Any(), requestWithState(request, entity.RequestStateLanded), int32(2), int32(3)).Return(storage.ErrVersionMismatch) mockStorage := orchstoragemock.NewMockStorage(ctrl) @@ -417,15 +417,15 @@ func TestController_Process(t *testing.T) { { name: "empty contains list succeeds", batch: entity.Batch{ - ID: "test-queue/batch/7", + ID: "7", Queue: "test-queue", State: entity.BatchStateSucceeded, Version: 1, }, setupStore: func(ctrl *gomock.Controller) *orchstoragemock.MockStorage { mockBatchStore := storagemock.NewMockBatchStore(ctrl) - mockBatchStore.EXPECT().Get(gomock.Any(), "test-queue/batch/7").Return(entity.Batch{ - ID: "test-queue/batch/7", + mockBatchStore.EXPECT().Get(gomock.Any(), "7").Return(entity.Batch{ + ID: "7", Queue: "test-queue", State: entity.BatchStateSucceeded, Version: 1, @@ -476,9 +476,9 @@ func TestController_Process_FailedBatchCarriesReasonToRequestLog(t *testing.T) { const reason = "merge failed: conflict in pkg/a/foo.go" batch := entity.Batch{ - ID: "test-queue/batch/9", + ID: "9", Queue: "test-queue", - Contains: []string{"test-queue/9"}, + Contains: []string{"9"}, State: entity.BatchStateFailed, Version: 2, } @@ -486,9 +486,9 @@ func TestController_Process_FailedBatchCarriesReasonToRequestLog(t *testing.T) { batchStore := storagemock.NewMockBatchStore(ctrl) batchStore.EXPECT().Get(gomock.Any(), batch.ID).Return(batch, nil) - request := entity.Request{ID: "test-queue/9", Queue: "test-queue", Version: 1, State: entity.RequestStateProcessing} + request := entity.Request{ID: "9", Queue: "test-queue", Version: 1, State: entity.RequestStateProcessing} requestStore := storagemock.NewMockRequestStore(ctrl) - requestStore.EXPECT().Get(gomock.Any(), "test-queue/9").Return(request, nil) + requestStore.EXPECT().Get(gomock.Any(), "9").Return(request, nil) requestStore.EXPECT().Update(gomock.Any(), requestWithState(request, entity.RequestStateError), int32(1), int32(2)).Return(nil) store := orchstoragemock.NewMockStorage(ctrl) @@ -523,14 +523,14 @@ func TestController_Process_StorageFailure(t *testing.T) { ctrl := gomock.NewController(t) mockBatchStore := storagemock.NewMockBatchStore(ctrl) - mockBatchStore.EXPECT().Get(gomock.Any(), "test-queue/batch/1").Return(entity.Batch{}, fmt.Errorf("db connection lost")) + mockBatchStore.EXPECT().Get(gomock.Any(), "1").Return(entity.Batch{}, fmt.Errorf("db connection lost")) mockStorage := orchstoragemock.NewMockStorage(ctrl) mockStorage.EXPECT().GetBatchStore().Return(mockBatchStore).AnyTimes() controller, _ := newTestController(t, ctrl, mockStorage, false) - msg := entityqueue.NewMessage("test-queue/batch/1", batchIDPayload(t, "test-queue/batch/1"), "test-queue", nil) + msg := entityqueue.NewMessage("1", batchIDPayload(t, "1"), "test-queue", nil) msg.Tenant = "test-queue" delivery := consumermock.NewMockDelivery(ctrl) delivery.EXPECT().Message().Return(msg).AnyTimes() diff --git a/submitqueue/orchestrator/controller/dependencyanalysis/dependencyanalysis_test.go b/submitqueue/orchestrator/controller/dependencyanalysis/dependencyanalysis_test.go index 9748afb53..d16c07ec2 100644 --- a/submitqueue/orchestrator/controller/dependencyanalysis/dependencyanalysis_test.go +++ b/submitqueue/orchestrator/controller/dependencyanalysis/dependencyanalysis_test.go @@ -43,8 +43,8 @@ import ( const ( testQueue = "test-queue" - testRequestID = "test-queue/123" - testBatchID = "test-queue/batch/3" + testRequestID = "123" + testBatchID = "3" ) // analyzerCfg is the per-queue identity handed to the conflict analyzer in @@ -183,7 +183,7 @@ func TestNewController(t *testing.T) { func TestController_Process_RejectsTenantPayloadQueueMismatch(t *testing.T) { ctrl := gomock.NewController(t) controller := newTestController(t, ctrl, orchstoragemock.NewMockStorage(ctrl), nil, nil) - msg := entityqueue.NewMessage("test-queue/batch/1", batchIDPayload(t, "test-queue/batch/1", "test-queue"), "test-queue", nil) + msg := entityqueue.NewMessage("1", batchIDPayload(t, "1", "test-queue"), "test-queue", nil) msg.Tenant = "other-queue" delivery := consumermock.NewMockDelivery(ctrl) delivery.EXPECT().Message().Return(msg).AnyTimes() @@ -198,8 +198,8 @@ func TestController_Process_AnalyzesAndTransitionsToCreated(t *testing.T) { batch := testBatch() inFlight := []entity.Batch{ - {ID: "test-queue/batch/1", Queue: testQueue, State: entity.BatchStateCreated, Version: 1}, - {ID: "test-queue/batch/2", Queue: testQueue, State: entity.BatchStateSpeculating, Version: 2}, + {ID: "1", Queue: testQueue, State: entity.BatchStateCreated, Version: 1}, + {ID: "2", Queue: testQueue, State: entity.BatchStateSpeculating, Version: 2}, } batchStore := storagemock.NewMockBatchStore(ctrl) @@ -232,12 +232,12 @@ func TestController_Process_AnalyzesAndTransitionsToCreated(t *testing.T) { }, int32(1), int32(2)).Return(nil) dependentStore.EXPECT().Get(gomock.Any(), inFlight[1].ID).Return(entity.BatchDependent{ BatchID: inFlight[1].ID, - Dependents: []string{"test-queue/batch/99"}, + Dependents: []string{"99"}, Version: 2, }, nil) dependentStore.EXPECT().Update(gomock.Any(), entity.BatchDependent{ BatchID: inFlight[1].ID, - Dependents: []string{"test-queue/batch/99", batch.ID}, + Dependents: []string{"99", batch.ID}, Version: 2, }, int32(2), int32(3)).Return(nil) @@ -258,7 +258,7 @@ func TestController_Process_DedupesAnalyzerConflicts(t *testing.T) { ctrl := gomock.NewController(t) batch := testBatch() - inFlight := entity.Batch{ID: "test-queue/batch/2", Queue: testQueue, State: entity.BatchStateSpeculating, Version: 2} + inFlight := entity.Batch{ID: "2", Queue: testQueue, State: entity.BatchStateSpeculating, Version: 2} batchStore := storagemock.NewMockBatchStore(ctrl) batchStore.EXPECT().Get(gomock.Any(), batch.ID).Return(batch, nil) @@ -308,7 +308,7 @@ func TestController_Process_RedeliveryAfterTransitionOnlyRepublishes(t *testing. batch := testBatch() batch.State = entity.BatchStateCreated - batch.Dependencies = []string{"test-queue/batch/1"} + batch.Dependencies = []string{"1"} batch.Version = 2 batchStore := storagemock.NewMockBatchStore(ctrl) @@ -335,7 +335,7 @@ func TestController_Process_RedeliveryMidIndexDoesNotDoubleAppend(t *testing.T) ctrl := gomock.NewController(t) batch := testBatch() - inFlight := entity.Batch{ID: "test-queue/batch/1", Queue: testQueue, State: entity.BatchStateCreated, Version: 1} + inFlight := entity.Batch{ID: "1", Queue: testQueue, State: entity.BatchStateCreated, Version: 1} batchStore := storagemock.NewMockBatchStore(ctrl) batchStore.EXPECT().Get(gomock.Any(), batch.ID).Return(batch, nil) @@ -531,10 +531,10 @@ func TestController_Process_IndexUpdateFailureDoesNotMutateFetchedDependents(t * ctrl := gomock.NewController(t) batch := testBatch() - inFlight := entity.Batch{ID: "test-queue/batch/1", Queue: testQueue, State: entity.BatchStateCreated, Version: 1} + inFlight := entity.Batch{ID: "1", Queue: testQueue, State: entity.BatchStateCreated, Version: 1} dependents := make([]string, 1, 2) - dependents[0] = "test-queue/batch/98" + dependents[0] = "98" existing := entity.BatchDependent{BatchID: inFlight.ID, Dependents: dependents, Version: 4} batchStore := storagemock.NewMockBatchStore(ctrl) @@ -546,7 +546,7 @@ func TestController_Process_IndexUpdateFailureDoesNotMutateFetchedDependents(t * dependentStore.EXPECT().Get(gomock.Any(), inFlight.ID).Return(existing, nil) dependentStore.EXPECT().Update(gomock.Any(), entity.BatchDependent{ BatchID: inFlight.ID, - Dependents: []string{"test-queue/batch/98", batch.ID}, + Dependents: []string{"98", batch.ID}, Version: existing.Version, }, existing.Version, existing.Version+1).Return(errors.New("update failed")) @@ -565,7 +565,7 @@ func TestController_Process_IndexUpdateFailureDoesNotMutateFetchedDependents(t * func TestController_Process_PromotionErrors(t *testing.T) { batch := testBatch() - dependencyID := "test-queue/batch/1" + dependencyID := "1" inFlight := entity.Batch{ID: dependencyID, Queue: testQueue, State: entity.BatchStateCreated, Version: 1} storeErr := errors.New("storage failed") @@ -686,7 +686,7 @@ func TestController_Process_AbandonsBatchWhoseRequestIsAlreadyEnrolled(t *testin batch := testBatch() winner := entity.Batch{ - ID: "test-queue/batch/2", Queue: testQueue, Contains: []string{testRequestID}, + ID: "2", Queue: testQueue, Contains: []string{testRequestID}, State: entity.BatchStateSpeculating, Version: 3, } diff --git a/submitqueue/orchestrator/controller/dlq/batch_test.go b/submitqueue/orchestrator/controller/dlq/batch_test.go index 048a0437d..0c0b4c045 100644 --- a/submitqueue/orchestrator/controller/dlq/batch_test.go +++ b/submitqueue/orchestrator/controller/dlq/batch_test.go @@ -47,17 +47,17 @@ func TestDLQBatchController_Process_FailsAndFansOut(t *testing.T) { batchStore := storagemock.NewMockBatchStore(ctrl) batch := entity.Batch{ - ID: "q/batch/9", Queue: "q", Contains: []string{"q/1"}, + ID: "9", Queue: "q", Contains: []string{"1"}, State: entity.BatchStateLanding, Version: 2, } - batchStore.EXPECT().Get(gomock.Any(), "q/batch/9").Return(batch, nil) + batchStore.EXPECT().Get(gomock.Any(), "9").Return(batch, nil) batchStore.EXPECT().Update(gomock.Any(), batchWithState(batch, entity.BatchStateFailed), int32(2), int32(3)).Return(nil) requestStore := storagemock.NewMockRequestStore(ctrl) request := entity.Request{ - ID: "q/1", Queue: "q", Version: 1, State: entity.RequestStateProcessing, + ID: "1", Queue: "q", Version: 1, State: entity.RequestStateProcessing, } - requestStore.EXPECT().Get(gomock.Any(), "q/1").Return(request, nil) + requestStore.EXPECT().Get(gomock.Any(), "1").Return(request, nil) requestStore.EXPECT().Update(gomock.Any(), requestWithState(request, entity.RequestStateError), int32(1), int32(2)).Return(nil) registry := newTestLogRegistry(t, ctrl, 1, func(entity.RequestLog) error { @@ -71,7 +71,7 @@ func TestDLQBatchController_Process_FailsAndFansOut(t *testing.T) { c := NewDLQBatchController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, registry, TopicKey(topickey.TopicKeyLand), "orchestrator-land-dlq") - payload, err := sqmq.MarshalID(sqmq.TopicKeyLand, "q/batch/9", "q") + payload, err := sqmq.MarshalID(sqmq.TopicKeyLand, "9", "q") require.NoError(t, err) delivery := newMockDelivery(ctrl, payload) @@ -83,7 +83,7 @@ func TestDLQBatchController_Process_TenantPayloadQueueMismatchAcks(t *testing.T) store := orchstoragemock.NewMockStorage(ctrl) c := NewDLQBatchController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(topickey.TopicKeyLand), "orchestrator-land-dlq") - payload, err := sqmq.MarshalID(sqmq.TopicKeyLand, "q/batch/9", "q") + payload, err := sqmq.MarshalID(sqmq.TopicKeyLand, "9", "q") require.NoError(t, err) require.NoError(t, c.Process(context.Background(), newMockDeliveryWithTenant(ctrl, payload, "other-queue"))) diff --git a/submitqueue/orchestrator/controller/dlq/buildsignal_test.go b/submitqueue/orchestrator/controller/dlq/buildsignal_test.go index c573fa539..c58b195de 100644 --- a/submitqueue/orchestrator/controller/dlq/buildsignal_test.go +++ b/submitqueue/orchestrator/controller/dlq/buildsignal_test.go @@ -48,23 +48,23 @@ func TestDLQBuildSignalController_Process_FansOutToBatch(t *testing.T) { buildStore := storagemock.NewMockBuildStore(ctrl) buildStore.EXPECT().Get(gomock.Any(), "build-1").Return(entity.Build{ - ID: "build-1", BatchID: "q/batch/2", PathID: "path-1", Attempt: 1, + ID: "build-1", BatchID: "2", PathID: "path-1", Attempt: 1, Status: entity.BuildStatusRunning, }, nil) batchStore := storagemock.NewMockBatchStore(ctrl) batch := entity.Batch{ - ID: "q/batch/2", Queue: "q", Contains: []string{"q/1"}, + ID: "2", Queue: "q", Contains: []string{"1"}, State: entity.BatchStateSpeculating, Version: 3, } - batchStore.EXPECT().Get(gomock.Any(), "q/batch/2").Return(batch, nil) + batchStore.EXPECT().Get(gomock.Any(), "2").Return(batch, nil) batchStore.EXPECT().Update(gomock.Any(), batchWithState(batch, entity.BatchStateFailed), int32(3), int32(4)).Return(nil) requestStore := storagemock.NewMockRequestStore(ctrl) request := entity.Request{ - ID: "q/1", Queue: "q", Version: 1, State: entity.RequestStateProcessing, + ID: "1", Queue: "q", Version: 1, State: entity.RequestStateProcessing, } - requestStore.EXPECT().Get(gomock.Any(), "q/1").Return(request, nil) + requestStore.EXPECT().Get(gomock.Any(), "1").Return(request, nil) requestStore.EXPECT().Update(gomock.Any(), requestWithState(request, entity.RequestStateError), int32(1), int32(2)).Return(nil) registry := newTestLogRegistry(t, ctrl, 1, func(entity.RequestLog) error { diff --git a/submitqueue/orchestrator/controller/dlq/dlq_test.go b/submitqueue/orchestrator/controller/dlq/dlq_test.go index 64d46c000..02fd82063 100644 --- a/submitqueue/orchestrator/controller/dlq/dlq_test.go +++ b/submitqueue/orchestrator/controller/dlq/dlq_test.go @@ -86,8 +86,8 @@ func TestFailRequest_TerminalStates(t *testing.T) { ctrl := gomock.NewController(t) requestStore := storagemock.NewMockRequestStore(ctrl) - requestStore.EXPECT().Get(gomock.Any(), "q/1").Return(entity.Request{ - ID: "q/1", Queue: "q", Version: 5, State: tt.state, + requestStore.EXPECT().Get(gomock.Any(), "1").Return(entity.Request{ + ID: "1", Queue: "q", Version: 5, State: tt.state, }, nil) store := orchstoragemock.NewMockStorage(ctrl) @@ -96,14 +96,14 @@ func TestFailRequest_TerminalStates(t *testing.T) { registry := consumer.TopicRegistry{} if tt.wantLog { registry = newTestLogRegistry(t, ctrl, 1, func(l entity.RequestLog) error { - assert.Equal(t, "q/1", l.RequestID) + assert.Equal(t, "1", l.RequestID) assert.Equal(t, entity.RequestStatusError, l.Status) assert.Equal(t, int32(5), l.RequestVersion) return nil }) } - err := failRequest(context.Background(), store, registry, zaptest.NewLogger(t).Sugar(), "q/1", "", nil) + err := failRequest(context.Background(), store, registry, zaptest.NewLogger(t).Sugar(), "1", "", nil) require.NoError(t, err) }) } @@ -119,13 +119,13 @@ func TestFailRequest_CancellingTransitionsToError(t *testing.T) { requestStore := storagemock.NewMockRequestStore(ctrl) request := entity.Request{ - ID: "q/1", Queue: "q", Version: 7, State: entity.RequestStateCancelling, + ID: "1", Queue: "q", Version: 7, State: entity.RequestStateCancelling, } - requestStore.EXPECT().Get(gomock.Any(), "q/1").Return(request, nil) + requestStore.EXPECT().Get(gomock.Any(), "1").Return(request, nil) requestStore.EXPECT().Update(gomock.Any(), requestWithState(request, entity.RequestStateError), int32(7), int32(8)).Return(nil) registry := newTestLogRegistry(t, ctrl, 1, func(l entity.RequestLog) error { - assert.Equal(t, "q/1", l.RequestID) + assert.Equal(t, "1", l.RequestID) assert.Equal(t, entity.RequestStatusError, l.Status) assert.Equal(t, int32(8), l.RequestVersion) return nil @@ -135,7 +135,7 @@ func TestFailRequest_CancellingTransitionsToError(t *testing.T) { store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() store.EXPECT().GetRequestStore().Return(requestStore).AnyTimes() - err := failRequest(context.Background(), store, registry, zaptest.NewLogger(t).Sugar(), "q/1", "", nil) + err := failRequest(context.Background(), store, registry, zaptest.NewLogger(t).Sugar(), "1", "", nil) require.NoError(t, err) } @@ -144,13 +144,13 @@ func TestFailRequest_TransitionsToError(t *testing.T) { requestStore := storagemock.NewMockRequestStore(ctrl) request := entity.Request{ - ID: "q/1", Queue: "q", Version: 3, State: entity.RequestStateValidated, + ID: "1", Queue: "q", Version: 3, State: entity.RequestStateValidated, } - requestStore.EXPECT().Get(gomock.Any(), "q/1").Return(request, nil) + requestStore.EXPECT().Get(gomock.Any(), "1").Return(request, nil) requestStore.EXPECT().Update(gomock.Any(), requestWithState(request, entity.RequestStateError), int32(3), int32(4)).Return(nil) registry := newTestLogRegistry(t, ctrl, 1, func(l entity.RequestLog) error { - assert.Equal(t, "q/1", l.RequestID) + assert.Equal(t, "1", l.RequestID) assert.Equal(t, entity.RequestStatusError, l.Status) assert.Equal(t, int32(4), l.RequestVersion) return nil @@ -160,7 +160,7 @@ func TestFailRequest_TransitionsToError(t *testing.T) { store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() store.EXPECT().GetRequestStore().Return(requestStore).AnyTimes() - err := failRequest(context.Background(), store, registry, zaptest.NewLogger(t).Sugar(), "q/1", "", nil) + err := failRequest(context.Background(), store, registry, zaptest.NewLogger(t).Sugar(), "1", "", nil) require.NoError(t, err) } @@ -172,9 +172,9 @@ func TestFailRequest_LogPublishErrorPropagates(t *testing.T) { requestStore := storagemock.NewMockRequestStore(ctrl) request := entity.Request{ - ID: "q/1", Queue: "q", Version: 3, State: entity.RequestStateValidated, + ID: "1", Queue: "q", Version: 3, State: entity.RequestStateValidated, } - requestStore.EXPECT().Get(gomock.Any(), "q/1").Return(request, nil) + requestStore.EXPECT().Get(gomock.Any(), "1").Return(request, nil) requestStore.EXPECT().Update(gomock.Any(), requestWithState(request, entity.RequestStateError), int32(3), int32(4)).Return(nil) registry := newTestLogRegistry(t, ctrl, 1, func(entity.RequestLog) error { @@ -185,7 +185,7 @@ func TestFailRequest_LogPublishErrorPropagates(t *testing.T) { store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() store.EXPECT().GetRequestStore().Return(requestStore).AnyTimes() - err := failRequest(context.Background(), store, registry, zaptest.NewLogger(t).Sugar(), "q/1", "", nil) + err := failRequest(context.Background(), store, registry, zaptest.NewLogger(t).Sugar(), "1", "", nil) require.Error(t, err) } @@ -193,13 +193,13 @@ func TestFailRequest_NotFoundIsNoOp(t *testing.T) { ctrl := gomock.NewController(t) requestStore := storagemock.NewMockRequestStore(ctrl) - requestStore.EXPECT().Get(gomock.Any(), "q/1").Return(entity.Request{}, storage.ErrNotFound) + requestStore.EXPECT().Get(gomock.Any(), "1").Return(entity.Request{}, storage.ErrNotFound) store := orchstoragemock.NewMockStorage(ctrl) store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() store.EXPECT().GetRequestStore().Return(requestStore).AnyTimes() - err := failRequest(context.Background(), store, consumer.TopicRegistry{}, zaptest.NewLogger(t).Sugar(), "q/1", "", nil) + err := failRequest(context.Background(), store, consumer.TopicRegistry{}, zaptest.NewLogger(t).Sugar(), "1", "", nil) require.NoError(t, err) } @@ -207,13 +207,13 @@ func TestFailRequest_GenericGetErrorIsNonRetryable(t *testing.T) { ctrl := gomock.NewController(t) requestStore := storagemock.NewMockRequestStore(ctrl) - requestStore.EXPECT().Get(gomock.Any(), "q/1").Return(entity.Request{}, fmt.Errorf("boom")) + requestStore.EXPECT().Get(gomock.Any(), "1").Return(entity.Request{}, fmt.Errorf("boom")) store := orchstoragemock.NewMockStorage(ctrl) store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() store.EXPECT().GetRequestStore().Return(requestStore).AnyTimes() - err := failRequest(context.Background(), store, consumer.TopicRegistry{}, zaptest.NewLogger(t).Sugar(), "q/1", "", nil) + err := failRequest(context.Background(), store, consumer.TopicRegistry{}, zaptest.NewLogger(t).Sugar(), "1", "", nil) require.Error(t, err) assert.False(t, errs.IsRetryable(err)) } @@ -225,22 +225,22 @@ func TestFailBatch_TransitionsAndFansOut(t *testing.T) { batchStore := storagemock.NewMockBatchStore(ctrl) batch := entity.Batch{ - ID: "q/batch/1", Queue: "q", Contains: []string{"q/1", "q/2"}, + ID: "1", Queue: "q", Contains: []string{"1", "2"}, State: entity.BatchStateLanding, Version: 4, } - batchStore.EXPECT().Get(gomock.Any(), "q/batch/1").Return(batch, nil) + batchStore.EXPECT().Get(gomock.Any(), "1").Return(batch, nil) batchStore.EXPECT().Update(gomock.Any(), batchWithState(batch, entity.BatchStateFailed), int32(4), int32(5)).Return(nil) requestStore := storagemock.NewMockRequestStore(ctrl) request1 := entity.Request{ - ID: "q/1", Queue: "q", Version: 2, State: entity.RequestStateProcessing, + ID: "1", Queue: "q", Version: 2, State: entity.RequestStateProcessing, } - requestStore.EXPECT().Get(gomock.Any(), "q/1").Return(request1, nil) + requestStore.EXPECT().Get(gomock.Any(), "1").Return(request1, nil) requestStore.EXPECT().Update(gomock.Any(), requestWithState(request1, entity.RequestStateError), int32(2), int32(3)).Return(nil) request2 := entity.Request{ - ID: "q/2", Queue: "q", Version: 1, State: entity.RequestStateProcessing, + ID: "2", Queue: "q", Version: 1, State: entity.RequestStateProcessing, } - requestStore.EXPECT().Get(gomock.Any(), "q/2").Return(request2, nil) + requestStore.EXPECT().Get(gomock.Any(), "2").Return(request2, nil) requestStore.EXPECT().Update(gomock.Any(), requestWithState(request2, entity.RequestStateError), int32(1), int32(2)).Return(nil) registry := newTestLogRegistry(t, ctrl, 2, func(entity.RequestLog) error { @@ -252,7 +252,7 @@ func TestFailBatch_TransitionsAndFansOut(t *testing.T) { store.EXPECT().GetBatchStore().Return(batchStore).AnyTimes() store.EXPECT().GetRequestStore().Return(requestStore).AnyTimes() - _, err := failBatch(context.Background(), store, registry, zaptest.NewLogger(t).Sugar(), "q/batch/1", "", nil) + _, err := failBatch(context.Background(), store, registry, zaptest.NewLogger(t).Sugar(), "1", "", nil) require.NoError(t, err) } @@ -260,17 +260,17 @@ func TestFailBatch_FailedFansOutForRepair(t *testing.T) { ctrl := gomock.NewController(t) batchStore := storagemock.NewMockBatchStore(ctrl) - batchStore.EXPECT().Get(gomock.Any(), "q/batch/1").Return(entity.Batch{ - ID: "q/batch/1", Queue: "q", Contains: []string{"q/1"}, + batchStore.EXPECT().Get(gomock.Any(), "1").Return(entity.Batch{ + ID: "1", Queue: "q", Contains: []string{"1"}, State: entity.BatchStateFailed, Version: 5, }, nil) // no batchStore.Update expected requestStore := storagemock.NewMockRequestStore(ctrl) request := entity.Request{ - ID: "q/1", Queue: "q", Version: 2, State: entity.RequestStateProcessing, + ID: "1", Queue: "q", Version: 2, State: entity.RequestStateProcessing, } - requestStore.EXPECT().Get(gomock.Any(), "q/1").Return(request, nil) + requestStore.EXPECT().Get(gomock.Any(), "1").Return(request, nil) requestStore.EXPECT().Update(gomock.Any(), requestWithState(request, entity.RequestStateError), int32(2), int32(3)).Return(nil) registry := newTestLogRegistry(t, ctrl, 1, func(entity.RequestLog) error { @@ -282,7 +282,7 @@ func TestFailBatch_FailedFansOutForRepair(t *testing.T) { store.EXPECT().GetBatchStore().Return(batchStore).AnyTimes() store.EXPECT().GetRequestStore().Return(requestStore).AnyTimes() - _, err := failBatch(context.Background(), store, registry, zaptest.NewLogger(t).Sugar(), "q/batch/1", "", nil) + _, err := failBatch(context.Background(), store, registry, zaptest.NewLogger(t).Sugar(), "1", "", nil) require.NoError(t, err) } @@ -291,15 +291,15 @@ func TestFailBatch_DifferentTerminalOutcomeSkipsFanOut(t *testing.T) { t.Run(string(state), func(t *testing.T) { ctrl := gomock.NewController(t) batchStore := storagemock.NewMockBatchStore(ctrl) - batchStore.EXPECT().Get(gomock.Any(), "q/batch/1").Return(entity.Batch{ - ID: "q/batch/1", Queue: "q", Contains: []string{"q/1"}, State: state, Version: 5, + batchStore.EXPECT().Get(gomock.Any(), "1").Return(entity.Batch{ + ID: "1", Queue: "q", Contains: []string{"1"}, State: state, Version: 5, }, nil) store := orchstoragemock.NewMockStorage(ctrl) store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() store.EXPECT().GetBatchStore().Return(batchStore).AnyTimes() - _, err := failBatch(context.Background(), store, consumer.TopicRegistry{}, zaptest.NewLogger(t).Sugar(), "q/batch/1", "", nil) + _, err := failBatch(context.Background(), store, consumer.TopicRegistry{}, zaptest.NewLogger(t).Sugar(), "1", "", nil) require.NoError(t, err) }) } @@ -316,17 +316,17 @@ func TestFailBatch_CancellingTransitionsToFailed(t *testing.T) { batchStore := storagemock.NewMockBatchStore(ctrl) batch := entity.Batch{ - ID: "q/batch/1", Queue: "q", Contains: []string{"q/1"}, + ID: "1", Queue: "q", Contains: []string{"1"}, State: entity.BatchStateCancelling, Version: 6, } - batchStore.EXPECT().Get(gomock.Any(), "q/batch/1").Return(batch, nil) + batchStore.EXPECT().Get(gomock.Any(), "1").Return(batch, nil) batchStore.EXPECT().Update(gomock.Any(), batchWithState(batch, entity.BatchStateFailed), int32(6), int32(7)).Return(nil) requestStore := storagemock.NewMockRequestStore(ctrl) request := entity.Request{ - ID: "q/1", Queue: "q", Version: 3, State: entity.RequestStateCancelling, + ID: "1", Queue: "q", Version: 3, State: entity.RequestStateCancelling, } - requestStore.EXPECT().Get(gomock.Any(), "q/1").Return(request, nil) + requestStore.EXPECT().Get(gomock.Any(), "1").Return(request, nil) requestStore.EXPECT().Update(gomock.Any(), requestWithState(request, entity.RequestStateError), int32(3), int32(4)).Return(nil) registry := newTestLogRegistry(t, ctrl, 1, func(entity.RequestLog) error { @@ -338,7 +338,7 @@ func TestFailBatch_CancellingTransitionsToFailed(t *testing.T) { store.EXPECT().GetBatchStore().Return(batchStore).AnyTimes() store.EXPECT().GetRequestStore().Return(requestStore).AnyTimes() - _, err := failBatch(context.Background(), store, registry, zaptest.NewLogger(t).Sugar(), "q/batch/1", "", nil) + _, err := failBatch(context.Background(), store, registry, zaptest.NewLogger(t).Sugar(), "1", "", nil) require.NoError(t, err) } @@ -346,13 +346,13 @@ func TestFailBatch_NotFoundIsNoOp(t *testing.T) { ctrl := gomock.NewController(t) batchStore := storagemock.NewMockBatchStore(ctrl) - batchStore.EXPECT().Get(gomock.Any(), "q/batch/1").Return(entity.Batch{}, storage.ErrNotFound) + batchStore.EXPECT().Get(gomock.Any(), "1").Return(entity.Batch{}, storage.ErrNotFound) store := orchstoragemock.NewMockStorage(ctrl) store.EXPECT().GetQueueBatchStateStore().Return(newQueueBatchStateStore(ctrl)).AnyTimes() store.EXPECT().GetBatchStore().Return(batchStore).AnyTimes() - _, err := failBatch(context.Background(), store, consumer.TopicRegistry{}, zaptest.NewLogger(t).Sugar(), "q/batch/1", "", nil) + _, err := failBatch(context.Background(), store, consumer.TopicRegistry{}, zaptest.NewLogger(t).Sugar(), "1", "", nil) require.NoError(t, err) } diff --git a/submitqueue/orchestrator/controller/dlq/landconflictsignal_test.go b/submitqueue/orchestrator/controller/dlq/landconflictsignal_test.go index 867d10486..4c7479438 100644 --- a/submitqueue/orchestrator/controller/dlq/landconflictsignal_test.go +++ b/submitqueue/orchestrator/controller/dlq/landconflictsignal_test.go @@ -47,9 +47,9 @@ func TestDLQLandConflictSignalController_Process_ReconcilesRequest(t *testing.T) requestStore := storagemock.NewMockRequestStore(ctrl) request := entity.Request{ - ID: "q/1", Queue: "q", Version: 1, State: entity.RequestStateProcessing, + ID: "1", Queue: "q", Version: 1, State: entity.RequestStateProcessing, } - requestStore.EXPECT().Get(gomock.Any(), "q/1").Return(request, nil) + requestStore.EXPECT().Get(gomock.Any(), "1").Return(request, nil) requestStore.EXPECT().Update(gomock.Any(), requestWithState(request, entity.RequestStateError), int32(1), int32(2)).Return(nil) registry := newTestLogRegistry(t, ctrl, 1, func(entity.RequestLog) error { @@ -62,7 +62,7 @@ func TestDLQLandConflictSignalController_Process_ReconcilesRequest(t *testing.T) c := NewDLQLandConflictSignalController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, registry, TopicKey(runwaymq.TopicKeyMergeConflictCheckSignal), "orchestrator-landconflictsignal-dlq") - payload, err := runwaymq.Marshal(&runwaymq.MergeResult{Id: "q/1", QueueName: "q", Outcome: runwaypb.Outcome_FAILED, Reason: "boom"}) + payload, err := runwaymq.Marshal(&runwaymq.MergeResult{Id: "1", QueueName: "q", Outcome: runwaypb.Outcome_FAILED, Reason: "boom"}) require.NoError(t, err) delivery := newMockDelivery(ctrl, payload) @@ -74,7 +74,7 @@ func TestDLQLandConflictSignalController_Process_TenantPayloadQueueMismatchAcks( store := orchstoragemock.NewMockStorage(ctrl) c := NewDLQLandConflictSignalController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(runwaymq.TopicKeyMergeConflictCheckSignal), "orchestrator-landconflictsignal-dlq") - payload, err := runwaymq.Marshal(&runwaymq.MergeResult{Id: "q/1", QueueName: "q", Outcome: runwaypb.Outcome_FAILED, Reason: "boom"}) + payload, err := runwaymq.Marshal(&runwaymq.MergeResult{Id: "1", QueueName: "q", Outcome: runwaypb.Outcome_FAILED, Reason: "boom"}) require.NoError(t, err) require.NoError(t, c.Process(context.Background(), newMockDeliveryWithTenant(ctrl, payload, "other-queue"))) diff --git a/submitqueue/orchestrator/controller/dlq/landsignal_test.go b/submitqueue/orchestrator/controller/dlq/landsignal_test.go index 28a6157e0..938e3da8c 100644 --- a/submitqueue/orchestrator/controller/dlq/landsignal_test.go +++ b/submitqueue/orchestrator/controller/dlq/landsignal_test.go @@ -49,17 +49,17 @@ func TestDLQLandSignalController_Process_ReconcilesBatch(t *testing.T) { batchStore := storagemock.NewMockBatchStore(ctrl) batch := entity.Batch{ - ID: "q/batch/1", Queue: "q", Contains: []string{"q/1"}, + ID: "1", Queue: "q", Contains: []string{"1"}, State: entity.BatchStateLanding, Version: 2, } - batchStore.EXPECT().Get(gomock.Any(), "q/batch/1").Return(batch, nil) + batchStore.EXPECT().Get(gomock.Any(), "1").Return(batch, nil) batchStore.EXPECT().Update(gomock.Any(), batchWithState(batch, entity.BatchStateFailed), int32(2), int32(3)).Return(nil) requestStore := storagemock.NewMockRequestStore(ctrl) request := entity.Request{ - ID: "q/1", Queue: "q", Version: 1, State: entity.RequestStateProcessing, + ID: "1", Queue: "q", Version: 1, State: entity.RequestStateProcessing, } - requestStore.EXPECT().Get(gomock.Any(), "q/1").Return(request, nil) + requestStore.EXPECT().Get(gomock.Any(), "1").Return(request, nil) requestStore.EXPECT().Update(gomock.Any(), requestWithState(request, entity.RequestStateError), int32(1), int32(2)).Return(nil) registry := newTestLogRegistry(t, ctrl, 1, func(entity.RequestLog) error { @@ -73,7 +73,7 @@ func TestDLQLandSignalController_Process_ReconcilesBatch(t *testing.T) { c := NewDLQLandSignalController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, registry, TopicKey(runwaymq.TopicKeyMergeSignal), "orchestrator-landsignal-dlq") - payload, err := runwaymq.Marshal(&runwaymq.MergeResult{Id: "q/batch/1", QueueName: "q", Outcome: runwaypb.Outcome_FAILED, Reason: "boom"}) + payload, err := runwaymq.Marshal(&runwaymq.MergeResult{Id: "1", QueueName: "q", Outcome: runwaypb.Outcome_FAILED, Reason: "boom"}) require.NoError(t, err) delivery := newMockDelivery(ctrl, payload) @@ -85,7 +85,7 @@ func TestDLQLandSignalController_Process_TenantPayloadQueueMismatchAcks(t *testi store := orchstoragemock.NewMockStorage(ctrl) c := NewDLQLandSignalController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, TopicKey(runwaymq.TopicKeyMergeSignal), "orchestrator-landsignal-dlq") - payload, err := runwaymq.Marshal(&runwaymq.MergeResult{Id: "q/batch/1", QueueName: "q", Outcome: runwaypb.Outcome_FAILED, Reason: "boom"}) + payload, err := runwaymq.Marshal(&runwaymq.MergeResult{Id: "1", QueueName: "q", Outcome: runwaypb.Outcome_FAILED, Reason: "boom"}) require.NoError(t, err) require.NoError(t, c.Process(context.Background(), newMockDeliveryWithTenant(ctrl, payload, "other-queue"))) diff --git a/submitqueue/orchestrator/controller/dlq/request_test.go b/submitqueue/orchestrator/controller/dlq/request_test.go index 559a40745..5cc6c6f3a 100644 --- a/submitqueue/orchestrator/controller/dlq/request_test.go +++ b/submitqueue/orchestrator/controller/dlq/request_test.go @@ -51,9 +51,9 @@ func TestDLQRequestController_Process_LandRequestPayload(t *testing.T) { requestStore := storagemock.NewMockRequestStore(ctrl) request := entity.Request{ - ID: "q/1", Queue: "q", Version: 1, State: entity.RequestStateStarted, + ID: "1", Queue: "q", Version: 1, State: entity.RequestStateStarted, } - requestStore.EXPECT().Get(gomock.Any(), "q/1").Return(request, nil) + requestStore.EXPECT().Get(gomock.Any(), "1").Return(request, nil) requestStore.EXPECT().Update(gomock.Any(), requestWithState(request, entity.RequestStateError), int32(1), int32(2)).Return(nil) registry := newTestLogRegistry(t, ctrl, 1, func(entity.RequestLog) error { @@ -67,7 +67,7 @@ func TestDLQRequestController_Process_LandRequestPayload(t *testing.T) { c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, registry, DecodeLandRequestID, TopicKey(topickey.TopicKeyStart), "orchestrator-start-dlq") - payload, err := sqmq.Marshal(sqmq.StartFromLandRequest(entity.LandRequest{ID: "q/1", Queue: "q"})) + payload, err := sqmq.Marshal(sqmq.StartFromLandRequest(entity.LandRequest{ID: "1", Queue: "q"})) require.NoError(t, err) delivery := newMockDelivery(ctrl, payload) @@ -79,7 +79,7 @@ func TestDLQRequestController_Process_TenantPayloadQueueMismatchAcks(t *testing. store := orchstoragemock.NewMockStorage(ctrl) c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, DecodeRequestID(topickey.TopicKeyValidate), TopicKey(topickey.TopicKeyValidate), "orchestrator-validate-dlq") - payload, err := sqmq.MarshalID(sqmq.TopicKeyValidate, "q/1", "q") + payload, err := sqmq.MarshalID(sqmq.TopicKeyValidate, "1", "q") require.NoError(t, err) require.NoError(t, c.Process(context.Background(), newMockDeliveryWithTenant(ctrl, payload, "other-queue"))) @@ -90,9 +90,9 @@ func TestDLQRequestController_Process_CancelRequestPayload(t *testing.T) { requestStore := storagemock.NewMockRequestStore(ctrl) request := entity.Request{ - ID: "q/7", Queue: "q", Version: 2, State: entity.RequestStateBatched, + ID: "7", Queue: "q", Version: 2, State: entity.RequestStateBatched, } - requestStore.EXPECT().Get(gomock.Any(), "q/7").Return(request, nil) + requestStore.EXPECT().Get(gomock.Any(), "7").Return(request, nil) requestStore.EXPECT().Update(gomock.Any(), requestWithState(request, entity.RequestStateError), int32(2), int32(3)).Return(nil) registry := newTestLogRegistry(t, ctrl, 1, func(entity.RequestLog) error { @@ -106,7 +106,7 @@ func TestDLQRequestController_Process_CancelRequestPayload(t *testing.T) { c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, registry, DecodeCancelRequestID, TopicKey(topickey.TopicKeyCancel), "orchestrator-cancel-dlq") - payload, err := sqmq.Marshal(sqmq.CancelFromEntity(entity.CancelRequest{ID: "q/7", Queue: "q", Reason: "user"})) + payload, err := sqmq.Marshal(sqmq.CancelFromEntity(entity.CancelRequest{ID: "7", Queue: "q", Reason: "user"})) require.NoError(t, err) delivery := newMockDelivery(ctrl, payload) @@ -118,9 +118,9 @@ func TestDLQRequestController_Process_RequestIDPayload(t *testing.T) { requestStore := storagemock.NewMockRequestStore(ctrl) request := entity.Request{ - ID: "q/3", Queue: "q", Version: 1, State: entity.RequestStateValidated, + ID: "3", Queue: "q", Version: 1, State: entity.RequestStateValidated, } - requestStore.EXPECT().Get(gomock.Any(), "q/3").Return(request, nil) + requestStore.EXPECT().Get(gomock.Any(), "3").Return(request, nil) requestStore.EXPECT().Update(gomock.Any(), requestWithState(request, entity.RequestStateError), int32(1), int32(2)).Return(nil) registry := newTestLogRegistry(t, ctrl, 1, func(log entity.RequestLog) error { @@ -135,7 +135,7 @@ func TestDLQRequestController_Process_RequestIDPayload(t *testing.T) { c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, registry, DecodeRequestID(topickey.TopicKeyBatch), TopicKey(topickey.TopicKeyBatch), "orchestrator-batch-dlq") - payload, err := sqmq.MarshalID(sqmq.TopicKeyBatch, "q/3", "q") + payload, err := sqmq.MarshalID(sqmq.TopicKeyBatch, "3", "q") require.NoError(t, err) delivery := newMockDelivery(ctrl, payload) @@ -146,8 +146,8 @@ func TestDLQRequestController_Process_DifferentTerminalOutcomeSkips(t *testing.T ctrl := gomock.NewController(t) requestStore := storagemock.NewMockRequestStore(ctrl) - requestStore.EXPECT().Get(gomock.Any(), "q/1").Return(entity.Request{ - ID: "q/1", Queue: "q", Version: 5, State: entity.RequestStateLanded, + requestStore.EXPECT().Get(gomock.Any(), "1").Return(entity.Request{ + ID: "1", Queue: "q", Version: 5, State: entity.RequestStateLanded, }, nil) store := orchstoragemock.NewMockStorage(ctrl) @@ -157,7 +157,7 @@ func TestDLQRequestController_Process_DifferentTerminalOutcomeSkips(t *testing.T c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, DecodeRequestID(topickey.TopicKeyValidate), TopicKey(topickey.TopicKeyValidate), "orchestrator-validate-dlq") - payload, err := sqmq.MarshalID(sqmq.TopicKeyValidate, "q/1", "q") + payload, err := sqmq.MarshalID(sqmq.TopicKeyValidate, "1", "q") require.NoError(t, err) delivery := newMockDelivery(ctrl, payload) @@ -241,13 +241,13 @@ func TestDLQRequestController_Process_SkipsRequestOwnedByLiveBatch(t *testing.T) // Request store with no EXPECTs — the request must not be touched. associations := storagemock.NewMockRequestBatchStore(ctrl) - associations.EXPECT().GetByRequestID(gomock.Any(), "q/1").Return([]entity.RequestBatch{ - {RequestID: "q/1", BatchID: "q/batch/1", Version: 1}, + associations.EXPECT().GetByRequestID(gomock.Any(), "1").Return([]entity.RequestBatch{ + {RequestID: "1", BatchID: "1", Version: 1}, }, nil) batchStore := storagemock.NewMockBatchStore(ctrl) - batchStore.EXPECT().Get(gomock.Any(), "q/batch/1").Return(entity.Batch{ - ID: "q/batch/1", Queue: "q", Contains: []string{"q/1"}, State: state, Version: 1, + batchStore.EXPECT().Get(gomock.Any(), "1").Return(entity.Batch{ + ID: "1", Queue: "q", Contains: []string{"1"}, State: state, Version: 1, }, nil) store := orchstoragemock.NewMockStorage(ctrl) @@ -259,7 +259,7 @@ func TestDLQRequestController_Process_SkipsRequestOwnedByLiveBatch(t *testing.T) c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, DecodeRequestID(topickey.TopicKeyBatch), TopicKey(topickey.TopicKeyBatch), "orchestrator-batch-dlq") - payload, err := sqmq.MarshalID(sqmq.TopicKeyBatch, "q/1", "q") + payload, err := sqmq.MarshalID(sqmq.TopicKeyBatch, "1", "q") require.NoError(t, err) require.NoError(t, c.Process(context.Background(), newMockDelivery(ctrl, payload))) @@ -272,19 +272,19 @@ func TestDLQRequestController_Process_SkipsRequestOwnedByLiveBatch(t *testing.T) func TestDLQRequestController_Process_FailsWhenEveryBatchIsTerminal(t *testing.T) { ctrl := gomock.NewController(t) - request := entity.Request{ID: "q/1", Queue: "q", Version: 1, State: entity.RequestStateBatched} + request := entity.Request{ID: "1", Queue: "q", Version: 1, State: entity.RequestStateBatched} requestStore := storagemock.NewMockRequestStore(ctrl) - requestStore.EXPECT().Get(gomock.Any(), "q/1").Return(request, nil) + requestStore.EXPECT().Get(gomock.Any(), "1").Return(request, nil) requestStore.EXPECT().Update(gomock.Any(), requestWithState(request, entity.RequestStateError), int32(1), int32(2)).Return(nil) associations := storagemock.NewMockRequestBatchStore(ctrl) - associations.EXPECT().GetByRequestID(gomock.Any(), "q/1").Return([]entity.RequestBatch{ - {RequestID: "q/1", BatchID: "q/batch/1", Version: 1}, + associations.EXPECT().GetByRequestID(gomock.Any(), "1").Return([]entity.RequestBatch{ + {RequestID: "1", BatchID: "1", Version: 1}, }, nil) batchStore := storagemock.NewMockBatchStore(ctrl) - batchStore.EXPECT().Get(gomock.Any(), "q/batch/1").Return(entity.Batch{ - ID: "q/batch/1", Queue: "q", Contains: []string{"q/1"}, State: entity.BatchStateFailed, Version: 1, + batchStore.EXPECT().Get(gomock.Any(), "1").Return(entity.Batch{ + ID: "1", Queue: "q", Contains: []string{"1"}, State: entity.BatchStateFailed, Version: 1, }, nil) registry := newTestLogRegistry(t, ctrl, 1, func(entity.RequestLog) error { return nil }) @@ -298,7 +298,7 @@ func TestDLQRequestController_Process_FailsWhenEveryBatchIsTerminal(t *testing.T c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, registry, DecodeRequestID(topickey.TopicKeyBatch), TopicKey(topickey.TopicKeyBatch), "orchestrator-batch-dlq") - payload, err := sqmq.MarshalID(sqmq.TopicKeyBatch, "q/1", "q") + payload, err := sqmq.MarshalID(sqmq.TopicKeyBatch, "1", "q") require.NoError(t, err) require.NoError(t, c.Process(context.Background(), newMockDelivery(ctrl, payload))) @@ -316,19 +316,19 @@ func TestDLQRequestController_Process_FailsWhenCreatingBatchNeverClaimed(t *test t.Run(string(state), func(t *testing.T) { ctrl := gomock.NewController(t) - request := entity.Request{ID: "q/1", Queue: "q", Version: 1, State: state} + request := entity.Request{ID: "1", Queue: "q", Version: 1, State: state} requestStore := storagemock.NewMockRequestStore(ctrl) - requestStore.EXPECT().Get(gomock.Any(), "q/1").Return(request, nil).Times(2) + requestStore.EXPECT().Get(gomock.Any(), "1").Return(request, nil).Times(2) requestStore.EXPECT().Update(gomock.Any(), requestWithState(request, entity.RequestStateError), int32(1), int32(2)).Return(nil) associations := storagemock.NewMockRequestBatchStore(ctrl) - associations.EXPECT().GetByRequestID(gomock.Any(), "q/1").Return([]entity.RequestBatch{ - {RequestID: "q/1", BatchID: "q/batch/1", Version: 1}, + associations.EXPECT().GetByRequestID(gomock.Any(), "1").Return([]entity.RequestBatch{ + {RequestID: "1", BatchID: "1", Version: 1}, }, nil) batchStore := storagemock.NewMockBatchStore(ctrl) - batchStore.EXPECT().Get(gomock.Any(), "q/batch/1").Return(entity.Batch{ - ID: "q/batch/1", Queue: "q", Contains: []string{"q/1"}, State: entity.BatchStateCreating, Version: 1, + batchStore.EXPECT().Get(gomock.Any(), "1").Return(entity.Batch{ + ID: "1", Queue: "q", Contains: []string{"1"}, State: entity.BatchStateCreating, Version: 1, }, nil) registry := newTestLogRegistry(t, ctrl, 1, func(entity.RequestLog) error { return nil }) @@ -342,7 +342,7 @@ func TestDLQRequestController_Process_FailsWhenCreatingBatchNeverClaimed(t *test c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, registry, DecodeRequestID(topickey.TopicKeyBatch), TopicKey(topickey.TopicKeyBatch), "orchestrator-batch-dlq") - payload, err := sqmq.MarshalID(sqmq.TopicKeyBatch, "q/1", "q") + payload, err := sqmq.MarshalID(sqmq.TopicKeyBatch, "1", "q") require.NoError(t, err) require.NoError(t, c.Process(context.Background(), newMockDelivery(ctrl, payload))) @@ -357,18 +357,18 @@ func TestDLQRequestController_Process_SkipsWhenCreatingBatchAlreadyClaimed(t *te ctrl := gomock.NewController(t) requestStore := storagemock.NewMockRequestStore(ctrl) - requestStore.EXPECT().Get(gomock.Any(), "q/1"). - Return(entity.Request{ID: "q/1", Queue: "q", Version: 2, State: entity.RequestStateBatched}, nil) + requestStore.EXPECT().Get(gomock.Any(), "1"). + Return(entity.Request{ID: "1", Queue: "q", Version: 2, State: entity.RequestStateBatched}, nil) // Update must NOT be called — the batch owns the outcome. associations := storagemock.NewMockRequestBatchStore(ctrl) - associations.EXPECT().GetByRequestID(gomock.Any(), "q/1").Return([]entity.RequestBatch{ - {RequestID: "q/1", BatchID: "q/batch/1", Version: 1}, + associations.EXPECT().GetByRequestID(gomock.Any(), "1").Return([]entity.RequestBatch{ + {RequestID: "1", BatchID: "1", Version: 1}, }, nil) batchStore := storagemock.NewMockBatchStore(ctrl) - batchStore.EXPECT().Get(gomock.Any(), "q/batch/1").Return(entity.Batch{ - ID: "q/batch/1", Queue: "q", Contains: []string{"q/1"}, State: entity.BatchStateCreating, Version: 1, + batchStore.EXPECT().Get(gomock.Any(), "1").Return(entity.Batch{ + ID: "1", Queue: "q", Contains: []string{"1"}, State: entity.BatchStateCreating, Version: 1, }, nil) store := orchstoragemock.NewMockStorage(ctrl) @@ -380,7 +380,7 @@ func TestDLQRequestController_Process_SkipsWhenCreatingBatchAlreadyClaimed(t *te c := NewDLQRequestController(zaptest.NewLogger(t).Sugar(), testScope(), staticStorageFactory{store: store}, consumer.TopicRegistry{}, DecodeRequestID(topickey.TopicKeyBatch), TopicKey(topickey.TopicKeyBatch), "orchestrator-batch-dlq") - payload, err := sqmq.MarshalID(sqmq.TopicKeyBatch, "q/1", "q") + payload, err := sqmq.MarshalID(sqmq.TopicKeyBatch, "1", "q") require.NoError(t, err) require.NoError(t, c.Process(context.Background(), newMockDelivery(ctrl, payload))) diff --git a/submitqueue/orchestrator/controller/dlq/speculate_test.go b/submitqueue/orchestrator/controller/dlq/speculate_test.go index 3c72345f2..a9e4479cf 100644 --- a/submitqueue/orchestrator/controller/dlq/speculate_test.go +++ b/submitqueue/orchestrator/controller/dlq/speculate_test.go @@ -122,16 +122,16 @@ func TestDLQSpeculateController_Process_Attribution(t *testing.T) { ctrl := gomock.NewController(t) blamed := entity.Batch{ - ID: tt.wantFailedBatch, Queue: "q", Contains: []string{"q/1"}, + ID: tt.wantFailedBatch, Queue: "q", Contains: []string{"1"}, State: entity.BatchStateSpeculating, Version: 2, } batchStore := storagemock.NewMockBatchStore(ctrl) batchStore.EXPECT().Get(gomock.Any(), tt.wantFailedBatch).Return(blamed, nil) batchStore.EXPECT().Update(gomock.Any(), batchWithState(blamed, entity.BatchStateFailed), int32(2), int32(3)).Return(nil) - request := entity.Request{ID: "q/1", Queue: "q", Version: 1, State: entity.RequestStateProcessing} + request := entity.Request{ID: "1", Queue: "q", Version: 1, State: entity.RequestStateProcessing} requestStore := storagemock.NewMockRequestStore(ctrl) - requestStore.EXPECT().Get(gomock.Any(), "q/1").Return(request, nil) + requestStore.EXPECT().Get(gomock.Any(), "1").Return(request, nil) requestStore.EXPECT().Update(gomock.Any(), requestWithState(request, entity.RequestStateError), int32(1), int32(2)).Return(nil) // The queue drained with the blamed batch, so no re-trigger. @@ -172,26 +172,26 @@ func TestDLQSpeculateController_Process_RetriggersQueue(t *testing.T) { ctrl := gomock.NewController(t) failedBatch := entity.Batch{ - ID: "q/batch/1", Queue: "q", Contains: nil, + ID: "1", Queue: "q", Contains: nil, State: entity.BatchStateSpeculating, Version: 1, } // Two still live afterwards; the lower ID is chosen so the wake-up is // reproducible despite ListByStates promising no order. - liveA := entity.Batch{ID: "q/batch/2", Queue: "q", State: entity.BatchStateSpeculating, Version: 1} - liveB := entity.Batch{ID: "q/batch/3", Queue: "q", State: entity.BatchStateSpeculating, Version: 1} + liveA := entity.Batch{ID: "2", Queue: "q", State: entity.BatchStateSpeculating, Version: 1} + liveB := entity.Batch{ID: "3", Queue: "q", State: entity.BatchStateSpeculating, Version: 1} batchStore := storagemock.NewMockBatchStore(ctrl) - batchStore.EXPECT().Get(gomock.Any(), "q/batch/1").Return(failedBatch, nil) + batchStore.EXPECT().Get(gomock.Any(), "1").Return(failedBatch, nil) batchStore.EXPECT().Update(gomock.Any(), batchWithState(failedBatch, entity.BatchStateFailed), int32(1), int32(2)).Return(nil) - batchStore.EXPECT().Get(gomock.Any(), "q/batch/3").Return(liveB, nil).AnyTimes() - batchStore.EXPECT().Get(gomock.Any(), "q/batch/2").Return(liveA, nil).AnyTimes() + batchStore.EXPECT().Get(gomock.Any(), "3").Return(liveB, nil).AnyTimes() + batchStore.EXPECT().Get(gomock.Any(), "2").Return(liveA, nil).AnyTimes() queueBatchState := storagemock.NewMockQueueBatchStateStore(ctrl) queueBatchState.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() queueBatchState.EXPECT().Delete(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() queueBatchState.EXPECT().List(gomock.Any(), entity.BatchStateSpeculating).Return([]entity.QueueBatchState{ - {Queue: "q", State: entity.BatchStateSpeculating, BatchID: "q/batch/3"}, - {Queue: "q", State: entity.BatchStateSpeculating, BatchID: "q/batch/2"}, + {Queue: "q", State: entity.BatchStateSpeculating, BatchID: "3"}, + {Queue: "q", State: entity.BatchStateSpeculating, BatchID: "2"}, }, nil).AnyTimes() queueBatchState.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() @@ -203,13 +203,13 @@ func TestDLQSpeculateController_Process_RetriggersQueue(t *testing.T) { registry := speculateRegistry(t, ctrl, 0, &republished, nil) c := newSpeculateController(registry, store, t) - payload, err := sqmq.MarshalID(sqmq.TopicKeySpeculate, "q/batch/1", "q") + payload, err := sqmq.MarshalID(sqmq.TopicKeySpeculate, "1", "q") require.NoError(t, err) delivery := newMockDeliveryWithFailure(ctrl, payload, failure.New("boom"), true) require.NoError(t, c.Process(context.Background(), delivery)) - assert.Equal(t, []string{"q/batch/2"}, republished) + assert.Equal(t, []string{"2"}, republished) } // Redelivery of a reconcile already done must publish nothing. Republishing @@ -219,18 +219,18 @@ func TestDLQSpeculateController_Process_NoRetriggerWithoutProgress(t *testing.T) ctrl := gomock.NewController(t) already := entity.Batch{ - ID: "q/batch/1", Queue: "q", Contains: nil, + ID: "1", Queue: "q", Contains: nil, State: entity.BatchStateFailed, Version: 4, } batchStore := storagemock.NewMockBatchStore(ctrl) - batchStore.EXPECT().Get(gomock.Any(), "q/batch/1").Return(already, nil) + batchStore.EXPECT().Get(gomock.Any(), "1").Return(already, nil) queueBatchState := storagemock.NewMockQueueBatchStateStore(ctrl) queueBatchState.EXPECT().Put(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() queueBatchState.EXPECT().Delete(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() // Live batches remain, so only the progress guard can be what stops it. queueBatchState.EXPECT().List(gomock.Any(), entity.BatchStateSpeculating).Return([]entity.QueueBatchState{ - {Queue: "q", State: entity.BatchStateSpeculating, BatchID: "q/batch/2"}, + {Queue: "q", State: entity.BatchStateSpeculating, BatchID: "2"}, }, nil).AnyTimes() queueBatchState.EXPECT().List(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() @@ -242,7 +242,7 @@ func TestDLQSpeculateController_Process_NoRetriggerWithoutProgress(t *testing.T) registry := speculateRegistry(t, ctrl, 0, &republished, nil) c := newSpeculateController(registry, store, t) - payload, err := sqmq.MarshalID(sqmq.TopicKeySpeculate, "q/batch/1", "q") + payload, err := sqmq.MarshalID(sqmq.TopicKeySpeculate, "1", "q") require.NoError(t, err) delivery := newMockDeliveryWithFailure(ctrl, payload, failure.New("boom"), true) diff --git a/submitqueue/orchestrator/controller/land/land_test.go b/submitqueue/orchestrator/controller/land/land_test.go index ca8a69107..67a8d5ccf 100644 --- a/submitqueue/orchestrator/controller/land/land_test.go +++ b/submitqueue/orchestrator/controller/land/land_test.go @@ -155,7 +155,7 @@ func TestNewController(t *testing.T) { func TestProcess_RejectsTenantPayloadQueueMismatch(t *testing.T) { ctrl := gomock.NewController(t) c := newController(t, orchstoragemock.NewMockStorage(ctrl), consumer.TopicRegistry{}) - msg := entityqueue.NewMessage("test-queue/batch/1", batchIDPayload(t, "test-queue/batch/1", "test-queue"), "test-queue", nil) + msg := entityqueue.NewMessage("1", batchIDPayload(t, "1", "test-queue"), "test-queue", nil) msg.Tenant = "other-queue" delivery := consumermock.NewMockDelivery(ctrl) delivery.EXPECT().Message().Return(msg).AnyTimes() @@ -166,15 +166,15 @@ func TestProcess_RejectsTenantPayloadQueueMismatch(t *testing.T) { func TestProcess_PublishesFullPayloadToRunway(t *testing.T) { ctrl := gomock.NewController(t) - const batchID = "test-queue/batch/1" + const batchID = "1" req1 := entity.Request{ - ID: "test-queue/1", + ID: "1", Queue: "test-queue", Change: change.Change{URIs: []string{"github://github.example.com/uber/repo/pull/1/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}, LandStrategy: mergestrategy.MergeStrategySquashRebase, } req2 := entity.Request{ - ID: "test-queue/2", + ID: "2", Queue: "test-queue", Change: change.Change{URIs: []string{"github://github.example.com/uber/repo/pull/2/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}}, LandStrategy: mergestrategy.MergeStrategyRebase, @@ -280,9 +280,9 @@ func TestProcess_HaltedBatchSkips(t *testing.T) { func TestProcess_ReportsLandingBeforeDispatch(t *testing.T) { ctrl := gomock.NewController(t) - const batchID = "test-queue/batch/3" - req1 := entity.Request{ID: "test-queue/1", Queue: "test-queue", LandStrategy: mergestrategy.MergeStrategyRebase} - req2 := entity.Request{ID: "test-queue/2", Queue: "test-queue", LandStrategy: mergestrategy.MergeStrategyRebase} + const batchID = "3" + req1 := entity.Request{ID: "1", Queue: "test-queue", LandStrategy: mergestrategy.MergeStrategyRebase} + req2 := entity.Request{ID: "2", Queue: "test-queue", LandStrategy: mergestrategy.MergeStrategyRebase} batch := entity.Batch{ ID: batchID, Queue: "test-queue", Contains: []string{req1.ID, req2.ID}, @@ -334,8 +334,8 @@ func TestProcess_PublishFailureReturnsError(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctrl := gomock.NewController(t) - const batchID = "test-queue/batch/2" - req := entity.Request{ID: "test-queue/1", Queue: "test-queue", LandStrategy: mergestrategy.MergeStrategyRebase} + const batchID = "2" + req := entity.Request{ID: "1", Queue: "test-queue", LandStrategy: mergestrategy.MergeStrategyRebase} batch := entity.Batch{ID: batchID, Queue: "test-queue", Contains: []string{req.ID}, State: entity.BatchStateLanding, Version: 1} batchStore := storagemock.NewMockBatchStore(ctrl) @@ -364,7 +364,7 @@ func TestProcess_PublishFailureReturnsError(t *testing.T) { func TestProcess_BatchStoreGetFailureNotRetryable(t *testing.T) { ctrl := gomock.NewController(t) - const batchID = "test-queue/batch/3" + const batchID = "3" batchStore := storagemock.NewMockBatchStore(ctrl) batchStore.EXPECT().Get(gomock.Any(), batchID).Return(entity.Batch{}, fmt.Errorf("db connection lost")) diff --git a/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go b/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go index f51e273da..ef2571383 100644 --- a/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go +++ b/submitqueue/orchestrator/controller/landconflictsignal/landconflictsignal_test.go @@ -69,7 +69,7 @@ func newDelivery(ctrl *gomock.Controller, msg entityqueue.Message) *consumermock } const ( - testRequestID = "test-queue/1" + testRequestID = "1" testQueue = "test-queue" ) diff --git a/submitqueue/orchestrator/controller/landsignal/landsignal_test.go b/submitqueue/orchestrator/controller/landsignal/landsignal_test.go index c713531e2..08ed5c05f 100644 --- a/submitqueue/orchestrator/controller/landsignal/landsignal_test.go +++ b/submitqueue/orchestrator/controller/landsignal/landsignal_test.go @@ -59,7 +59,7 @@ func batchWithState(batch entity.Batch, state entity.BatchState) entity.Batch { } const ( - testBatchID = "test-queue/batch/1" + testBatchID = "1" testQueue = "test-queue" ) @@ -144,8 +144,8 @@ func TestProcess_LandedAdvancesBatch(t *testing.T) { batch := entity.Batch{ ID: testBatchID, Queue: testQueue, - Contains: []string{"test-queue/1"}, - Dependencies: []string{"test-queue/batch/0"}, + Contains: []string{"1"}, + Dependencies: []string{"0"}, State: entity.BatchStateLanding, Version: 1, } @@ -162,7 +162,7 @@ func TestProcess_LandedAdvancesBatch(t *testing.T) { res := runwaymq.MergeResult{ Id: testBatchID, Outcome: runwaypb.Outcome_SUCCEEDED, - Steps: []*runwaymq.StepResult{{StepId: "test-queue/1", Outputs: []*runwaymq.StepOutput{{Id: "deadbeef"}}}}, + Steps: []*runwaymq.StepResult{{StepId: "1", Outputs: []*runwaymq.StepOutput{{Id: "deadbeef"}}}}, } msg := entityqueue.NewMessage(testBatchID, resultPayload(t, res), testQueue, nil) require.NoError(t, c.Process(context.Background(), newDelivery(ctrl, msg))) @@ -187,8 +187,8 @@ func TestProcess_FanoutDoesNotCollideWithTheBatchAnnouncement(t *testing.T) { batch := entity.Batch{ ID: testBatchID, Queue: testQueue, - Contains: []string{"test-queue/1"}, - Dependencies: []string{"test-queue/batch/0"}, + Contains: []string{"1"}, + Dependencies: []string{"0"}, State: entity.BatchStateLanding, Version: 1, } @@ -232,8 +232,8 @@ func TestProcess_NotLandedMarksBatchFailed(t *testing.T) { batch := entity.Batch{ ID: testBatchID, Queue: testQueue, - Contains: []string{"test-queue/1"}, - Dependencies: []string{"test-queue/batch/0"}, + Contains: []string{"1"}, + Dependencies: []string{"0"}, State: entity.BatchStateLanding, Version: 3, } diff --git a/submitqueue/orchestrator/controller/speculate/run_test.go b/submitqueue/orchestrator/controller/speculate/run_test.go index 6bf60c1d2..c40b172ce 100644 --- a/submitqueue/orchestrator/controller/speculate/run_test.go +++ b/submitqueue/orchestrator/controller/speculate/run_test.go @@ -282,7 +282,7 @@ func TestRun_PassesSnapshotToSpeculator(t *testing.T) { ctrl := gomock.NewController(t) spec := &scriptedSpeculator{} - landing := entity.Batch{ID: "q/batch/landing", Queue: "q", State: entity.BatchStateLanding, Version: 1} + landing := entity.Batch{ID: "100", Queue: "q", State: entity.BatchStateLanding, Version: 1} h := newRunHarness(t, ctrl, spec, []entity.Batch{speculatingHead(), landing}) h.noBuildsDispatched() h.batches.EXPECT().Get(gomock.Any(), dep1).Return(entity.Batch{ID: dep1, State: entity.BatchStateSucceeded}, nil) @@ -807,7 +807,7 @@ func TestRun_FailureCascadesWithinOneRun(t *testing.T) { // dependent is listed first, so a single forward pass would evaluate it // before the head it is stacked on has failed. dependent := entity.Batch{ - ID: "q/batch/dependent", Queue: "q", State: entity.BatchStateSpeculating, + ID: "101", Queue: "q", State: entity.BatchStateSpeculating, Dependencies: []string{head}, Version: 1, } failing := entity.Batch{ @@ -1010,7 +1010,7 @@ func TestRun_SpeculatorSeesPathSetsOfNonOpenHeads(t *testing.T) { spec := &scriptedSpeculator{} landing := entity.Batch{ - ID: "q/batch/landing", Queue: "q", State: entity.BatchStateLanding, Version: 1, + ID: "100", Queue: "q", State: entity.BatchStateLanding, Version: 1, } open := entity.Batch{ID: head, Queue: "q", State: entity.BatchStateSpeculating, Version: 1} @@ -1051,7 +1051,7 @@ func TestRun_PersistsObservationsWithNoOpenHead(t *testing.T) { spec := &scriptedSpeculator{} landing := entity.Batch{ - ID: "q/batch/landing", Queue: "q", State: entity.BatchStateLanding, Version: 1, + ID: "100", Queue: "q", State: entity.BatchStateLanding, Version: 1, } h := newRunHarness(t, ctrl, spec, []entity.Batch{landing}) @@ -1157,7 +1157,7 @@ func cascadePair(t *testing.T, ctrl *gomock.Controller, prerequisiteState entity prerequisite := entity.Batch{ID: head, Queue: "q", State: prerequisiteState, Version: 1} derived := entity.Batch{ - ID: "q/batch/derived", Queue: "q", State: entity.BatchStateSpeculating, + ID: "102", Queue: "q", State: entity.BatchStateSpeculating, Dependencies: []string{head}, Version: 1, } @@ -1230,7 +1230,7 @@ func TestRun_UnrelatedOutcomeStillCommitsAfterALostCAS(t *testing.T) { losing := entity.Batch{ID: head, Queue: "q", State: entity.BatchStateCancelling, Version: 1} independent := entity.Batch{ - ID: "q/batch/independent", Queue: "q", State: entity.BatchStateSpeculating, Version: 1, + ID: "103", Queue: "q", State: entity.BatchStateSpeculating, Version: 1, } h := newRunHarness(t, ctrl, &scriptedSpeculator{}, []entity.Batch{losing, independent}) @@ -1296,7 +1296,7 @@ func TestRun_FinalizedBatchPathSetIsWrittenOnce(t *testing.T) { func TestRun_CascadeDerivedBatchIsGivenARecoverySignalBeforeItIsTerminal(t *testing.T) { ctrl := gomock.NewController(t) - failing := entity.Batch{ID: "q/batch/derived", Queue: "q", State: entity.BatchStateSpeculating, Version: 1} + failing := entity.Batch{ID: "102", Queue: "q", State: entity.BatchStateSpeculating, Version: 1} h := newRunHarness(t, ctrl, &scriptedSpeculator{}, []entity.Batch{failing}) h.noBuildsDispatched() @@ -1351,7 +1351,7 @@ func TestRun_TriggerBatchNeedsNoRecoverySignal(t *testing.T) { func TestRun_CascadeDerivedBatchIsGivenARecoverySignalBeforeItLands(t *testing.T) { ctrl := gomock.NewController(t) - landing := entity.Batch{ID: "q/batch/derived", Queue: "q", State: entity.BatchStateSpeculating, Version: 1} + landing := entity.Batch{ID: "102", Queue: "q", State: entity.BatchStateSpeculating, Version: 1} h := newRunHarness(t, ctrl, &scriptedSpeculator{}, []entity.Batch{landing}) h.noBuildsDispatched() @@ -1412,7 +1412,7 @@ func TestRun_CancellingPathWithALiveBuildStaysCancelling(t *testing.T) { // run's request-log fan-out has somebody to report to. func memberHead() entity.Batch { h := speculatingHead() - h.Contains = []string{"q/1"} + h.Contains = []string{"1"} return h } @@ -1442,7 +1442,7 @@ func TestRun_ReportsPassedPathWhileWaiting(t *testing.T) { assert.Empty(t, h.published, "an unsettled head lands nowhere") require.Len(t, h.logs, 1) - assert.Equal(t, "q/1", h.logs[0].RequestID) + assert.Equal(t, "1", h.logs[0].RequestID) assert.Equal(t, entity.RequestLogTypeEvent, h.logs[0].Type) assert.Equal(t, entity.RequestEventWaiting, h.logs[0].Event) assert.Equal(t, head, h.logs[0].Metadata["batch_id"]) @@ -1487,7 +1487,7 @@ func TestRun_ReportsInvalidatedWhenTheDependencyFailsInTheSameRun(t *testing.T) leader := entity.Batch{ID: dep1, Queue: "q", State: entity.BatchStateSpeculating, Version: 1} followerBatch := entity.Batch{ - ID: head, Queue: "q", Contains: []string{"q/1"}, + ID: head, Queue: "q", Contains: []string{"1"}, State: entity.BatchStateSpeculating, Dependencies: []string{dep1}, Version: 1, } @@ -1624,7 +1624,7 @@ func TestRun_AdmitsBatchLeftInCreated(t *testing.T) { trigger := entity.Batch{ID: head, Queue: "q", State: entity.BatchStateSpeculating, Version: 1} straggler := entity.Batch{ - ID: "q/batch/9", Queue: "q", Contains: []string{"q/9"}, + ID: "9", Queue: "q", Contains: []string{"9"}, State: entity.BatchStateCreated, Version: 1, } @@ -1652,7 +1652,7 @@ func TestRun_AdmitLostRaceDoesNotFailRun(t *testing.T) { trigger := entity.Batch{ID: head, Queue: "q", State: entity.BatchStateSpeculating, Version: 1} straggler := entity.Batch{ - ID: "q/batch/9", Queue: "q", Contains: []string{"q/9"}, + ID: "9", Queue: "q", Contains: []string{"9"}, State: entity.BatchStateCreated, Version: 1, } @@ -1674,7 +1674,7 @@ func TestRun_AdmitStorageErrorDoesNotFailRun(t *testing.T) { trigger := entity.Batch{ID: head, Queue: "q", State: entity.BatchStateSpeculating, Version: 1} straggler := entity.Batch{ - ID: "q/batch/9", Queue: "q", Contains: []string{"q/9"}, + ID: "9", Queue: "q", Contains: []string{"9"}, State: entity.BatchStateCreated, Version: 1, } diff --git a/submitqueue/orchestrator/controller/speculate/speculate_test.go b/submitqueue/orchestrator/controller/speculate/speculate_test.go index 5855bf7d2..f826d2d9a 100644 --- a/submitqueue/orchestrator/controller/speculate/speculate_test.go +++ b/submitqueue/orchestrator/controller/speculate/speculate_test.go @@ -97,9 +97,9 @@ func batchIDPayload(t *testing.T, id string) []byte { func testBatch(state entity.BatchState, deps ...string) entity.Batch { return entity.Batch{ - ID: "test-queue/batch/1", + ID: "1", Queue: "test-queue", - Contains: []string{"test-queue/1"}, + Contains: []string{"1"}, Dependencies: deps, State: state, Version: 1, @@ -200,7 +200,7 @@ func TestNewController(t *testing.T) { func TestProcess_RejectsTenantPayloadQueueMismatch(t *testing.T) { ctrl := gomock.NewController(t) h := newProcHarness(t, ctrl, nil) - msg := entityqueue.NewMessage("test-queue/batch/1", batchIDPayload(t, "test-queue/batch/1"), "test-queue", nil) + msg := entityqueue.NewMessage("1", batchIDPayload(t, "1"), "test-queue", nil) msg.Tenant = "other-queue" d := consumermock.NewMockDelivery(ctrl) d.EXPECT().Message().Return(msg).AnyTimes() @@ -227,7 +227,7 @@ func TestProcess_AdmitsCreatedBatch(t *testing.T) { // Admission is the first thing a member hears after being batched: without // it the request reads "batched" for the whole of speculation. require.Len(t, h.logs, 1) - assert.Equal(t, "test-queue/1", h.logs[0].RequestID) + assert.Equal(t, "1", h.logs[0].RequestID) assert.Equal(t, entity.RequestStatusSpeculating, h.logs[0].Status) assert.Equal(t, batch.ID, h.logs[0].Metadata["batch_id"]) } @@ -279,7 +279,7 @@ func TestProcess_TerminalReplansQueue(t *testing.T) { batch := testBatch(entity.BatchStateSucceeded) dependent := entity.Batch{ - ID: "test-queue/batch/2", Queue: "test-queue", + ID: "2", Queue: "test-queue", State: entity.BatchStateSpeculating, Dependencies: []string{batch.ID}, Version: 1, } @@ -325,10 +325,10 @@ func TestProcess_Errors(t *testing.T) { t.Run("batch read failure", func(t *testing.T) { ctrl := gomock.NewController(t) h := newProcHarness(t, ctrl, nil) - h.batches.EXPECT().Get(gomock.Any(), "test-queue/batch/1"). + h.batches.EXPECT().Get(gomock.Any(), "1"). Return(entity.Batch{}, storage.ErrNotFound) - require.Error(t, h.process(t, ctrl, "test-queue/batch/1")) + require.Error(t, h.process(t, ctrl, "1")) }) t.Run("conclude publish failure on a terminal batch", func(t *testing.T) { diff --git a/submitqueue/orchestrator/controller/start/start_test.go b/submitqueue/orchestrator/controller/start/start_test.go index 461c634ce..0b7f027be 100644 --- a/submitqueue/orchestrator/controller/start/start_test.go +++ b/submitqueue/orchestrator/controller/start/start_test.go @@ -117,7 +117,7 @@ func TestController_Process_Success(t *testing.T) { controller := newTestController(t, ctrl, newMockStorage(ctrl), nil) delivery := makeDelivery(t, ctrl, entity.LandRequest{ - ID: "test-queue/123", + ID: "123", Queue: "test-queue", Change: change.Change{URIs: []string{"github://github.example.com/uber/service/pull/456/abcdef0123456789abcdef0123456789abcdef01"}}, LandStrategy: mergestrategy.MergeStrategyRebase, @@ -129,7 +129,7 @@ func TestController_Process_Success(t *testing.T) { func TestController_Process_RejectsTenantPayloadQueueMismatch(t *testing.T) { ctrl := gomock.NewController(t) controller := newTestController(t, ctrl, newMockStorage(ctrl), nil) - request := entity.LandRequest{ID: "test-queue/123", Queue: "test-queue"} + request := entity.LandRequest{ID: "123", Queue: "test-queue"} payload, err := sqmq.Marshal(sqmq.StartFromLandRequest(request)) require.NoError(t, err) msg := entityqueue.NewMessage(request.ID, payload, request.Queue, nil) @@ -175,7 +175,7 @@ func TestController_Process_ConstructsRequestWithStateAndVersion(t *testing.T) { controller := newTestController(t, ctrl, store, nil) landRequest := entity.LandRequest{ - ID: "test-queue/42", + ID: "42", Queue: "test-queue", Change: change.Change{URIs: []string{"github://github.example.com/uber/service/pull/1/abcdef0123456789abcdef0123456789abcdef01"}}, LandStrategy: mergestrategy.MergeStrategySquashRebase, @@ -224,7 +224,7 @@ func TestController_Process_PublishFailure(t *testing.T) { controller := newTestController(t, ctrl, newMockStorage(ctrl), fmt.Errorf("publish failed")) delivery := makeDelivery(t, ctrl, entity.LandRequest{ - ID: "test-queue/123", + ID: "123", Queue: "test-queue", Change: change.Change{URIs: []string{"github://github.example.com/uber/service/pull/1/789abc1234567890abcdef1234567890abcdef12"}}, LandStrategy: mergestrategy.MergeStrategyRebase, @@ -244,7 +244,7 @@ func TestController_Process_StorageFailure(t *testing.T) { controller := newTestController(t, ctrl, store, nil) delivery := makeDelivery(t, ctrl, entity.LandRequest{ - ID: "test-queue/123", + ID: "123", Queue: "test-queue", Change: change.Change{URIs: []string{"github://github.example.com/uber/service/pull/1/789abc1234567890abcdef1234567890abcdef12"}}, LandStrategy: mergestrategy.MergeStrategyRebase, @@ -266,7 +266,7 @@ func TestController_Process_AlreadyExistsSucceeds(t *testing.T) { controller := newTestController(t, ctrl, store, nil) delivery := makeDelivery(t, ctrl, entity.LandRequest{ - ID: "test-queue/123", + ID: "123", Queue: "test-queue", Change: change.Change{URIs: []string{"github://github.example.com/uber/service/pull/1/789abc1234567890abcdef1234567890abcdef12"}}, LandStrategy: mergestrategy.MergeStrategyRebase, diff --git a/submitqueue/orchestrator/controller/validate/validate_test.go b/submitqueue/orchestrator/controller/validate/validate_test.go index 1cdcd95d0..04653c62d 100644 --- a/submitqueue/orchestrator/controller/validate/validate_test.go +++ b/submitqueue/orchestrator/controller/validate/validate_test.go @@ -167,7 +167,7 @@ func newTestController( func TestNewController(t *testing.T) { ctrl := gomock.NewController(t) request := entity.Request{ - ID: "test-queue/123", + ID: "123", Queue: "test-queue", Change: change.Change{URIs: []string{"github://github.example.com/uber/service/pull/456/abcdef0123456789abcdef0123456789abcdef01"}}, LandStrategy: mergestrategy.MergeStrategyRebase, @@ -187,7 +187,7 @@ func TestController_Process_Success(t *testing.T) { ctrl := gomock.NewController(t) request := entity.Request{ - ID: "test-queue/123", + ID: "123", Queue: "test-queue", Change: change.Change{URIs: []string{"github://github.example.com/uber/service/pull/456/abcdef0123456789abcdef0123456789abcdef01"}}, LandStrategy: mergestrategy.MergeStrategyRebase, @@ -197,7 +197,7 @@ func TestController_Process_Success(t *testing.T) { store, _ := newMockStorage(ctrl, request) controller := newTestController(t, ctrl, store, newMockChangeStore(ctrl), nil) - msg := entityqueue.NewMessage("test-queue/123", requestIDPayload(t, request.ID), "test-queue", nil) + msg := entityqueue.NewMessage("123", requestIDPayload(t, request.ID), "test-queue", nil) msg.Tenant = "test-queue" delivery := consumermock.NewMockDelivery(ctrl) delivery.EXPECT().Message().Return(msg).AnyTimes() @@ -210,7 +210,7 @@ func TestController_Process_RejectsTenantPayloadQueueMismatch(t *testing.T) { ctrl := gomock.NewController(t) store := orchstoragemock.NewMockStorage(ctrl) controller := newTestController(t, ctrl, store, storagemock.NewMockChangeStore(ctrl), nil) - msg := entityqueue.NewMessage("test-queue/123", requestIDPayload(t, "test-queue/123"), "test-queue", nil) + msg := entityqueue.NewMessage("123", requestIDPayload(t, "123"), "test-queue", nil) msg.Tenant = "other-queue" delivery := consumermock.NewMockDelivery(ctrl) delivery.EXPECT().Message().Return(msg).AnyTimes() @@ -225,7 +225,7 @@ func TestController_Process_PublishesCheckToRunway(t *testing.T) { ctrl := gomock.NewController(t) request := entity.Request{ - ID: "test-queue/123", + ID: "123", Queue: "test-queue", Change: change.Change{URIs: []string{"github://github.example.com/uber/service/pull/456/abcdef0123456789abcdef0123456789abcdef01"}}, LandStrategy: mergestrategy.MergeStrategyRebase, @@ -298,7 +298,7 @@ func TestController_Process_ClaimsChangeRecordsWithDetails(t *testing.T) { // claim carries that change's details. const uri = "github://github.example.com/org/repo/pull/123/abcdef0123456789abcdef0123456789abcdef01" request := entity.Request{ - ID: "test-queue/123", + ID: "123", Queue: "test-queue", Change: change.Change{URIs: []string{uri}}, LandStrategy: mergestrategy.MergeStrategyRebase, @@ -340,13 +340,13 @@ func TestController_Process_StorageFailure(t *testing.T) { ctrl := gomock.NewController(t) mockReqStore := storagemock.NewMockRequestStore(ctrl) - mockReqStore.EXPECT().Get(gomock.Any(), "test-queue/123").Return(entity.Request{}, fmt.Errorf("db connection lost")) + mockReqStore.EXPECT().Get(gomock.Any(), "123").Return(entity.Request{}, fmt.Errorf("db connection lost")) store := orchstoragemock.NewMockStorage(ctrl) store.EXPECT().GetRequestStore().Return(mockReqStore).AnyTimes() controller := newTestController(t, ctrl, store, newMockChangeStore(ctrl), nil) - msg := entityqueue.NewMessage("test-queue/123", requestIDPayload(t, "test-queue/123"), "test-queue", nil) + msg := entityqueue.NewMessage("123", requestIDPayload(t, "123"), "test-queue", nil) msg.Tenant = "test-queue" delivery := consumermock.NewMockDelivery(ctrl) delivery.EXPECT().Message().Return(msg).AnyTimes() @@ -361,7 +361,7 @@ func TestController_Process_PublishFailure(t *testing.T) { ctrl := gomock.NewController(t) request := entity.Request{ - ID: "test-queue/123", + ID: "123", Queue: "test-queue", Change: change.Change{URIs: []string{"github://github.example.com/uber/service/pull/1/789abc1234567890abcdef1234567890abcdef12"}}, LandStrategy: mergestrategy.MergeStrategyRebase, @@ -384,7 +384,7 @@ func TestController_Process_PublishFailure(t *testing.T) { func TestController_InterfaceImplementation(t *testing.T) { ctrl := gomock.NewController(t) - request := entity.Request{ID: "test-queue/123", Queue: "test-queue"} + request := entity.Request{ID: "123", Queue: "test-queue"} store, _ := newMockStorage(ctrl, request) controller := newTestController(t, ctrl, store, newMockChangeStore(ctrl), nil) @@ -578,7 +578,7 @@ func TestController_Process_ChangeStoreQueryFailure(t *testing.T) { ctrl := gomock.NewController(t) request := entity.Request{ - ID: "test-queue/123", + ID: "123", Queue: "test-queue", Change: change.Change{URIs: []string{"github://github.example.com/uber/service/pull/1/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}}, LandStrategy: mergestrategy.MergeStrategyRebase, @@ -618,7 +618,7 @@ func TestController_Process_TerminalShortCircuit(t *testing.T) { ctrl := gomock.NewController(t) request := entity.Request{ - ID: "test-queue/123", + ID: "123", Queue: "test-queue", State: state, Version: 5, @@ -647,7 +647,7 @@ func TestController_Process_CustomValidatorPasses(t *testing.T) { ctrl := gomock.NewController(t) request := entity.Request{ - ID: "test-queue/123", + ID: "123", Queue: "test-queue", Change: change.Change{URIs: []string{"github://uber/service/pull/456/abcdef0123456789abcdef0123456789abcdef01"}}, LandStrategy: mergestrategy.MergeStrategyRebase, @@ -697,7 +697,7 @@ func TestController_Process_CustomValidatorFails(t *testing.T) { ctrl := gomock.NewController(t) request := entity.Request{ - ID: "test-queue/123", + ID: "123", Queue: "test-queue", Change: change.Change{URIs: []string{"github://uber/service/pull/456/abcdef0123456789abcdef0123456789abcdef01"}}, LandStrategy: mergestrategy.MergeStrategyRebase, @@ -762,7 +762,7 @@ func TestController_Process_CustomValidatorFailure_TerminationPublishFails(t *te ctrl := gomock.NewController(t) request := entity.Request{ - ID: "test-queue/123", + ID: "123", Queue: "test-queue", Change: change.Change{URIs: []string{"github://uber/service/pull/456/abcdef0123456789abcdef0123456789abcdef01"}}, LandStrategy: mergestrategy.MergeStrategyRebase, diff --git a/submitqueue/orchestrator/core/batch/find_test.go b/submitqueue/orchestrator/core/batch/find_test.go index cc0eec224..4706096ab 100644 --- a/submitqueue/orchestrator/core/batch/find_test.go +++ b/submitqueue/orchestrator/core/batch/find_test.go @@ -29,7 +29,7 @@ import ( orchstoragemock "github.com/uber/submitqueue/submitqueue/orchestrator/extension/storage/mock" ) -const testRequestID = "monorepo/4" +const testRequestID = "4" // association builds a RequestBatch linking testRequestID to a batch. func association(batchID string) entity.RequestBatch { diff --git a/submitqueue/orchestrator/core/batch/transition_test.go b/submitqueue/orchestrator/core/batch/transition_test.go index dee1ae0a4..9e41cad25 100644 --- a/submitqueue/orchestrator/core/batch/transition_test.go +++ b/submitqueue/orchestrator/core/batch/transition_test.go @@ -45,9 +45,9 @@ func testStores(t *testing.T) (*orchstoragemock.MockStorage, *storagemock.MockBa func TestTransition(t *testing.T) { base := entity.Batch{ - ID: "monorepo/batch/7", + ID: "7", Queue: "monorepo", - Contains: []string{"monorepo/1"}, + Contains: []string{"1"}, State: entity.BatchStateCreated, Version: 3, } @@ -146,7 +146,7 @@ func TestTransition(t *testing.T) { func TestEnsureRecord(t *testing.T) { batch := entity.Batch{ - ID: "monorepo/batch/7", + ID: "7", Queue: "monorepo", State: entity.BatchStateLanding, Version: 5, diff --git a/submitqueue/orchestrator/core/request/terminate_test.go b/submitqueue/orchestrator/core/request/terminate_test.go index 4db2e4f04..87b483189 100644 --- a/submitqueue/orchestrator/core/request/terminate_test.go +++ b/submitqueue/orchestrator/core/request/terminate_test.go @@ -65,7 +65,7 @@ func recordingRegistry(t *testing.T, ctrl *gomock.Controller, publishErr error) } func TestTerminateRequest(t *testing.T) { - const requestID = "q/1" + const requestID = "1" validated := entity.Request{ID: requestID, Queue: "q", State: entity.RequestStateValidated, Version: 3} originalValidated := validated @@ -101,7 +101,7 @@ func TestTerminateRequest(t *testing.T) { rs.EXPECT().Get(gomock.Any(), requestID).Return(entity.Request{}, fmt.Errorf("db down")) }, wantResult: TerminationResult{Outcome: TerminationOutcomeUnknown}, - errMsg: "failed to get request q/1: db down", + errMsg: "failed to get request 1: db down", }, "reconciled from non-terminal state": { targetState: entity.RequestStateError, @@ -147,7 +147,7 @@ func TestTerminateRequest(t *testing.T) { }, publishErr: fmt.Errorf("connection refused"), wantResult: TerminationResult{Outcome: TerminationOutcomeUnknown}, - errMsg: "failed to publish request log for q/1", + errMsg: "failed to publish request log for 1", }, "diverged terminal state is left untouched": { targetState: entity.RequestStateError, @@ -179,7 +179,7 @@ func TestTerminateRequest(t *testing.T) { }, publishErr: fmt.Errorf("connection refused"), wantResult: TerminationResult{Outcome: TerminationOutcomeUnknown}, - errMsg: "failed to publish request log for q/1", + errMsg: "failed to publish request log for 1", }, } diff --git a/submitqueue/orchestrator/extension/storage/mysql/batch_dependent_store_test.go b/submitqueue/orchestrator/extension/storage/mysql/batch_dependent_store_test.go index 84393da34..54be00a80 100644 --- a/submitqueue/orchestrator/extension/storage/mysql/batch_dependent_store_test.go +++ b/submitqueue/orchestrator/extension/storage/mysql/batch_dependent_store_test.go @@ -43,8 +43,8 @@ func setupBatchDependentStoreTest(t *testing.T) (*sql.DB, sqlmock.Sqlmock, stora func TestBatchDependentStore_Get(t *testing.T) { want := entity.BatchDependent{ - BatchID: "monorepo/batch/1", - Dependents: []string{"monorepo/batch/2", "monorepo/batch/3"}, + BatchID: "1", + Dependents: []string{"2", "3"}, Version: 1, } dependentsJSON, err := json.Marshal(want.Dependents) @@ -72,31 +72,31 @@ func TestBatchDependentStore_Get(t *testing.T) { }, { name: "found with nil dependents", - batchID: "monorepo/batch/nil", + batchID: "10", setup: func(mock sqlmock.Sqlmock) { rows := sqlmock.NewRows([]string{"batch_id", "dependents", "version"}). - AddRow("monorepo/batch/nil", []byte("null"), int32(2)) + AddRow("10", []byte("null"), int32(2)) mock.ExpectQuery("SELECT batch_id, dependents, version FROM batch_dependent"). - WithArgs("monorepo", "monorepo/batch/nil"). + WithArgs("monorepo", "10"). WillReturnRows(rows) }, want: entity.BatchDependent{ - BatchID: "monorepo/batch/nil", + BatchID: "10", Version: 2, }, }, { name: "found with empty dependents", - batchID: "monorepo/batch/empty", + batchID: "11", setup: func(mock sqlmock.Sqlmock) { rows := sqlmock.NewRows([]string{"batch_id", "dependents", "version"}). - AddRow("monorepo/batch/empty", []byte("[]"), int32(3)) + AddRow("11", []byte("[]"), int32(3)) mock.ExpectQuery("SELECT batch_id, dependents, version FROM batch_dependent"). - WithArgs("monorepo", "monorepo/batch/empty"). + WithArgs("monorepo", "11"). WillReturnRows(rows) }, want: entity.BatchDependent{ - BatchID: "monorepo/batch/empty", + BatchID: "11", Dependents: []string{}, Version: 3, }, @@ -148,8 +148,8 @@ func TestBatchDependentStore_Get(t *testing.T) { func TestBatchDependentStore_Create(t *testing.T) { bd := entity.BatchDependent{ - BatchID: "monorepo/batch/1", - Dependents: []string{"monorepo/batch/2"}, + BatchID: "1", + Dependents: []string{"2"}, Version: 1, } @@ -212,8 +212,8 @@ func TestBatchDependentStore_Create(t *testing.T) { func TestBatchDependentStore_Update(t *testing.T) { const oldVersion, newVersion = int32(1), int32(2) batchDependent := entity.BatchDependent{ - BatchID: "monorepo/batch/1", - Dependents: []string{"monorepo/batch/2", "monorepo/batch/3"}, + BatchID: "1", + Dependents: []string{"2", "3"}, Version: oldVersion, } @@ -229,32 +229,32 @@ func TestBatchDependentStore_Update(t *testing.T) { entity: batchDependent, setup: func(mock sqlmock.Sqlmock) { mock.ExpectExec("UPDATE batch_dependent"). - WithArgs([]byte(`["monorepo/batch/2","monorepo/batch/3"]`), newVersion, "monorepo", batchDependent.BatchID, oldVersion). + WithArgs([]byte(`["2","3"]`), newVersion, "monorepo", batchDependent.BatchID, oldVersion). WillReturnResult(sqlmock.NewResult(0, 1)) }, }, { name: "success with nil dependents", entity: entity.BatchDependent{ - BatchID: "monorepo/batch/nil", + BatchID: "10", Version: oldVersion, }, setup: func(mock sqlmock.Sqlmock) { mock.ExpectExec("UPDATE batch_dependent"). - WithArgs([]byte("null"), newVersion, "monorepo", "monorepo/batch/nil", oldVersion). + WithArgs([]byte("null"), newVersion, "monorepo", "10", oldVersion). WillReturnResult(sqlmock.NewResult(0, 1)) }, }, { name: "success with empty dependents", entity: entity.BatchDependent{ - BatchID: "monorepo/batch/empty", + BatchID: "11", Dependents: []string{}, Version: oldVersion, }, setup: func(mock sqlmock.Sqlmock) { mock.ExpectExec("UPDATE batch_dependent"). - WithArgs([]byte("[]"), newVersion, "monorepo", "monorepo/batch/empty", oldVersion). + WithArgs([]byte("[]"), newVersion, "monorepo", "11", oldVersion). WillReturnResult(sqlmock.NewResult(0, 1)) }, }, @@ -263,7 +263,7 @@ func TestBatchDependentStore_Update(t *testing.T) { entity: batchDependent, setup: func(mock sqlmock.Sqlmock) { mock.ExpectExec("UPDATE batch_dependent"). - WithArgs([]byte(`["monorepo/batch/2","monorepo/batch/3"]`), newVersion, "monorepo", batchDependent.BatchID, oldVersion). + WithArgs([]byte(`["2","3"]`), newVersion, "monorepo", batchDependent.BatchID, oldVersion). WillReturnResult(sqlmock.NewResult(0, 0)) }, wantErr: true, @@ -274,7 +274,7 @@ func TestBatchDependentStore_Update(t *testing.T) { entity: batchDependent, setup: func(mock sqlmock.Sqlmock) { mock.ExpectExec("UPDATE batch_dependent"). - WithArgs([]byte(`["monorepo/batch/2","monorepo/batch/3"]`), newVersion, "monorepo", batchDependent.BatchID, oldVersion). + WithArgs([]byte(`["2","3"]`), newVersion, "monorepo", batchDependent.BatchID, oldVersion). WillReturnError(fmt.Errorf("connection reset")) }, wantErr: true, @@ -284,7 +284,7 @@ func TestBatchDependentStore_Update(t *testing.T) { entity: batchDependent, setup: func(mock sqlmock.Sqlmock) { mock.ExpectExec("UPDATE batch_dependent"). - WithArgs([]byte(`["monorepo/batch/2","monorepo/batch/3"]`), newVersion, "monorepo", batchDependent.BatchID, oldVersion). + WithArgs([]byte(`["2","3"]`), newVersion, "monorepo", batchDependent.BatchID, oldVersion). WillReturnResult(sqlmock.NewErrorResult(fmt.Errorf("driver error"))) }, wantErr: true, diff --git a/submitqueue/orchestrator/extension/storage/mysql/batch_store_test.go b/submitqueue/orchestrator/extension/storage/mysql/batch_store_test.go index 145fd7620..89fd9992e 100644 --- a/submitqueue/orchestrator/extension/storage/mysql/batch_store_test.go +++ b/submitqueue/orchestrator/extension/storage/mysql/batch_store_test.go @@ -43,10 +43,10 @@ func setupBatchStoreTest(t *testing.T) (*sql.DB, sqlmock.Sqlmock, storage.BatchS func TestBatchStore_Get(t *testing.T) { want := entity.Batch{ - ID: "monorepo/batch/1", + ID: "1", Queue: "monorepo", - Contains: []string{"monorepo/1", "monorepo/2"}, - Dependencies: []string{"monorepo/batch/0"}, + Contains: []string{"1", "2"}, + Dependencies: []string{"0"}, State: entity.BatchStateCreated, Version: 1, } @@ -134,9 +134,9 @@ func TestBatchStore_Get(t *testing.T) { func TestBatchStore_Create(t *testing.T) { batch := entity.Batch{ - ID: "monorepo/batch/1", + ID: "1", Queue: "monorepo", - Contains: []string{"monorepo/1"}, + Contains: []string{"1"}, Dependencies: nil, State: entity.BatchStateCreated, Version: 1, @@ -201,10 +201,10 @@ func TestBatchStore_Create(t *testing.T) { func TestBatchStore_Update(t *testing.T) { const oldVersion, newVersion = int32(1), int32(2) batch := entity.Batch{ - ID: "monorepo/batch/1", + ID: "1", Queue: "monorepo", - Contains: []string{"monorepo/3", "monorepo/4"}, - Dependencies: []string{"monorepo/batch/1", "monorepo/batch/2"}, + Contains: []string{"3", "4"}, + Dependencies: []string{"1", "2"}, State: entity.BatchStateLanding, Version: oldVersion, } diff --git a/submitqueue/orchestrator/extension/storage/mysql/build_store_test.go b/submitqueue/orchestrator/extension/storage/mysql/build_store_test.go index a3be9d5d5..dee62ec0e 100644 --- a/submitqueue/orchestrator/extension/storage/mysql/build_store_test.go +++ b/submitqueue/orchestrator/extension/storage/mysql/build_store_test.go @@ -43,7 +43,7 @@ func setupBuildStoreTest(t *testing.T) (*sql.DB, sqlmock.Sqlmock, storage.BuildS func TestBuildStore_Get(t *testing.T) { want := entity.Build{ ID: "bk-1001", - BatchID: "monorepo/batch/1", + BatchID: "1", PathID: "path-1", Attempt: 1, Status: entity.BuildStatusRunning, @@ -117,7 +117,7 @@ func TestBuildStore_Get(t *testing.T) { func TestBuildStore_Create(t *testing.T) { build := entity.Build{ ID: "bk-1001", - BatchID: "monorepo/batch/1", + BatchID: "1", PathID: "path-1", Attempt: 1, Status: entity.BuildStatusAccepted, @@ -182,7 +182,7 @@ func TestBuildStore_Create(t *testing.T) { func TestBuildStore_Update(t *testing.T) { build := entity.Build{ ID: "bk-1001", - BatchID: "monorepo/batch/2", + BatchID: "2", Status: entity.BuildStatusSucceeded, } diff --git a/submitqueue/orchestrator/extension/storage/mysql/change_store_test.go b/submitqueue/orchestrator/extension/storage/mysql/change_store_test.go index cf686a171..f3b507e9f 100644 --- a/submitqueue/orchestrator/extension/storage/mysql/change_store_test.go +++ b/submitqueue/orchestrator/extension/storage/mysql/change_store_test.go @@ -42,7 +42,7 @@ func setupChangeStoreTest(t *testing.T) (*sql.DB, sqlmock.Sqlmock, storage.Chang func TestChangeStore_Create(t *testing.T) { record := entity.ChangeRecord{ URI: "github://github.example.com/uber/submitqueue/pull/123/deadbeef", - RequestID: "monorepo/1", + RequestID: "1", Queue: "monorepo", CreatedAt: 1000, UpdatedAt: 1000, @@ -102,7 +102,7 @@ func TestChangeStore_Create(t *testing.T) { func TestChangeStore_GetByURI(t *testing.T) { record := entity.ChangeRecord{ URI: "github://github.example.com/uber/submitqueue/pull/123/deadbeef", - RequestID: "monorepo/1", + RequestID: "1", Queue: "monorepo", CreatedAt: 1000, UpdatedAt: 1000, diff --git a/submitqueue/orchestrator/extension/storage/mysql/queue_batch_state_store_test.go b/submitqueue/orchestrator/extension/storage/mysql/queue_batch_state_store_test.go index 6bca7bd1f..e37a1bc1e 100644 --- a/submitqueue/orchestrator/extension/storage/mysql/queue_batch_state_store_test.go +++ b/submitqueue/orchestrator/extension/storage/mysql/queue_batch_state_store_test.go @@ -41,12 +41,12 @@ func TestQueueBatchStateStore_List(t *testing.T) { record1 := entity.QueueBatchState{ Queue: "monorepo", State: entity.BatchStateSpeculating, - BatchID: "monorepo/batch/1", + BatchID: "1", } record2 := entity.QueueBatchState{ Queue: record1.Queue, State: record1.State, - BatchID: "monorepo/batch/2", + BatchID: "2", } storeErr := errors.New("storage failed") tests := map[string]struct { @@ -115,7 +115,7 @@ func TestQueueBatchStateStore_Put(t *testing.T) { record := entity.QueueBatchState{ Queue: "monorepo", State: entity.BatchStateCreated, - BatchID: "monorepo/batch/1", + BatchID: "1", } tests := map[string]struct { setup func(sqlmock.Sqlmock) @@ -167,7 +167,7 @@ func TestQueueBatchStateStore_Delete(t *testing.T) { record := entity.QueueBatchState{ Queue: "monorepo", State: entity.BatchStateSucceeded, - BatchID: "monorepo/batch/1", + BatchID: "1", } tests := map[string]struct { setup func(sqlmock.Sqlmock) diff --git a/submitqueue/orchestrator/extension/storage/mysql/request_batch_store_test.go b/submitqueue/orchestrator/extension/storage/mysql/request_batch_store_test.go index 9496a3aac..db03d0e8d 100644 --- a/submitqueue/orchestrator/extension/storage/mysql/request_batch_store_test.go +++ b/submitqueue/orchestrator/extension/storage/mysql/request_batch_store_test.go @@ -40,13 +40,13 @@ func setupRequestBatchStoreTest(t *testing.T) (*sql.DB, sqlmock.Sqlmock, storage func TestRequestBatchStore_GetByRequestID(t *testing.T) { association1 := entity.RequestBatch{ - RequestID: "monorepo/1", - BatchID: "monorepo/batch/1", + RequestID: "1", + BatchID: "1", Version: 1, } association2 := entity.RequestBatch{ RequestID: association1.RequestID, - BatchID: "monorepo/batch/2", + BatchID: "2", Version: 1, } storeErr := errors.New("storage failed") @@ -124,8 +124,8 @@ func TestRequestBatchStore_GetByRequestID(t *testing.T) { func TestRequestBatchStore_Create(t *testing.T) { association := entity.RequestBatch{ - RequestID: "monorepo/1", - BatchID: "monorepo/batch/2", + RequestID: "1", + BatchID: "2", Version: 1, } tests := map[string]struct { diff --git a/submitqueue/orchestrator/extension/storage/mysql/request_store_test.go b/submitqueue/orchestrator/extension/storage/mysql/request_store_test.go index 80c05a59d..76e9bdca2 100644 --- a/submitqueue/orchestrator/extension/storage/mysql/request_store_test.go +++ b/submitqueue/orchestrator/extension/storage/mysql/request_store_test.go @@ -45,7 +45,7 @@ func setupRequestStoreTest(t *testing.T) (*sql.DB, sqlmock.Sqlmock, storage.Requ func TestRequestStore_Get(t *testing.T) { want := entity.Request{ - ID: "monorepo/1", + ID: "1", Queue: "monorepo", Change: change.Change{URIs: []string{"github://github.example.com/uber/submitqueue/pull/123/deadbeef"}}, LandStrategy: mergestrategy.MergeStrategyRebase, @@ -134,7 +134,7 @@ func TestRequestStore_Get(t *testing.T) { func TestRequestStore_Create(t *testing.T) { request := entity.Request{ - ID: "monorepo/1", + ID: "1", Queue: "monorepo", Change: change.Change{URIs: []string{"github://github.example.com/uber/submitqueue/pull/123/deadbeef"}}, LandStrategy: mergestrategy.MergeStrategyRebase, @@ -201,7 +201,7 @@ func TestRequestStore_Create(t *testing.T) { func TestRequestStore_Update(t *testing.T) { const oldVersion, newVersion = int32(1), int32(2) request := entity.Request{ - ID: "monorepo/1", + ID: "1", Queue: "monorepo", Change: change.Change{URIs: []string{"github://github.example.com/uber/submitqueue/pull/456/cafebabe"}}, LandStrategy: mergestrategy.MergeStrategySquashRebase, diff --git a/submitqueue/orchestrator/extension/storage/mysql/speculation_path_set_store_test.go b/submitqueue/orchestrator/extension/storage/mysql/speculation_path_set_store_test.go index d5c6588d3..2c038ec0a 100644 --- a/submitqueue/orchestrator/extension/storage/mysql/speculation_path_set_store_test.go +++ b/submitqueue/orchestrator/extension/storage/mysql/speculation_path_set_store_test.go @@ -47,7 +47,7 @@ func setupSpeculationPathSetStoreTest(t *testing.T) (*sql.DB, sqlmock.Sqlmock, s // testPathSet returns a two-path set for the given head: one assuming its // dependency succeeds, one assuming it fails. func testPathSet(head string) entity.SpeculationPathSet { - const dep = "monorepo/batch/1" + const dep = "1" succeeds := entity.SpeculationPath{ Head: head, Dependencies: []entity.PathDependency{{Batch: dep, Assumption: entity.DependencyAssumptionSucceeds}}, @@ -84,7 +84,7 @@ func testPathSet(head string) entity.SpeculationPathSet { } func TestSpeculationPathSetStore_Get(t *testing.T) { - want := testPathSet("monorepo/batch/2") + want := testPathSet("2") pathsJSON, err := json.Marshal(want.Paths) require.NoError(t, err) @@ -166,8 +166,8 @@ func TestSpeculationPathSetStore_Get(t *testing.T) { } func TestSpeculationPathSetStore_Create(t *testing.T) { - set := testPathSet("monorepo/batch/2") - otherQueueSet := testPathSet("monorepo/batch/2") + set := testPathSet("2") + otherQueueSet := testPathSet("2") otherQueueSet.Queue = "other-queue" tests := []struct { @@ -238,8 +238,8 @@ func TestSpeculationPathSetStore_Create(t *testing.T) { func TestSpeculationPathSetStore_Update(t *testing.T) { const oldVersion, newVersion = int32(3), int32(4) - set := testPathSet("monorepo/batch/2") - otherQueueSet := testPathSet("monorepo/batch/2") + set := testPathSet("2") + otherQueueSet := testPathSet("2") otherQueueSet.Queue = "other-queue" tests := []struct { @@ -326,7 +326,7 @@ func TestSpeculationPathSetStore_UpdateIgnoresEntityVersion(t *testing.T) { db, mock, store := setupSpeculationPathSetStoreTest(t) defer db.Close() - set := testPathSet("monorepo/batch/2") + set := testPathSet("2") set.Version = 99 // deliberately disagrees with both arguments const oldVersion, newVersion = int32(3), int32(4) diff --git a/test/e2e/runway/harness_test.go b/test/e2e/runway/harness_test.go index c31812bb8..e81330260 100644 --- a/test/e2e/runway/harness_test.go +++ b/test/e2e/runway/harness_test.go @@ -183,7 +183,7 @@ func (s *RunwayE2ESuite) publishRaw(topic, id, partitionKey string, payload []by func (s *RunwayE2ESuite) mergeRequest(queue string, steps ...*runwaymq.MergeStep) *runwaymq.MergeRequest { s.seq++ return &runwaymq.MergeRequest{ - Id: fmt.Sprintf("%s/%d", queue, s.seq), + Id: fmt.Sprintf("%d", s.seq), QueueName: queue, Steps: steps, } diff --git a/test/e2e/stovepipe/BUILD.bazel b/test/e2e/stovepipe/BUILD.bazel index 75e7ae96a..155472721 100644 --- a/test/e2e/stovepipe/BUILD.bazel +++ b/test/e2e/stovepipe/BUILD.bazel @@ -7,6 +7,7 @@ go_test( "suite_test.go", ], data = [ + "//platform/extension/counter/mysql/schema", "//platform/extension/messagequeue/mysql/schema", "//service/stovepipe:docker-compose.yml", "//service/stovepipe/server:docker_test_context", diff --git a/test/e2e/stovepipe/harness_test.go b/test/e2e/stovepipe/harness_test.go index 6d9be5757..81173a41a 100644 --- a/test/e2e/stovepipe/harness_test.go +++ b/test/e2e/stovepipe/harness_test.go @@ -66,12 +66,12 @@ func (s *StovepipeE2ESuite) ingest(queue string) string { return resp.Id } -// requestRowCount returns the number of request rows with the given id (0 or 1). -func (s *StovepipeE2ESuite) requestRowCount(id string) int { +// requestRowCount returns the number of request rows with the given queue and id (0 or 1). +func (s *StovepipeE2ESuite) requestRowCount(queue, id string) int { t := s.T() var count int - require.NoError(t, s.db.QueryRow("SELECT COUNT(*) FROM request WHERE id = ?", id).Scan(&count), - "failed to count request rows for %s", id) + require.NoError(t, s.db.QueryRow("SELECT COUNT(*) FROM request WHERE queue = ? AND id = ?", queue, id).Scan(&count), + "failed to count request rows for queue=%s id=%s", queue, id) return count } @@ -89,7 +89,7 @@ func (s *StovepipeE2ESuite) uriMapping(queue string) string { func (s *StovepipeE2ESuite) publishedMessageCount(tenant, id string) int { t := s.T() var count int - require.NoError(t, s.queueDB.QueryRow("SELECT COUNT(*) FROM queue_messages WHERE tenant = ? AND id = ?", tenant, id).Scan(&count), + require.NoError(t, s.queueDB.QueryRow("SELECT COUNT(*) FROM queue_messages WHERE tenant = ? AND topic = ? AND id = ?", tenant, processTopic, id).Scan(&count), "failed to count queue messages for %s", id) return count } @@ -126,7 +126,7 @@ func (s *StovepipeE2ESuite) awaitProcessed(queue string) { // and exactly one published process message. func (s *StovepipeE2ESuite) assertIngestPersisted(queue, id string) { t := s.T() - assert.Equal(t, 1, s.requestRowCount(id), "request row should be persisted for %s", id) + assert.Equal(t, 1, s.requestRowCount(queue, id), "request row should be persisted for queue=%s id=%s", queue, id) assert.Equal(t, id, s.uriMapping(queue), "URI mapping should point at the minted request id") assert.Equal(t, 1, s.publishedMessageCount(queue, id), "should have published one process message for %s", id) } @@ -134,14 +134,14 @@ func (s *StovepipeE2ESuite) assertIngestPersisted(queue, id string) { // awaitRequestState blocks until the request row reaches want. buildsignal projects // the build's terminal status onto the request, so this is the durable, black-box // signal that the whole ingest→process→build→buildsignal chain converged. -func (s *StovepipeE2ESuite) awaitRequestState(id, want string) { +func (s *StovepipeE2ESuite) awaitRequestState(queue, id, want string) { pollUntil(processPollInterval, func() bool { var state string - if err := s.db.QueryRow("SELECT state FROM request WHERE id = ?", id).Scan(&state); err != nil { - s.log.Logf("request %s state not readable yet: %v", id, err) + if err := s.db.QueryRow("SELECT state FROM request WHERE queue = ? AND id = ?", queue, id).Scan(&state); err != nil { + s.log.Logf("request queue=%s id=%s state not readable yet: %v", queue, id, err) return false } - s.log.Logf("request %s state = %q (want %q)", id, state, want) + s.log.Logf("request queue=%s id=%s state = %q (want %q)", queue, id, state, want) return state == want }) } @@ -160,16 +160,16 @@ func (s *StovepipeE2ESuite) inFlightCount(queue string) int32 { // pipeline runs inside the stovepipe-service container, so the build's own status // column is the durable, black-box signal that the poll loop converged: buildsignal // is its only writer after build creates the row. -func (s *StovepipeE2ESuite) awaitBuildStatus(requestID, want string) { +func (s *StovepipeE2ESuite) awaitBuildStatus(queue, requestID, want string) { pollUntil(processPollInterval, func() bool { var status string - err := s.db.QueryRow("SELECT status FROM build WHERE request_id = ?", requestID).Scan(&status) + err := s.db.QueryRow("SELECT status FROM build WHERE queue = ? AND request_id = ?", queue, requestID).Scan(&status) if err != nil { // sql.ErrNoRows means the build row is not created yet. - s.log.Logf("build for %s not readable yet: %v", requestID, err) + s.log.Logf("build for queue=%s request=%s not readable yet: %v", queue, requestID, err) return false } - s.log.Logf("build for %s status = %q (want %q)", requestID, status, want) + s.log.Logf("build for queue=%s request=%s status = %q (want %q)", queue, requestID, status, want) return status == want }) } diff --git a/test/e2e/stovepipe/suite_test.go b/test/e2e/stovepipe/suite_test.go index c8084e408..65c3d77c6 100644 --- a/test/e2e/stovepipe/suite_test.go +++ b/test/e2e/stovepipe/suite_test.go @@ -101,6 +101,7 @@ func (s *StovepipeE2ESuite) SetupSuite() { // Apply schemas after the stack is up; the service connects lazily and the // consumer retries, so the boot ordering is tolerated. + testutil.ApplySchema(t, s.log, s.db, testutil.SchemaDir("platform/extension/counter/mysql/schema")) testutil.ApplySchema(t, s.log, s.db, testutil.SchemaDir("stovepipe/extension/storage/mysql/schema")) testutil.ApplySchema(t, s.log, s.queueDB, testutil.SchemaDir("platform/extension/messagequeue/mysql/schema")) @@ -159,6 +160,21 @@ func (s *StovepipeE2ESuite) TestIngest_Idempotent() { assert.Equal(s.T(), id, id2, "re-ingest of the same head should dedup to the same id") } +func (s *StovepipeE2ESuite) TestIngest_IndependentQueuesReuseIDs() { + queues := []string{"monorepo/main", "monorepo/release"} + ids := []string{s.ingest(queues[0]), s.ingest(queues[1])} + require.Equal(s.T(), ids[0], ids[1], "independent queues may share a resource ID") + + for i, queue := range queues { + s.Run(queue, func() { + assert.Equal(s.T(), 1, s.requestRowCount(queue, ids[i])) + assert.Equal(s.T(), ids[i], s.uriMapping(queue)) + s.awaitBuildStatus(queue, ids[i], "succeeded") + s.awaitRequestState(queue, ids[i], "succeeded") + }) + } +} + func (s *StovepipeE2ESuite) TestIngest_RejectsUnconfiguredTenant() { resp, err := s.client.Ingest(s.ctx, &pb.IngestRequest{Queue: "monorepo/unconfigured"}) require.Error(s.T(), err) @@ -186,11 +202,11 @@ func (s *StovepipeE2ESuite) TestIngest_SlowBuild_PollsToCompletion() { s.assertIngestPersisted(queue, id) // Getting here at all means the held delivery redelivered and re-polled. - s.awaitBuildStatus(id, "succeeded") + s.awaitBuildStatus(queue, id, "succeeded") // buildsignal projects the terminal build status onto the request and, in the // same step, releases the build slot that reopens the process gate. - s.awaitRequestState(id, "succeeded") + s.awaitRequestState(queue, id, "succeeded") assert.Equal(s.T(), int32(0), s.inFlightCount(queue), "a terminal build should release the queue's build slot") } diff --git a/test/e2e/submitqueue/suite_test.go b/test/e2e/submitqueue/suite_test.go index 98335bca3..26fa6a48b 100644 --- a/test/e2e/submitqueue/suite_test.go +++ b/test/e2e/submitqueue/suite_test.go @@ -223,7 +223,7 @@ func (s *E2EIntegrationSuite) TestConflictAnalyzerFailure_ReconcilesFromDLQ() { t := s.T() const ( queue = "e2e-conflict-error-queue" - batchID = queue + "/batch/1" + batchID = "1" ) req := s.land(queue, "github://github.example.com/uber/e2e-conflict-error/pull/1/abcdef0123456789abcdef0123456789abcdef01") @@ -644,14 +644,14 @@ func (s *E2EIntegrationSuite) TestReadAPIs() { // leader resolves. // // The wait is forced rather than raced. Batch IDs come from a per-queue counter -// as "/batch/", so the leader on a fresh queue is batch/1, and the +// as decimal strings, so the leader on a fresh queue is batch 1, and the // build topic partitions by batch — closing the gate on that partition before // anything is published holds the leader's build and nothing else, so the // follower reaches a passed path while its dependency is still outstanding. func (s *E2EIntegrationSuite) TestLand_DependentBatch_BypassesAnUnresolvedDependency() { const queue = "e2e-respeculate-queue" const gateGroup = "orchestrator" - leaderBatch := queue + "/batch/1" + leaderBatch := "1" s.closeGate(queue, gateGroup, leaderBatch, "e2e: hold the leader's build so its dependent speculates first") defer s.openGate(queue, gateGroup, leaderBatch) @@ -791,7 +791,7 @@ func (s *E2EIntegrationSuite) TestBatchRedelivery_DoesNotEnrolTheRequestTwice() const gateGroup = "orchestrator" // Nothing has landed on this queue, so the first batch is predictable, and // the build topic partitions by batch ID. - const heldBatch = queue + "/batch/1" + const heldBatch = "1" s.closeGate(queue, gateGroup, heldBatch, "e2e: hold the build so the request stays in flight for the redelivery") // Reopen even if an assertion below fails, so teardown does not stop the @@ -833,7 +833,7 @@ func (s *E2EIntegrationSuite) TestStrandedBatch_IsAdmittedByALaterRun() { const queue = "e2e-strand-queue" const gateGroup = "orchestrator" - const heldBatch = queue + "/batch/1" + const heldBatch = "1" s.closeGate(queue, gateGroup, heldBatch, "e2e: hold the build so the batch can be stranded while still in flight") defer s.openGate(queue, gateGroup, heldBatch) diff --git a/test/integration/extension/counter/mysql/counter_test.go b/test/integration/extension/counter/mysql/counter_test.go index 532a6331e..a007cafd0 100644 --- a/test/integration/extension/counter/mysql/counter_test.go +++ b/test/integration/extension/counter/mysql/counter_test.go @@ -105,3 +105,20 @@ func (s *MySQLCounterIntegrationSuite) TearDownSuite() { s.log.Logf("Tearing down MySQL Counter integration test suite") // Cleanup handled automatically by testutil.ComposeStack } + +func (s *MySQLCounterIntegrationSuite) TestCounter_Restart() { + const ( + queue = "restart" + domain = "request" + ) + + originalCounter := mysqlcounter.NewCounter(s.db, tally.NoopScope, queue) + firstID, err := originalCounter.Next(context.Background(), domain) + require.NoError(s.T(), err) + require.Equal(s.T(), int64(1), firstID) + + reopenedCounter := mysqlcounter.NewCounter(s.db, tally.NoopScope, queue) + nextID, err := reopenedCounter.Next(context.Background(), domain) + require.NoError(s.T(), err) + require.Equal(s.T(), firstID+1, nextID) +} diff --git a/test/integration/extension/counter/suite.go b/test/integration/extension/counter/suite.go index be42e12f9..35440adae 100644 --- a/test/integration/extension/counter/suite.go +++ b/test/integration/extension/counter/suite.go @@ -84,7 +84,7 @@ func (s *CounterContractSuite) TestCounter_Next() { assert.Equal(t, seq2+1, seq3, "sequence should continue incrementing") } -// TestCounter_MultipleDomains tests independent counters +// TestCounter_MultipleDomains tests independent counters. func (s *CounterContractSuite) TestCounter_MultipleDomains() { t := s.T() ctx := s.ctx @@ -92,7 +92,7 @@ func (s *CounterContractSuite) TestCounter_MultipleDomains() { domain1 := "test-counter-1" domain2 := "test-counter-2" - // Get sequences from both domains + // Get sequences from both domains. seq1a, err := s.counter.Next(ctx, domain1) require.NoError(t, err) @@ -105,9 +105,9 @@ func (s *CounterContractSuite) TestCounter_MultipleDomains() { seq2b, err := s.counter.Next(ctx, domain2) require.NoError(t, err) - // Each domain should increment independently - assert.Equal(t, seq1a+1, seq1b, "domain1 should increment") - assert.Equal(t, seq2a+1, seq2b, "domain2 should increment") + // Each domain should increment independently. + assert.Equal(t, seq1a+1, seq1b, "domain 1 should increment") + assert.Equal(t, seq2a+1, seq2b, "domain 2 should increment") } // TestCounter_Concurrency tests concurrent access to the same counter diff --git a/test/integration/stovepipe/BUILD.bazel b/test/integration/stovepipe/BUILD.bazel index 320fca6cf..3eefca4a6 100644 --- a/test/integration/stovepipe/BUILD.bazel +++ b/test/integration/stovepipe/BUILD.bazel @@ -4,6 +4,7 @@ go_test( name = "go_default_test", srcs = ["suite_test.go"], data = [ + "//platform/extension/counter/mysql/schema", "//platform/extension/messagequeue/mysql/schema", "//service/stovepipe:docker-compose.yml", "//service/stovepipe/server:docker_test_context", diff --git a/test/integration/stovepipe/extension/storage/mysql/storage_test.go b/test/integration/stovepipe/extension/storage/mysql/storage_test.go index ab24ebb3a..aa243abe2 100644 --- a/test/integration/stovepipe/extension/storage/mysql/storage_test.go +++ b/test/integration/stovepipe/extension/storage/mysql/storage_test.go @@ -140,7 +140,7 @@ type MySQLRequestSummaryStoreSuite struct { func (s *MySQLRequestSummaryStoreSuite) TestCreateGetAndUpdate() { summary := entity.RequestSummary{ - RequestID: "request/monorepo/main/summary", Queue: "monorepo/main", URI: "git://repo/head", + RequestID: "1", Queue: "monorepo/main", URI: "git://repo/head", State: entity.RequestStateAccepted, RequestVersion: 1, StateTimestampMs: 1000, Version: 1, } require.NoError(s.T(), s.store.Create(s.ctx, summary)) @@ -164,7 +164,7 @@ func (s *MySQLRequestSummaryStoreSuite) TestCreateGetAndUpdate() { } func (s *MySQLRequestSummaryStoreSuite) TestGetNotFound() { - _, err := s.store.Get(s.ctx, "request/monorepo/main/missing-summary") + _, err := s.store.Get(s.ctx, "999") require.True(s.T(), storage.IsNotFound(err)) } @@ -179,7 +179,7 @@ type MySQLRequestStoreSuite struct { func (s *MySQLRequestStoreSuite) TestCreateAndGet() { req := entity.Request{ - ID: "request/monorepo/main/1", + ID: "1", Queue: "monorepo/main", URI: "git://remote/monorepo/main/aaaa1111", State: entity.RequestStateAccepted, @@ -194,7 +194,7 @@ func (s *MySQLRequestStoreSuite) TestCreateAndGet() { func (s *MySQLRequestStoreSuite) TestCreateAndGetWithProcessFields() { req := entity.Request{ - ID: "request/monorepo/main/process-fields", + ID: "3", Queue: "monorepo/main", URI: "git://remote/monorepo/main/cccc3333", State: entity.RequestStateProcessing, @@ -210,13 +210,13 @@ func (s *MySQLRequestStoreSuite) TestCreateAndGetWithProcessFields() { } func (s *MySQLRequestStoreSuite) TestGetNotFound() { - _, err := s.store.Get(s.ctx, "request/monorepo/main/does-not-exist") + _, err := s.store.Get(s.ctx, "999") require.True(s.T(), storage.IsNotFound(err)) } func (s *MySQLRequestStoreSuite) TestUpdateCAS() { req := entity.Request{ - ID: "request/monorepo/main/update", + ID: "4", Queue: "monorepo/main", State: entity.RequestStateAccepted, Version: 1, @@ -243,14 +243,14 @@ func (s *MySQLRequestStoreSuite) TestUpdateCAS() { } func (s *MySQLRequestStoreSuite) TestUpdateNotFoundIsVersionMismatch() { - missing := entity.Request{ID: "request/monorepo/main/missing", Queue: "monorepo/main", State: entity.RequestStateAccepted} + missing := entity.Request{ID: "999", Queue: "monorepo/main", State: entity.RequestStateAccepted} err := s.store.Update(s.ctx, missing, 1, 2) require.ErrorIs(s.T(), err, storage.ErrVersionMismatch) } func (s *MySQLRequestStoreSuite) TestCreateDuplicateID() { req := entity.Request{ - ID: "request/monorepo/main/2", + ID: "2", Queue: "monorepo/main", State: entity.RequestStateAccepted, Version: 1, @@ -265,7 +265,7 @@ func (s *MySQLRequestStoreSuite) TestURIMappingCreateAndGet() { const ( queue = "monorepo/main" uri = "git://remote/monorepo/main/bbbb2222" - id = "request/monorepo/main/3" + id = "3" ) require.NoError(s.T(), s.uriStore.Create(s.ctx, uri, id)) @@ -284,10 +284,10 @@ func (s *MySQLRequestStoreSuite) TestURIMappingDuplicate() { queue = "monorepo/main" uri = "git://remote/monorepo/main/cccc3333" ) - require.NoError(s.T(), s.uriStore.Create(s.ctx, uri, "request/monorepo/main/4")) + require.NoError(s.T(), s.uriStore.Create(s.ctx, uri, "4")) // A second request claiming the same (queue, uri) is rejected — the dedup signal. - err := s.uriStore.Create(s.ctx, uri, "request/monorepo/main/5") + err := s.uriStore.Create(s.ctx, uri, "5") require.ErrorIs(s.T(), err, storage.ErrAlreadyExists) } @@ -297,16 +297,16 @@ func (s *MySQLRequestStoreSuite) TestURIMappingDistinctAcrossQueues() { require.NoError(s.T(), err) boundB, err := s.backend.For("queue-b") require.NoError(s.T(), err) - require.NoError(s.T(), boundA.GetRequestURIStore().Create(s.ctx, uri, "request/queue-a/1")) - require.NoError(s.T(), boundB.GetRequestURIStore().Create(s.ctx, uri, "request/queue-b/1")) + require.NoError(s.T(), boundA.GetRequestURIStore().Create(s.ctx, uri, "1")) + require.NoError(s.T(), boundB.GetRequestURIStore().Create(s.ctx, uri, "1")) idA, err := boundA.GetRequestURIStore().GetIDByURI(s.ctx, uri) require.NoError(s.T(), err) - require.Equal(s.T(), "request/queue-a/1", idA) + require.Equal(s.T(), "1", idA) idB, err := boundB.GetRequestURIStore().GetIDByURI(s.ctx, uri) require.NoError(s.T(), err) - require.Equal(s.T(), "request/queue-b/1", idB) + require.Equal(s.T(), "1", idB) // The other queue's mapping is invisible through this queue's binding. _, err = boundA.GetRequestURIStore().GetIDByURI(s.ctx, "git://remote/monorepo/shared/only-b") diff --git a/test/integration/stovepipe/extension/storage/suite.go b/test/integration/stovepipe/extension/storage/suite.go index 8e5665093..46ca2449e 100644 --- a/test/integration/stovepipe/extension/storage/suite.go +++ b/test/integration/stovepipe/extension/storage/suite.go @@ -89,8 +89,8 @@ func (s *QueueStoreContractSuite) TestQueueStore_CreateWithFields() { toCreate := entity.Queue{ Name: name, LastGreenURI: "git://remote/monorepo/main/green-bbbb", - LastGreenRequestID: "request/contract/defaults/98", - LatestRequestID: "request/contract/defaults/99", + LastGreenRequestID: "98", + LatestRequestID: "99", Version: 1, } require.NoError(t, s.storeFor(name).Create(s.ctx, toCreate)) @@ -105,13 +105,13 @@ func (s *QueueStoreContractSuite) TestQueueStore_CreateAlreadyExists() { t := s.T() const name = "contract/already-exists" - first := entity.Queue{Name: name, LatestRequestID: "request/contract/already-exists/3", Version: 1} + first := entity.Queue{Name: name, LatestRequestID: "3", Version: 1} require.NoError(t, s.storeFor(name).Create(s.ctx, first)) err := s.storeFor(name).Create(s.ctx, entity.Queue{ Name: name, LastGreenURI: "git://remote/monorepo/main/ignored-on-race", - LatestRequestID: "request/contract/already-exists/500", + LatestRequestID: "500", Version: 1, }) assert.ErrorIs(t, err, storage.ErrAlreadyExists) @@ -139,16 +139,16 @@ func (s *QueueStoreContractSuite) TestQueueStore_UpdateCAS() { updated := created updated.LastGreenURI = "git://remote/monorepo/main/green-cccc" - updated.LastGreenRequestID = "request/contract/update-cas/41" - updated.LatestRequestID = "request/contract/update-cas/42" + updated.LastGreenRequestID = "41" + updated.LatestRequestID = "42" updated.InFlightCount = 1 require.NoError(t, s.storeFor(name).Update(s.ctx, updated, 1, 2)) got, err := s.storeFor(name).Get(s.ctx, name) require.NoError(t, err) assert.Equal(t, updated.LastGreenURI, got.LastGreenURI) - assert.Equal(t, "request/contract/update-cas/41", got.LastGreenRequestID) - assert.Equal(t, "request/contract/update-cas/42", got.LatestRequestID) + assert.Equal(t, "41", got.LastGreenRequestID) + assert.Equal(t, "42", got.LatestRequestID) assert.Equal(t, int32(1), got.InFlightCount) assert.Equal(t, int32(2), got.Version) @@ -171,15 +171,15 @@ func (s *QueueStoreContractSuite) TestQueueStore_UpdateSequentialCAS() { require.NoError(t, s.storeFor(name).Create(s.ctx, entity.Queue{Name: name, Version: 1})) - v2 := entity.Queue{Name: name, LatestRequestID: "request/contract/sequential-cas/10", Version: 1} + v2 := entity.Queue{Name: name, LatestRequestID: "10", Version: 1} require.NoError(t, s.storeFor(name).Update(s.ctx, v2, 1, 2)) - v3 := entity.Queue{Name: name, LatestRequestID: "request/contract/sequential-cas/10", InFlightCount: 1, Version: 2} + v3 := entity.Queue{Name: name, LatestRequestID: "10", InFlightCount: 1, Version: 2} require.NoError(t, s.storeFor(name).Update(s.ctx, v3, 2, 3)) got, err := s.storeFor(name).Get(s.ctx, name) require.NoError(t, err) - assert.Equal(t, "request/contract/sequential-cas/10", got.LatestRequestID) + assert.Equal(t, "10", got.LatestRequestID) assert.Equal(t, int32(1), got.InFlightCount) assert.Equal(t, int32(3), got.Version) } @@ -224,7 +224,7 @@ func (s *RequestLogStoreContractSuite) entry(queue, requestID, id string, timest func (s *RequestLogStoreContractSuite) TestRequestLogStore_CreateAndGet() { const ( queue = "contract/history-create" - requestID = "request/contract/history-create/1" + requestID = "1" ) entry := entity.RequestLog{ ID: "fact/repository", @@ -249,7 +249,7 @@ func (s *RequestLogStoreContractSuite) TestRequestLogStore_CreateAndGet() { func (s *RequestLogStoreContractSuite) TestRequestLogStore_CreateAlreadyExists() { const ( queue = "contract/history-duplicate" - requestID = "request/contract/history-duplicate/1" + requestID = "1" ) store := s.storeFor(queue) entry := s.entry(queue, requestID, "state/1", 1735689600000, 1) @@ -265,7 +265,7 @@ func (s *RequestLogStoreContractSuite) TestRequestLogStore_CreateAlreadyExists() // TestRequestLogStore_GetNotFound verifies a missing stable identity returns ErrNotFound. func (s *RequestLogStoreContractSuite) TestRequestLogStore_GetNotFound() { - _, err := s.storeFor("contract/history-missing").Get(s.ctx, "request/missing/1", "state/1") + _, err := s.storeFor("contract/history-missing").Get(s.ctx, "1", "state/1") assert.True(s.T(), storage.IsNotFound(err)) } @@ -273,7 +273,7 @@ func (s *RequestLogStoreContractSuite) TestRequestLogStore_GetNotFound() { func (s *RequestLogStoreContractSuite) TestRequestLogStore_List() { const ( queue = "contract/history-list" - requestID = "request/contract/history-list/1" + requestID = "1" ) store := s.storeFor(queue) last := s.entry(queue, requestID, "state/z", 2000, 3) @@ -290,13 +290,13 @@ func (s *RequestLogStoreContractSuite) TestRequestLogStore_List() { // TestRequestLogStore_ListNotFound verifies a request with no retained records returns ErrNotFound. func (s *RequestLogStoreContractSuite) TestRequestLogStore_ListNotFound() { - _, err := s.storeFor("contract/history-list-missing").List(s.ctx, "request/shared/1") + _, err := s.storeFor("contract/history-list-missing").List(s.ctx, "1") assert.ErrorIs(s.T(), err, storage.ErrNotFound) } // TestRequestLogStore_QueueIsolation verifies identical request and entry IDs remain queue-scoped. func (s *RequestLogStoreContractSuite) TestRequestLogStore_QueueIsolation() { - const requestID = "request/shared/1" + const requestID = "1" entryA := s.entry("contract/history-a", requestID, "state/1", 1000, 1) entryB := s.entry("contract/history-b", requestID, "state/1", 2000, 1) require.NoError(s.T(), s.storeFor(entryA.Queue).Create(s.ctx, entryA)) @@ -345,7 +345,7 @@ func (s *BuildStoreContractSuite) TestBuildStore_CreateAndGet() { build := entity.Build{ ID: id, - RequestID: "request/contract/create/1", + RequestID: "1", Status: entity.BuildStatusAccepted, Version: 1, } @@ -365,7 +365,7 @@ func (s *BuildStoreContractSuite) TestBuildStore_CreateAlreadyExists() { first := entity.Build{ ID: id, - RequestID: "request/contract/already-exists/1", + RequestID: "1", Status: entity.BuildStatusAccepted, Version: 1, } @@ -373,7 +373,7 @@ func (s *BuildStoreContractSuite) TestBuildStore_CreateAlreadyExists() { err := s.buildStore.Create(s.ctx, entity.Build{ ID: id, - RequestID: "request/contract/already-exists/ignored-on-race", + RequestID: "2", Status: entity.BuildStatusRunning, Version: 1, }) @@ -403,7 +403,7 @@ func (s *BuildStoreContractSuite) TestBuildStore_UpdateCAS() { created := entity.Build{ ID: id, - RequestID: "request/contract/update-cas/1", + RequestID: "1", Status: entity.BuildStatusAccepted, Version: 1, } @@ -442,7 +442,7 @@ func (s *BuildStoreContractSuite) TestBuildStore_UpdateSequentialCAS() { require.NoError(t, s.buildStore.Create(s.ctx, entity.Build{ ID: id, - RequestID: "request/contract/sequential-cas/1", + RequestID: "1", Status: entity.BuildStatusAccepted, Version: 1, })) @@ -470,11 +470,11 @@ func (s *BuildStoreContractSuite) TestBuildStore_QueueIsolation() { ) require.NoError(t, s.buildStore.Create(s.ctx, entity.Build{ - ID: idA, RequestID: "request/contract/isolation-a/1", + ID: idA, RequestID: "1", Status: entity.BuildStatusAccepted, Version: 1, })) require.NoError(t, s.buildStore.Create(s.ctx, entity.Build{ - ID: idB, RequestID: "request/contract/isolation-b/1", + ID: idB, RequestID: "1", Status: entity.BuildStatusAccepted, Version: 1, })) @@ -482,7 +482,7 @@ func (s *BuildStoreContractSuite) TestBuildStore_QueueIsolation() { require.NoError(t, err) updatedA := entity.Build{ - ID: idA, RequestID: "request/contract/isolation-a/1", + ID: idA, RequestID: "1", Status: entity.BuildStatusRunning, Version: 1, } require.NoError(t, s.buildStore.Update(s.ctx, updatedA, 1, 2)) @@ -513,7 +513,7 @@ func (s *QueueStoreContractSuite) TestQueueStore_QueueIsolation() { updatedA := entity.Queue{ Name: nameA, LastGreenURI: "git://remote/monorepo/a/green", - LatestRequestID: "request/contract/isolation-a/7", + LatestRequestID: "7", InFlightCount: 2, Version: 1, } @@ -567,7 +567,7 @@ func (s *ValidationFactStoreContractSuite) TestValidationFactStore_CreateAndGet( fact := entity.ValidationFact{ URI: "git://remote/contract/fact-create/aaaa", Degree: entity.DegreeGreen, - RequestID: "request/contract/fact-create/1", + RequestID: "1", CreatedAt: 1735689600000, } require.NoError(t, s.storeFor(queue).Create(s.ctx, fact)) @@ -586,7 +586,7 @@ func (s *ValidationFactStoreContractSuite) TestValidationFactStore_CreateAlready first := entity.ValidationFact{ URI: "git://remote/contract/fact-exists/aaaa", Degree: entity.DegreeGreen, - RequestID: "request/contract/fact-exists/1", + RequestID: "1", CreatedAt: 1735689600000, } require.NoError(t, s.storeFor(queue).Create(s.ctx, first)) @@ -594,7 +594,7 @@ func (s *ValidationFactStoreContractSuite) TestValidationFactStore_CreateAlready err := s.storeFor(queue).Create(s.ctx, entity.ValidationFact{ URI: first.URI, Degree: entity.DegreeBroken, - RequestID: "request/contract/fact-exists/2", + RequestID: "2", CreatedAt: 1735689700000, }) assert.ErrorIs(t, err, storage.ErrAlreadyExists) @@ -623,14 +623,14 @@ func (s *ValidationFactStoreContractSuite) TestValidationFactStore_ProjectIsPart wholeRepo := entity.ValidationFact{ URI: uri, Degree: entity.DegreeGreen, - RequestID: "request/contract/fact-project/1", + RequestID: "1", CreatedAt: 1735689600000, } scoped := entity.ValidationFact{ URI: uri, Project: "billing", Degree: entity.DegreeBroken, - RequestID: "request/contract/fact-project/1", + RequestID: "1", CreatedAt: 1735689600000, } require.NoError(t, s.storeFor(queue).Create(s.ctx, wholeRepo)) @@ -658,7 +658,7 @@ func (s *ValidationFactStoreContractSuite) TestValidationFactStore_QueueIsolatio factA := entity.ValidationFact{ URI: uri, Degree: entity.DegreeGreen, - RequestID: "request/contract/fact-isolation-a/1", + RequestID: "1", CreatedAt: 1735689600000, } require.NoError(t, s.storeFor(queueA).Create(s.ctx, factA)) @@ -671,7 +671,7 @@ func (s *ValidationFactStoreContractSuite) TestValidationFactStore_QueueIsolatio factB := entity.ValidationFact{ URI: uri, Degree: entity.DegreeBroken, - RequestID: "request/contract/fact-isolation-b/1", + RequestID: "1", CreatedAt: 1735689700000, } require.NoError(t, s.storeFor(queueB).Create(s.ctx, factB)) diff --git a/test/integration/stovepipe/suite_test.go b/test/integration/stovepipe/suite_test.go index 1a891783f..e9402df70 100644 --- a/test/integration/stovepipe/suite_test.go +++ b/test/integration/stovepipe/suite_test.go @@ -83,6 +83,7 @@ func (s *StovepipeIntegrationSuite) SetupSuite() { // Apply schemas after the stack is up; the service connects lazily and the // consumer retries, so the boot ordering is tolerated. + testutil.ApplySchema(t, s.log, s.db, testutil.SchemaDir("platform/extension/counter/mysql/schema")) testutil.ApplySchema(t, s.log, s.db, testutil.SchemaDir("stovepipe/extension/storage/mysql/schema")) testutil.ApplySchema(t, s.log, s.queueDB, testutil.SchemaDir("platform/extension/messagequeue/mysql/schema")) @@ -122,7 +123,7 @@ func (s *StovepipeIntegrationSuite) TestIngestAPI() { // Request persisted. var reqCount int - require.NoError(t, s.db.QueryRow("SELECT COUNT(*) FROM request WHERE id = ?", id).Scan(&reqCount)) + require.NoError(t, s.db.QueryRow("SELECT COUNT(*) FROM request WHERE queue = ? AND id = ?", queue, id).Scan(&reqCount)) assert.Equal(t, 1, reqCount, "request row should be persisted") // (queue, URI) mapping persisted and points at the minted id. @@ -132,7 +133,7 @@ func (s *StovepipeIntegrationSuite) TestIngestAPI() { // Message published to the process topic. var msgCount int - require.NoError(t, s.queueDB.QueryRow("SELECT COUNT(*) FROM queue_messages WHERE id = ?", id).Scan(&msgCount)) + require.NoError(t, s.queueDB.QueryRow("SELECT COUNT(*) FROM queue_messages WHERE tenant = ? AND topic = ? AND id = ?", queue, "process", id).Scan(&msgCount)) assert.Equal(t, 1, msgCount, "should have published one process message") // Re-ingesting the same queue resolves the same head URI and dedups. @@ -158,7 +159,7 @@ func (s *StovepipeIntegrationSuite) TestRequestHistoryAPIs() { t := s.T() const ( queue = "history-api/main" - requestID = "request/history-api/main/7" + requestID = "7" uri = "git://history-api/main/abc123" ) @@ -181,7 +182,7 @@ func (s *StovepipeIntegrationSuite) TestRequestHistoryAPIs() { require.NoError(t, err) var requestRows int - require.NoError(t, s.db.QueryRow("SELECT COUNT(*) FROM request WHERE id = ?", requestID).Scan(&requestRows)) + require.NoError(t, s.db.QueryRow("SELECT COUNT(*) FROM request WHERE queue = ? AND id = ?", queue, requestID).Scan(&requestRows)) require.Zero(t, requestRows) byID, err := s.client.GetRequestHistoryByID(s.ctx, &pb.GetRequestHistoryByIDRequest{Queue: queue, RequestId: requestID}) @@ -199,11 +200,11 @@ func (s *StovepipeIntegrationSuite) TestRequestHistoryAbsence() { t := s.T() const ( queue = "history-api-absence/main" - mappedID = "request/history-api-absence/main/1" + mappedID = "1" mappedURI = "git://history-api-absence/main/mapped" - scopedID = "request/history-api-absence/main/2" + scopedID = "2" scopedURI = "git://history-api-absence/main/scoped" - missingID = "request/history-api-absence/main/missing" + missingID = "999" missingURI = "git://history-api-absence/main/missing" wrongQueue = "history-api-absence/other" ) diff --git a/test/integration/submitqueue/extension/storage/suite.go b/test/integration/submitqueue/extension/storage/suite.go index 23caca7bd..52050e092 100644 --- a/test/integration/submitqueue/extension/storage/suite.go +++ b/test/integration/submitqueue/extension/storage/suite.go @@ -269,8 +269,8 @@ func (s *StorageContractSuite) TestStorage_BatchDependentUpdate() { ctx := s.ctx batchDependent := entity.BatchDependent{ - BatchID: "test/batch-dependent-update", - Dependents: []string{"test/dependent/1"}, + BatchID: "1", + Dependents: []string{"2"}, Version: 1, } require.NoError(t, s.forQueue("test-queue").GetBatchDependentStore().Create(ctx, batchDependent)) @@ -295,7 +295,7 @@ func (s *StorageContractSuite) TestStorage_BatchDependentUpdate() { assert.Equal(t, int32(3), retrieved.Version) staleUpdate := retrieved - staleUpdate.Dependents = []string{"test/dependent/stale"} + staleUpdate.Dependents = []string{"3"} err = s.forQueue("test-queue").GetBatchDependentStore().Update(ctx, staleUpdate, 2, 4) assert.ErrorIs(t, err, storage.ErrVersionMismatch) @@ -311,10 +311,10 @@ func (s *StorageContractSuite) TestStorage_BatchUpdateReplacesAllNonKeyFields() ctx := s.ctx store := s.forQueue("batch-update").GetBatchStore() batch := entity.Batch{ - ID: "batch-update/batch/1", + ID: "1", Queue: "batch-update", Contains: []string{"batch-update/1"}, - Dependencies: []string{"batch-update/batch/0"}, + Dependencies: []string{"0"}, State: entity.BatchStateCreated, Version: 1, } @@ -370,10 +370,10 @@ func (s *StorageContractSuite) TestStorage_QueueBatchStateRecordLifecycle() { storeA := s.forQueue("qbs-queue-a").GetQueueBatchStateStore() storeB := s.forQueue("qbs-queue-b").GetQueueBatchStateStore() - created1 := entity.QueueBatchState{Queue: "qbs-queue-a", State: entity.BatchStateCreated, BatchID: "qbs-queue-a/batch/1"} - created2 := entity.QueueBatchState{Queue: "qbs-queue-a", State: entity.BatchStateCreated, BatchID: "qbs-queue-a/batch/2"} - speculating := entity.QueueBatchState{Queue: "qbs-queue-a", State: entity.BatchStateSpeculating, BatchID: "qbs-queue-a/batch/3"} - otherQueue := entity.QueueBatchState{Queue: "qbs-queue-b", State: entity.BatchStateCreated, BatchID: "qbs-queue-b/batch/1"} + created1 := entity.QueueBatchState{Queue: "qbs-queue-a", State: entity.BatchStateCreated, BatchID: "1"} + created2 := entity.QueueBatchState{Queue: "qbs-queue-a", State: entity.BatchStateCreated, BatchID: "2"} + speculating := entity.QueueBatchState{Queue: "qbs-queue-a", State: entity.BatchStateSpeculating, BatchID: "3"} + otherQueue := entity.QueueBatchState{Queue: "qbs-queue-b", State: entity.BatchStateCreated, BatchID: "1"} require.NoError(t, storeA.Put(ctx, created1)) require.NoError(t, storeA.Put(ctx, created2)) @@ -428,13 +428,13 @@ func (s *StorageContractSuite) TestStorage_QueueIsolation() { storeA := s.forQueue("iso-queue-a") storeB := s.forQueue("iso-queue-b") - request := entity.Request{ID: "iso-a/1", Queue: "iso-queue-a", State: entity.RequestStateStarted, LandStrategy: mergestrategy.MergeStrategyMerge, Version: 1} + request := entity.Request{ID: "1", Queue: "iso-queue-a", State: entity.RequestStateStarted, LandStrategy: mergestrategy.MergeStrategyMerge, Version: 1} require.NoError(t, storeA.GetRequestStore().Create(ctx, request)) _, err := storeB.GetRequestStore().Get(ctx, request.ID) require.ErrorIs(t, err, storage.ErrNotFound, "a request must be invisible through another queue's binding") require.Error(t, storeB.GetRequestStore().Create(ctx, request), "a mismatched-queue write must be rejected") - batch := entity.Batch{ID: "iso-a/batch/1", Queue: "iso-queue-a", State: entity.BatchStateCreated, Version: 1} + batch := entity.Batch{ID: "1", Queue: "iso-queue-a", State: entity.BatchStateCreated, Version: 1} require.NoError(t, storeA.GetBatchStore().Create(ctx, batch)) _, err = storeB.GetBatchStore().Get(ctx, batch.ID) require.ErrorIs(t, err, storage.ErrNotFound) @@ -519,7 +519,7 @@ func (s *StorageContractSuite) TestStorage_ChangeCreateAndGet_NoMatch() { const queue = "cq-nomatch" require.NoError(t, s.forQueue(queue).GetChangeStore().Create(ctx, entity.ChangeRecord{ - URI: changeURI, RequestID: queue + "/1", Queue: queue, CreatedAt: 1, UpdatedAt: 1, Version: 1, + URI: changeURI, RequestID: "1", Queue: queue, CreatedAt: 1, UpdatedAt: 1, Version: 1, })) got, err := s.forQueue(queue).GetChangeStore().GetByURI(ctx, "github://github.example.com/uber/x/pull/2/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") @@ -534,13 +534,13 @@ func (s *StorageContractSuite) TestStorage_ChangeCreateAndGet_Match() { const queue = "cq-match" require.NoError(t, s.forQueue(queue).GetChangeStore().Create(ctx, entity.ChangeRecord{ - URI: changeURI, RequestID: queue + "/1", Queue: queue, CreatedAt: 1, UpdatedAt: 1, Version: 1, + URI: changeURI, RequestID: "1", Queue: queue, CreatedAt: 1, UpdatedAt: 1, Version: 1, })) got, err := s.forQueue(queue).GetChangeStore().GetByURI(ctx, changeURI) require.NoError(t, err) require.Len(t, got, 1) - assert.Equal(t, queue+"/1", got[0].RequestID) + assert.Equal(t, "1", got[0].RequestID) assert.Equal(t, changeURI, got[0].URI) assert.Equal(t, queue, got[0].Queue) assert.Equal(t, int32(1), got[0].Version) @@ -553,13 +553,13 @@ func (s *StorageContractSuite) TestStorage_ChangeGetByURI_DoesNotExcludeSelf() { const queue = "cq-self" require.NoError(t, s.forQueue(queue).GetChangeStore().Create(ctx, entity.ChangeRecord{ - URI: changeURI, RequestID: queue + "/1", Queue: queue, CreatedAt: 1, UpdatedAt: 1, Version: 1, + URI: changeURI, RequestID: "1", Queue: queue, CreatedAt: 1, UpdatedAt: 1, Version: 1, })) got, err := s.forQueue(queue).GetChangeStore().GetByURI(ctx, changeURI) require.NoError(t, err) require.Len(t, got, 1, "store returns the row even when caller might consider it self") - assert.Equal(t, queue+"/1", got[0].RequestID) + assert.Equal(t, "1", got[0].RequestID) } // TestStorage_ChangeGetByURI_QueueScoped verifies GetByURI never returns rows from another queue. @@ -568,7 +568,7 @@ func (s *StorageContractSuite) TestStorage_ChangeGetByURI_QueueScoped() { ctx := s.ctx require.NoError(t, s.forQueue("cq-scoped-A").GetChangeStore().Create(ctx, entity.ChangeRecord{ - URI: changeURI, RequestID: "cq-scoped-A/1", Queue: "cq-scoped-A", CreatedAt: 1, UpdatedAt: 1, Version: 1, + URI: changeURI, RequestID: "1", Queue: "cq-scoped-A", CreatedAt: 1, UpdatedAt: 1, Version: 1, })) got, err := s.forQueue("cq-scoped-B").GetChangeStore().GetByURI(ctx, changeURI) @@ -581,7 +581,7 @@ func (s *StorageContractSuite) TestStorage_ChangeCreate_Idempotent() { t := s.T() ctx := s.ctx const queue = "cq-idem" - rec := entity.ChangeRecord{URI: changeURI, RequestID: queue + "/1", Queue: queue, CreatedAt: 1, UpdatedAt: 1, Version: 1} + rec := entity.ChangeRecord{URI: changeURI, RequestID: "1", Queue: queue, CreatedAt: 1, UpdatedAt: 1, Version: 1} require.NoError(t, s.forQueue(queue).GetChangeStore().Create(ctx, rec)) require.NoError(t, s.forQueue(queue).GetChangeStore().Create(ctx, rec), "second insert with same PK must succeed (INSERT IGNORE)") @@ -598,10 +598,10 @@ func (s *StorageContractSuite) TestStorage_ChangeCreate_DifferentRequestSameURI( const queue = "cq-multi" require.NoError(t, s.forQueue(queue).GetChangeStore().Create(ctx, entity.ChangeRecord{ - URI: changeURI, RequestID: queue + "/1", Queue: queue, CreatedAt: 1, UpdatedAt: 1, Version: 1, + URI: changeURI, RequestID: "1", Queue: queue, CreatedAt: 1, UpdatedAt: 1, Version: 1, })) require.NoError(t, s.forQueue(queue).GetChangeStore().Create(ctx, entity.ChangeRecord{ - URI: changeURI, RequestID: queue + "/2", Queue: queue, CreatedAt: 2, UpdatedAt: 2, Version: 1, + URI: changeURI, RequestID: "2", Queue: queue, CreatedAt: 2, UpdatedAt: 2, Version: 1, })) got, err := s.forQueue(queue).GetChangeStore().GetByURI(ctx, changeURI) @@ -610,7 +610,7 @@ func (s *StorageContractSuite) TestStorage_ChangeCreate_DifferentRequestSameURI( ids := []string{got[0].RequestID, got[1].RequestID} sort.Strings(ids) - assert.Equal(t, []string{queue + "/1", queue + "/2"}, ids) + assert.Equal(t, []string{"1", "2"}, ids) } // sampleDetails is a representative ChangeDetails reused across change-store contract tests. @@ -632,7 +632,7 @@ func (s *StorageContractSuite) TestStorage_ChangeCreate_PreservesDetails() { details := sampleDetails() require.NoError(t, s.forQueue(queue).GetChangeStore().Create(ctx, entity.ChangeRecord{ - URI: changeURI, RequestID: queue + "/1", Queue: queue, Details: details, CreatedAt: 1, UpdatedAt: 1, Version: 1, + URI: changeURI, RequestID: "1", Queue: queue, Details: details, CreatedAt: 1, UpdatedAt: 1, Version: 1, })) got, err := s.forQueue(queue).GetChangeStore().GetByURI(ctx, changeURI) @@ -648,7 +648,7 @@ func (s *StorageContractSuite) TestStorage_ChangeCreate_EmptyDetails() { const queue = "cq-emptydetails" require.NoError(t, s.forQueue(queue).GetChangeStore().Create(ctx, entity.ChangeRecord{ - URI: changeURI, RequestID: queue + "/1", Queue: queue, CreatedAt: 1, UpdatedAt: 1, Version: 1, + URI: changeURI, RequestID: "1", Queue: queue, CreatedAt: 1, UpdatedAt: 1, Version: 1, })) got, err := s.forQueue(queue).GetChangeStore().GetByURI(ctx, changeURI) @@ -686,7 +686,7 @@ func (s *StorageContractSuite) TestStorage_RequestSummaryCreateGetAndCAS() { ctx := s.ctx const queue = "summary-q" summary := entity.RequestSummary{ - RequestID: "summary/1", Queue: queue, ChangeURIs: nil, ReceivedAtMs: 100, + RequestID: "1", Queue: queue, ChangeURIs: nil, ReceivedAtMs: 100, Status: entity.RequestStatusAccepted, RequestVersion: 1, StatusTimestampMs: 100, Version: 1, LastError: "", Metadata: nil, } @@ -773,9 +773,9 @@ func (s *StorageContractSuite) TestStorage_RequestQueueSummaryListAndCursor() { ctx := s.ctx store := s.forGatewayQueue("queue-summary").GetRequestQueueSummaryStore() rows := []entity.RequestQueueSummary{ - {RequestID: "queue-summary/1", Queue: "queue-summary", ChangeURIs: nil, ReceivedAtMs: 100, Status: entity.RequestStatusAccepted, Version: 1, Metadata: nil}, - {RequestID: "queue-summary/2", Queue: "queue-summary", ChangeURIs: []string{"uri/2"}, ReceivedAtMs: 200, Status: entity.RequestStatusLanded, Version: 1, Metadata: map[string]string{}}, - {RequestID: "queue-summary/3", Queue: "queue-summary", ChangeURIs: []string{"uri/3"}, ReceivedAtMs: 200, Status: entity.RequestStatusError, Version: 1, Metadata: map[string]string{}}, + {RequestID: "1", Queue: "queue-summary", ChangeURIs: nil, ReceivedAtMs: 100, Status: entity.RequestStatusAccepted, Version: 1, Metadata: nil}, + {RequestID: "2", Queue: "queue-summary", ChangeURIs: []string{"uri/2"}, ReceivedAtMs: 200, Status: entity.RequestStatusLanded, Version: 1, Metadata: map[string]string{}}, + {RequestID: "3", Queue: "queue-summary", ChangeURIs: []string{"uri/3"}, ReceivedAtMs: 200, Status: entity.RequestStatusError, Version: 1, Metadata: map[string]string{}}, } for _, row := range rows { require.NoError(t, store.Create(ctx, row)) @@ -839,15 +839,15 @@ func (s *StorageContractSuite) TestStorage_RequestQueueSummaryListAndCursor() { }) require.NoError(t, err) require.Len(t, firstPage, 2) - assert.Equal(t, []string{"queue-summary/3", "queue-summary/2"}, []string{firstPage[0].RequestID, firstPage[1].RequestID}) + assert.Equal(t, []string{"3", "2"}, []string{firstPage[0].RequestID, firstPage[1].RequestID}) secondPage, err := store.List(ctx, storage.RequestQueueSummaryQuery{ ReceivedAtOrAfterMs: 50, ReceivedBeforeMs: 250, Limit: 2, - HasCursor: true, Cursor: storage.RequestQueueSummaryCursor{ReceivedAtMs: 200, RequestID: "queue-summary/2"}, + HasCursor: true, Cursor: storage.RequestQueueSummaryCursor{ReceivedAtMs: 200, RequestID: "2"}, }) require.NoError(t, err) require.Len(t, secondPage, 1) - assert.Equal(t, "queue-summary/1", secondPage[0].RequestID) + assert.Equal(t, "1", secondPage[0].RequestID) assert.NotNil(t, secondPage[0].ChangeURIs) assert.NotNil(t, secondPage[0].Metadata) @@ -856,7 +856,7 @@ func (s *StorageContractSuite) TestStorage_RequestQueueSummaryListAndCursor() { }) require.NoError(t, err) require.Len(t, bounded, 1) - assert.Equal(t, "queue-summary/1", bounded[0].RequestID) + assert.Equal(t, "1", bounded[0].RequestID) empty, err := store.List(ctx, storage.RequestQueueSummaryQuery{ ReceivedAtOrAfterMs: 300, ReceivedBeforeMs: 400, Limit: 10, @@ -871,9 +871,9 @@ func (s *StorageContractSuite) TestStorage_RequestURIListIsBoundedAndOrdered() { const queue = "uri-q" store := s.forGatewayQueue(queue).GetRequestURIStore() rows := []entity.RequestURI{ - {ChangeURI: "uri/shared", Queue: queue, ReceivedAtMs: 100, RequestID: "uri/1"}, - {ChangeURI: "uri/shared", Queue: queue, ReceivedAtMs: 200, RequestID: "uri/2"}, - {ChangeURI: "uri/shared", Queue: queue, ReceivedAtMs: 200, RequestID: "uri/3"}, + {ChangeURI: "uri/shared", Queue: queue, ReceivedAtMs: 100, RequestID: "1"}, + {ChangeURI: "uri/shared", Queue: queue, ReceivedAtMs: 200, RequestID: "2"}, + {ChangeURI: "uri/shared", Queue: queue, ReceivedAtMs: 200, RequestID: "3"}, } for _, row := range rows { require.NoError(t, store.Create(ctx, row)) @@ -883,7 +883,7 @@ func (s *StorageContractSuite) TestStorage_RequestURIListIsBoundedAndOrdered() { got, err := store.ListByURI(ctx, "uri/shared", 2) require.NoError(t, err) require.Len(t, got, 2) - assert.Equal(t, []string{"uri/3", "uri/2"}, []string{got[0].RequestID, got[1].RequestID}) + assert.Equal(t, []string{"3", "2"}, []string{got[0].RequestID, got[1].RequestID}) empty, err := store.ListByURI(ctx, "uri/missing", 2) require.NoError(t, err) @@ -892,12 +892,12 @@ func (s *StorageContractSuite) TestStorage_RequestURIListIsBoundedAndOrdered() { // The same change URI in another queue is a distinct mapping set. otherStore := s.forGatewayQueue("uri-q-other").GetRequestURIStore() require.NoError(t, otherStore.Create(ctx, entity.RequestURI{ - ChangeURI: "uri/shared", Queue: "uri-q-other", ReceivedAtMs: 100, RequestID: "other/1", + ChangeURI: "uri/shared", Queue: "uri-q-other", ReceivedAtMs: 100, RequestID: "1", }), "the same change URI in another queue is a distinct row") otherGot, err := otherStore.ListByURI(ctx, "uri/shared", 10) require.NoError(t, err) require.Len(t, otherGot, 1, "one queue's mappings must not surface through another's binding") - assert.Equal(t, "other/1", otherGot[0].RequestID) + assert.Equal(t, "1", otherGot[0].RequestID) } // TestStorage_RequestLogAppendAndList tests the append-only audit trail and its @@ -909,18 +909,18 @@ func (s *StorageContractSuite) TestStorage_RequestLogAppendAndList() { const queue = "log-q" store := s.forGatewayQueue(queue).GetRequestLogStore() - _, err := store.List(ctx, "log/missing") + _, err := store.List(ctx, "999") require.ErrorIs(t, err, storage.ErrNotFound) entries := []entity.RequestLog{ - {RequestID: "log/1", Queue: queue, TimestampMs: 100, Status: entity.RequestStatusAccepted, Metadata: map[string]string{}}, - {RequestID: "log/1", Queue: queue, TimestampMs: 200, Status: entity.RequestStatusStarted, LastError: "detail", Metadata: map[string]string{"k": "v"}}, + {RequestID: "1", Queue: queue, TimestampMs: 100, Status: entity.RequestStatusAccepted, Metadata: map[string]string{}}, + {RequestID: "1", Queue: queue, TimestampMs: 200, Status: entity.RequestStatusStarted, LastError: "detail", Metadata: map[string]string{"k": "v"}}, } for _, entry := range entries { require.NoError(t, store.Insert(ctx, entry)) } - got, err := store.List(ctx, "log/1") + got, err := store.List(ctx, "1") require.NoError(t, err) require.Len(t, got, 2) assert.Equal(t, entity.RequestStatusAccepted, got[0].Status, "entries come back in timestamp order") @@ -928,15 +928,15 @@ func (s *StorageContractSuite) TestStorage_RequestLogAppendAndList() { // A log carrying another queue's name is rejected by the binding. assert.Error(t, store.Insert(ctx, entity.RequestLog{ - RequestID: "log/1", Queue: "log-q-other", TimestampMs: 300, Status: entity.RequestStatusLanded, Metadata: map[string]string{}, + RequestID: "1", Queue: "log-q-other", TimestampMs: 300, Status: entity.RequestStatusLanded, Metadata: map[string]string{}, })) // The same request ID in another queue is an independent history. otherStore := s.forGatewayQueue("log-q-other").GetRequestLogStore() require.NoError(t, otherStore.Insert(ctx, entity.RequestLog{ - RequestID: "log/1", Queue: "log-q-other", TimestampMs: 150, Status: entity.RequestStatusLanded, Metadata: map[string]string{}, + RequestID: "1", Queue: "log-q-other", TimestampMs: 150, Status: entity.RequestStatusLanded, Metadata: map[string]string{}, })) - otherGot, err := otherStore.List(ctx, "log/1") + otherGot, err := otherStore.List(ctx, "1") require.NoError(t, err) require.Len(t, otherGot, 1, "one queue's log must not surface through another's binding") assert.Equal(t, entity.RequestStatusLanded, otherGot[0].Status) diff --git a/test/integration/submitqueue/gateway/suite_test.go b/test/integration/submitqueue/gateway/suite_test.go index a39f8e275..096190785 100644 --- a/test/integration/submitqueue/gateway/suite_test.go +++ b/test/integration/submitqueue/gateway/suite_test.go @@ -194,8 +194,8 @@ func (s *GatewayIntegrationSuite) TestListAPI() { queueStore, err := store.For("test-queue") require.NoError(t, err) for _, summary := range []entity.RequestSummary{ - {RequestID: "test-queue/list-1", Queue: "test-queue", ChangeURIs: []string{"uri/1"}, ReceivedAtMs: 100, Status: entity.RequestStatusAccepted, StatusTimestampMs: 100, Version: 1, Metadata: map[string]string{}}, - {RequestID: "test-queue/list-2", Queue: "test-queue", ChangeURIs: []string{"uri/2"}, ReceivedAtMs: 200, Status: entity.RequestStatusLanded, StatusTimestampMs: 200, Version: 1, Metadata: map[string]string{}}, + {RequestID: "901", Queue: "test-queue", ChangeURIs: []string{"uri/1"}, ReceivedAtMs: 100, Status: entity.RequestStatusAccepted, StatusTimestampMs: 100, Version: 1, Metadata: map[string]string{}}, + {RequestID: "902", Queue: "test-queue", ChangeURIs: []string{"uri/2"}, ReceivedAtMs: 200, Status: entity.RequestStatusLanded, StatusTimestampMs: 200, Version: 1, Metadata: map[string]string{}}, } { publicStatus := summary.Status summary.Status = entity.RequestStatusAccepting @@ -212,13 +212,13 @@ func (s *GatewayIntegrationSuite) TestListAPI() { resp, err := s.client.List(s.ctx, &pb.ListRequest{Queue: "test-queue", ReceivedAtOrAfterMs: 50, ReceivedBeforeMs: 250, PageSize: 1}) require.NoError(t, err) require.Len(t, resp.Requests, 1) - assert.Equal(t, "test-queue/list-2", resp.Requests[0].Sqid) + assert.Equal(t, "902", resp.Requests[0].Sqid) require.NotEmpty(t, resp.NextPageToken) resp, err = s.client.List(s.ctx, &pb.ListRequest{Queue: "test-queue", ReceivedAtOrAfterMs: 50, ReceivedBeforeMs: 250, PageSize: 1, PageToken: resp.NextPageToken}) require.NoError(t, err) require.Len(t, resp.Requests, 1) - assert.Equal(t, "test-queue/list-1", resp.Requests[0].Sqid) + assert.Equal(t, "901", resp.Requests[0].Sqid) assert.Empty(t, resp.NextPageToken) } @@ -226,7 +226,7 @@ func (s *GatewayIntegrationSuite) TestListAPI() { func (s *GatewayIntegrationSuite) TestReadAPIErrorCodes() { t := s.T() - _, err := s.client.GetRequestSummaryByID(s.ctx, &pb.GetRequestSummaryByIDRequest{Sqid: "missing/1", Queue: "missing"}) + _, err := s.client.GetRequestSummaryByID(s.ctx, &pb.GetRequestSummaryByIDRequest{Sqid: "1", Queue: "missing"}) require.Error(t, err) assert.Equal(t, codes.NotFound, status.Code(err)) @@ -248,7 +248,7 @@ func (s *GatewayIntegrationSuite) TestReadAPIErrorCodes() { ChangeURI: overflowChangeURI, Queue: "overflow", ReceivedAtMs: int64(i), - RequestID: fmt.Sprintf("overflow/%d", i), + RequestID: fmt.Sprintf("%d", i), })) } @@ -267,7 +267,7 @@ func (s *GatewayIntegrationSuite) TestReadAPIErrorCodes() { ChangeURI: inconsistentChangeURI, Queue: "missing-summary", ReceivedAtMs: 1, - RequestID: "missing-summary/1", + RequestID: "1", })) _, err = s.client.GetRequestSummaryByChangeURI(s.ctx, &pb.GetRequestSummaryByChangeURIRequest{ChangeUri: inconsistentChangeURI, Queue: "missing-summary"}) require.Error(t, err) @@ -282,7 +282,7 @@ func (s *GatewayIntegrationSuite) TestReadAPIErrorCodes() { // entry to storage, observable through the request-summary RPC. func (s *GatewayIntegrationSuite) TestRequestLogConsumer() { t := s.T() - const sqid = "test-queue/log-consumer-test" + const sqid = "900" const logQueue = "test-queue" // Build a publisher against the shared queue database. NewQueue only wires up @@ -321,6 +321,9 @@ func (s *GatewayIntegrationSuite) TestRequestLogConsumer() { defer ticker.Stop() for { resp, statusErr := s.client.GetRequestSummaryByID(s.ctx, &pb.GetRequestSummaryByIDRequest{Sqid: sqid, Queue: logQueue}) + if statusErr != nil { + require.Equal(t, codes.NotFound, status.Code(statusErr), "unexpected request-summary lookup error: %v", statusErr) + } if statusErr == nil && resp.Request != nil && resp.Request.Status == string(entity.RequestStatusStarted) { break }