From 44020528cb188a2f58993ec0fed91e658537850e Mon Sep 17 00:00:00 2001 From: planadecu Date: Wed, 30 Sep 2026 22:22:17 +0200 Subject: [PATCH 01/37] GN-159: consume extracted Glassnode Go API client --- .github/workflows/release.yml | 14 +++ .github/workflows/test.yml | 23 +++- README.md | 13 +++ docs/go-sdk-extraction.md | 28 +++++ go.mod | 1 + go.sum | 2 + internal/api/api_usage.go | 21 ++-- internal/api/api_usage_test.go | 9 +- internal/api/client.go | 205 ++++++++++++++++++--------------- internal/api/client_test.go | 18 ++- internal/api/metadata.go | 167 +++++---------------------- internal/api/metrics.go | 46 ++++++-- internal/output/output.go | 8 +- internal/output/output_test.go | 2 +- 14 files changed, 295 insertions(+), 262 deletions(-) create mode 100644 docs/go-sdk-extraction.md diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 665f59b..80948c8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,6 +11,8 @@ permissions: jobs: release: runs-on: ubuntu-latest + env: + GOPRIVATE: github.com/glassnode/glassnode-api-go-client steps: - uses: actions/checkout@v4 with: @@ -20,9 +22,21 @@ jobs: with: go-version: '1.24' + - name: Authenticate private SDK dependency + env: + GH_TOKEN: ${{ secrets.GLASSNODE_SDK_READ_TOKEN }} + run: | + if [ -z "$GH_TOKEN" ]; then + echo "GLASSNODE_SDK_READ_TOKEN must grant read access to glassnode/glassnode-api-go-client." + exit 1 + fi + gh auth setup-git + go mod download + - uses: goreleaser/goreleaser-action@v6 with: version: '~> v2' args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GLASSNODE_SDK_READ_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 994e493..67bd7e3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,12 +2,14 @@ name: CI on: push: - branches: [main] + branches: [main, 'feat/GN-159-extract-go-api-client'] pull_request: jobs: test: runs-on: ubuntu-latest + env: + GOPRIVATE: github.com/glassnode/glassnode-api-go-client steps: - uses: actions/checkout@v4 @@ -15,11 +17,28 @@ jobs: with: go-version: '1.24' + - name: Authenticate private SDK dependency + env: + GH_TOKEN: ${{ secrets.GLASSNODE_SDK_READ_TOKEN }} + run: | + if [ -z "$GH_TOKEN" ]; then + echo "GLASSNODE_SDK_READ_TOKEN must grant read access to glassnode/glassnode-api-go-client. Fork PRs require a trusted maintainer build." + exit 1 + fi + gh auth setup-git + go mod download + - name: Vet + env: + GH_TOKEN: ${{ secrets.GLASSNODE_SDK_READ_TOKEN }} run: go vet ./... - name: Build + env: + GH_TOKEN: ${{ secrets.GLASSNODE_SDK_READ_TOKEN }} run: go build ./... - name: Test - run: go test ./... + env: + GH_TOKEN: ${{ secrets.GLASSNODE_SDK_READ_TOKEN }} + run: go test -race ./... diff --git a/README.md b/README.md index 8c90a63..d51de08 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,19 @@ Download the archive for your OS and architecture from [Releases](https://github ### From source +Source builds use the private `glassnode/glassnode-api-go-client` module. Your +GitHub account must have read access. Authenticate Git with GitHub (for example, +`gh auth setup-git`) and include the module in your existing `GOPRIVATE` list: + +```bash +go env -w GOPRIVATE=github.com/glassnode/glassnode-api-go-client +``` + +Preserve any existing entries in `GOPRIVATE`. Published CLI binaries do not +require GitHub access to run. CI and source releases require the repository +secret `GLASSNODE_SDK_READ_TOKEN`, scoped to read the private SDK repository; +fork pull requests require a trusted maintainer build. + ```bash go install github.com/glassnode/glassnode-cli@latest ``` diff --git a/docs/go-sdk-extraction.md b/docs/go-sdk-extraction.md new file mode 100644 index 0000000..37acd4a --- /dev/null +++ b/docs/go-sdk-extraction.md @@ -0,0 +1,28 @@ +# Go SDK extraction — GN-159 + +The reusable API implementation is now in the private +[`glassnode-api-go-client`](https://github.com/glassnode/glassnode-api-go-client) +module. The design and TS feature mapping live in that repository's +`docs/design.md`. Tracking: https://glassnode.atlassian.net/browse/GN-159 + +`internal/api` is a CLI adapter. It retains environment/config resolution, +OAuth refresh, dry-run URL rendering, asset pruning, credit presentation and +historical output shapes. Endpoint HTTP handling, retries and decoding are +provided by the SDK. The SDK defaults to header authentication; the CLI +explicitly retains query API keys to preserve current dry-run and wire behavior. +SDK clients are initialized lazily once per CLI adapter and reused. + +GET calls now retry transport/read failures, 429 and 5xx responses twice with +jitter. Context cancellation interrupts calls and backoff. Errors expose SDK +types and redact credentials. Redirects are no longer followed. + +The CLI remains public while the SDK is private, so source builds need GitHub +read access. Set `GOPRIVATE` and authenticate Git as described in README.md. +CI/release jobs use `GLASSNODE_SDK_READ_TOKEN`; no credential is committed. +The token must be configured before merging this branch. Do not use +`pull_request_target` to expose it to untrusted fork code. + +For local SDK development, create a temporary Go workspace outside either repo +and use both checkouts. If module graph resolution tries to download the +version in the CLI's go.mod, add a version-specific replace to that temporary +workspace. Never commit an absolute filesystem replace into the CLI module. diff --git a/go.mod b/go.mod index 3413a50..968e526 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/glassnode/glassnode-cli go 1.24.0 require ( + github.com/glassnode/glassnode-api-go-client v0.0.0-20260930201957-f2813ef697f6 github.com/olekukonko/tablewriter v0.0.5 github.com/spf13/cobra v1.10.2 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index f15cc87..81b9a27 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuh github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/glassnode/glassnode-api-go-client v0.0.0-20260930201957-f2813ef697f6 h1:Ej0OlDt91S3UD2b9o3c6GtMAfEsKnry/rts9hsPk/v8= +github.com/glassnode/glassnode-api-go-client v0.0.0-20260930201957-f2813ef697f6/go.mod h1:Jwk/v/xgvMtTXLMV/WnnVa+3ycFVZiBR4EgU8/JdcbQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= diff --git a/internal/api/api_usage.go b/internal/api/api_usage.go index a13cca7..186f6ce 100644 --- a/internal/api/api_usage.go +++ b/internal/api/api_usage.go @@ -2,13 +2,12 @@ package api import ( "context" - "encoding/json" "fmt" + + glassnode "github.com/glassnode/glassnode-api-go-client" ) -type APIAddon struct { - Value int `json:"value"` -} +type APIAddon = glassnode.APIAddon type APIUsageResponse struct { CreditsUsed int `json:"creditsUsed"` @@ -50,15 +49,13 @@ func (a *APIUsageResponse) Summary() CreditsSummary { // GetAPIUsage fetches the current API usage for the authenticated user func (c *Client) GetAPIUsage(ctx context.Context) (*APIUsageResponse, error) { - body, err := c.Do(ctx, "GET", "/v1/user/api_usage", nil) + client, err := c.sdk() if err != nil { - return nil, fmt.Errorf("fetching API usage: %w", err) + return nil, err } - - var out APIUsageResponse - if err := json.Unmarshal(body, &out); err != nil { - return nil, fmt.Errorf("decoding API usage response: %w", err) + usage, err := client.GetAPIUsage(ctx) + if err != nil { + return nil, fmt.Errorf("fetching API usage response: %w", err) } - - return &out, nil + return &APIUsageResponse{CreditsUsed: usage.CreditsUsed, APIAddons: usage.APIAddons}, nil } diff --git a/internal/api/api_usage_test.go b/internal/api/api_usage_test.go index 5872c6f..566a4c9 100644 --- a/internal/api/api_usage_test.go +++ b/internal/api/api_usage_test.go @@ -2,11 +2,13 @@ package api import ( "context" + "errors" "net/http" "net/http/httptest" "os" - "strings" "testing" + + glassnode "github.com/glassnode/glassnode-api-go-client" ) func TestGetAPIUsage(t *testing.T) { @@ -70,7 +72,8 @@ func TestGetAPIUsage_InvalidJSONReturnsError(t *testing.T) { t.Fatal("expected error for invalid JSON") } - if !strings.Contains(err.Error(), "decoding API usage response") { - t.Errorf("error = %v, want wrapping decode message", err) + var decode *glassnode.DecodeError + if !errors.As(err, &decode) || decode.Endpoint != "/v1/user/api_usage" { + t.Errorf("error = %v, want typed API usage decode error", err) } } diff --git a/internal/api/client.go b/internal/api/client.go index 9bd8589..4611606 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -1,6 +1,7 @@ package api import ( + "bytes" "context" "errors" "fmt" @@ -8,44 +9,132 @@ import ( "net/http" "net/url" "os" + "strings" + "sync" "time" + glassnode "github.com/glassnode/glassnode-api-go-client" "github.com/glassnode/glassnode-cli/internal/config" "github.com/glassnode/glassnode-cli/internal/oauth" "github.com/glassnode/glassnode-cli/internal/version" ) -const ( - defaultHTTPTimeout = 1 * time.Minute -) - -var ( - baseURL = "https://api.glassnode.com" - - // stderr is where warning messages are written. Overridable in tests. - stderr io.Writer = os.Stderr -) +var stderr io.Writer = os.Stderr +// Client adapts the public SDK to CLI authentication, configuration and output. +// HTTP requests, retries, errors and endpoint decoding are owned by the SDK. type Client struct { baseURL string apiKey string bearerToken string httpClient *http.Client + init sync.Once + sdkClient *glassnode.Client + initErr error } func NewClient(apiKey, bearerToken string) *Client { - if u := os.Getenv("GLASSNODE_BASE_URL"); u != "" { - baseURL = u + baseURL := os.Getenv("GLASSNODE_BASE_URL") + if baseURL == "" { + baseURL = glassnode.DefaultBaseURL } + return &Client{baseURL: baseURL, apiKey: apiKey, bearerToken: bearerToken, + httpClient: &http.Client{Timeout: time.Minute}} +} + +func (c *Client) sdk() (*glassnode.Client, error) { + c.init.Do(func() { + httpClient := *c.httpClient + if c.bearerToken != "" { + transport := httpClient.Transport + if transport == nil { + transport = http.DefaultTransport + } + httpClient.Transport = &oauthTransport{base: transport, token: c.bearerToken} + } + options := []glassnode.Option{ + glassnode.WithBaseURL(c.baseURL), glassnode.WithHTTPClient(&httpClient), + glassnode.WithAPIKeyInQuery(), glassnode.WithUserAgent("glassnode-cli-" + version.Version), + } + if c.bearerToken != "" { + options = append(options, glassnode.WithBearerToken(c.bearerToken)) + } + c.sdkClient, c.initErr = glassnode.NewClient(c.apiKey, options...) + }) + return c.sdkClient, c.initErr +} - return &Client{ - baseURL: baseURL, - apiKey: apiKey, - bearerToken: bearerToken, - httpClient: &http.Client{ - Timeout: defaultHTTPTimeout, - }, +// oauthTransport preserves the CLI's refresh-on-401 behavior without teaching +// the SDK about config files or OAuth session persistence. +type oauthTransport struct { + base http.RoundTripper + mu sync.Mutex + token string +} + +func (t *oauthTransport) RoundTrip(req *http.Request) (*http.Response, error) { + t.mu.Lock() + token := t.token + t.mu.Unlock() + request := req.Clone(req.Context()) + request.Header.Set("Authorization", "Bearer "+token) + response, err := t.base.RoundTrip(request) + if err != nil || response.StatusCode != http.StatusUnauthorized { + return redactOAuthErrorResponse(response, token), err + } + t.mu.Lock() + fresh := t.token + var refreshErr error + if fresh == token { + fresh, refreshErr = oauth.ForceRefreshAccessToken(req.Context()) + if refreshErr == nil && fresh != "" { + t.token = fresh + } } + t.mu.Unlock() + if refreshErr != nil || fresh == "" || fresh == token { + if refreshErr != nil && !errors.Is(refreshErr, oauth.ErrSessionExpired) { + _, _ = fmt.Fprintln(stderr, "warning: OAuth refresh after HTTP 401 failed") + } + return response, nil + } + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096)) + _ = response.Body.Close() + request = req.Clone(req.Context()) + request.Header.Set("Authorization", "Bearer "+fresh) + response, err = t.base.RoundTrip(request) + return redactOAuthErrorResponse(response, fresh), err +} + +// The SDK knows the original token; the CLI transport also redacts a rotated +// token if an error body echoes it after refresh. +func redactOAuthErrorResponse(response *http.Response, token string) *http.Response { + if response == nil || response.StatusCode < 300 || token == "" { + return response + } + body, err := io.ReadAll(io.LimitReader(response.Body, 4096)) + _ = response.Body.Close() + if err != nil { + body = []byte("failed to read OAuth API error response") + } + text := strings.ReplaceAll(string(body), token, "[redacted]") + text = strings.ReplaceAll(text, url.QueryEscape(token), "[redacted]") + response.Body = io.NopCloser(bytes.NewBufferString(text)) + response.ContentLength = int64(len(text)) + return response +} + +func queryValues(params map[string]string, repeated map[string][]string) url.Values { + q := url.Values{} + for key, value := range params { + q.Set(key, value) + } + for key, values := range repeated { + for _, value := range values { + q.Add(key, value) + } + } + return q } // ResolveAPIKey returns the first non-empty value from: @@ -117,81 +206,13 @@ func (c *Client) Do(ctx context.Context, method, path string, params map[string] return c.DoWithRepeatedParams(ctx, method, path, params, nil) } -func (c *Client) DoWithRepeatedParams(ctx context.Context, method, path string, params map[string]string, repeatedParams map[string][]string) ([]byte, error) { - u, err := c.buildURL(path, params, repeatedParams) - if err != nil { - return nil, err +func (c *Client) DoWithRepeatedParams(ctx context.Context, method, path string, params map[string]string, repeated map[string][]string) ([]byte, error) { + if method != http.MethodGet { + return nil, fmt.Errorf("unsupported HTTP method %s", method) } - - body, status, err := c.sendOnce(ctx, method, u) + client, err := c.sdk() if err != nil { return nil, err } - - // If we were using an OAuth bearer and the server returned Unauthorized, try exactly one refresh+retry. - // This handles the case where the access token was revoked before its expiry - if status == http.StatusUnauthorized && c.bearerToken != "" { - fresh, rerr := oauth.ForceRefreshAccessToken(ctx) - if rerr == nil && fresh != "" && fresh != c.bearerToken { - c.bearerToken = fresh - body, status, err = c.sendOnce(ctx, method, u) - if err != nil { - return nil, err - } - } else if rerr != nil && !errors.Is(rerr, oauth.ErrSessionExpired) { - _, _ = fmt.Fprintf(stderr, "warning: refresh after HTTP %d failed: %v\n", http.StatusUnauthorized, rerr) - } - } - - if status < http.StatusOK || status >= http.StatusMultipleChoices { - if status == http.StatusTooManyRequests { - return nil, fmt.Errorf("HTTP %d: rate limit exceeded. %s", status, string(body)) - } - return nil, fmt.Errorf("HTTP %d: %s", status, string(body)) - } - return body, nil -} - -func (c *Client) buildURL(path string, params map[string]string, repeatedParams map[string][]string) (*url.URL, error) { - u, err := url.Parse(c.baseURL + path) - if err != nil { - return nil, fmt.Errorf("parsing URL: %w", err) - } - q := u.Query() - if c.bearerToken == "" { - q.Set("api_key", c.apiKey) - } - for k, v := range params { - q.Set(k, v) - } - for k, vals := range repeatedParams { - for _, v := range vals { - q.Add(k, v) - } - } - u.RawQuery = q.Encode() - return u, nil -} - -func (c *Client) sendOnce(ctx context.Context, method string, u *url.URL) ([]byte, int, error) { - req, err := http.NewRequestWithContext(ctx, method, u.String(), nil) - if err != nil { - return nil, 0, fmt.Errorf("creating request: %w", err) - } - req.Header.Set("User-Agent", "glassnode-cli-"+version.Version) - if c.bearerToken != "" { - req.Header.Set("Authorization", "Bearer "+c.bearerToken) - } - - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, 0, fmt.Errorf("executing request: %w", err) - } - defer func() { _ = resp.Body.Close() }() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, resp.StatusCode, fmt.Errorf("reading response: %w", err) - } - return body, resp.StatusCode, nil + return client.Raw(ctx, path, queryValues(params, repeated)) } diff --git a/internal/api/client_test.go b/internal/api/client_test.go index d7127a2..17e6c5c 100644 --- a/internal/api/client_test.go +++ b/internal/api/client_test.go @@ -3,6 +3,7 @@ package api import ( "context" _ "embed" + "io" "net/http" "net/http/httptest" "os" @@ -277,9 +278,24 @@ func TestDo_401TriggersRefreshAndRetry(t *testing.T) { if authHdrs[0] != "Bearer stale" || authHdrs[1] != "Bearer fresh" { t.Errorf("auth headers %v", authHdrs) } + if _, err := client.Do(context.Background(), "GET", "/v1/anything", nil); err != nil { + t.Fatal(err) + } + if len(authHdrs) != 3 || authHdrs[2] != "Bearer fresh" { + t.Errorf("refreshed token was not reused: %v", authHdrs) + } }) } +func TestOAuthErrorResponseRedactsRotatedToken(t *testing.T) { + response := &http.Response{StatusCode: 401, Body: io.NopCloser(strings.NewReader("rotated-secret"))} + response = redactOAuthErrorResponse(response, "rotated-secret") + body, err := io.ReadAll(response.Body) + if err != nil || strings.Contains(string(body), "rotated-secret") { + t.Fatalf("unredacted response: %s %v", body, err) + } +} + func TestDo_401WithAPIKey_NoRetry(t *testing.T) { // When using an API key (not a bearer), a 401 must NOT trigger a refresh attempt; we just // surface the error. This guards against silently attempting OAuth refreshes for users who @@ -764,7 +780,7 @@ func TestDo_4xxReturnsErrorWithBody(t *testing.T) { })) defer server.Close() - client := NewClient("key", "") + client := NewClient("test-api-secret", "") client.baseURL = server.URL client.httpClient = server.Client() diff --git a/internal/api/metadata.go b/internal/api/metadata.go index b9e65cc..267a328 100644 --- a/internal/api/metadata.go +++ b/internal/api/metadata.go @@ -2,85 +2,22 @@ package api import ( "context" - "encoding/json" - "fmt" "net/url" "strings" -) - -type Asset struct { - ID string `json:"id"` - ExternalIDs map[string]string `json:"external_ids,omitempty"` - Symbol string `json:"symbol"` - Name string `json:"name"` - AssetType string `json:"asset_type"` - Blockchains []Blockchain `json:"blockchains"` - Categories []string `json:"categories,omitempty"` - LogoURL string `json:"logo_url,omitempty"` - SemanticTags []string `json:"semantic_tags,omitempty"` - DefaultNetwork string `json:"default_network,omitempty"` -} - -type Blockchain struct { - Blockchain string `json:"blockchain"` - Address string `json:"address,omitempty"` - Decimals int `json:"decimals,omitempty"` - OnChainSupport bool `json:"on_chain_support"` -} - -type AssetsResponse struct { - Data []Asset `json:"data"` -} - -// NamesResponse is the response of the metadata list endpoints -// (/v1/metadata/tags, /v1/metadata/assets/tags, ...): {"data":[{"name":"..."},...]}. -type NamesResponse struct { - Data []NameEntry `json:"data"` -} - -// NameEntry is a single named value in a names response. -type NameEntry struct { - Name string `json:"name"` -} - -type MetricMetadata struct { - Path string `json:"path,omitempty"` - Tier float64 `json:"tier,omitempty"` - IsPit bool `json:"is_pit,omitempty"` - Parameters map[string][]string `json:"parameters"` - Queried map[string]string `json:"queried,omitempty"` - Refs Refs `json:"refs,omitempty"` - BulkSupported bool `json:"bulk_supported"` - Timerange *Timerange `json:"timerange,omitempty"` - Modified int64 `json:"modified,omitempty"` - Descriptors *MetricDescriptors `json:"descriptors,omitempty"` -} - -type MetricVariant struct { - Base *string `json:"base,omitempty"` - Bulk *string `json:"bulk,omitempty"` - Pit *string `json:"pit,omitempty"` -} -type MetricDescriptors struct { - Name string `json:"name,omitempty"` - ShortName string `json:"short_name,omitempty"` - Group string `json:"group,omitempty"` - Tags []string `json:"tags,omitempty"` - Description map[string]string `json:"description,omitempty"` - DataSharingGroup string `json:"data_sharing_group,omitempty"` -} - -type Timerange struct { - Min int64 `json:"min,omitempty"` - Max int64 `json:"max,omitempty"` -} + glassnode "github.com/glassnode/glassnode-api-go-client" +) -type Refs struct { - Doc string `json:"docs,omitempty"` - Studio string `json:"studio,omitempty"` - MetricVariant *MetricVariant `json:"metric_variant,omitempty"` -} +type Asset = glassnode.Asset +type Blockchain = glassnode.Blockchain +type AssetsResponse = glassnode.AssetsResponse +type NamesResponse = glassnode.NamesResponse +type NameEntry = glassnode.NameEntry +type MetricMetadata = glassnode.MetricMetadata +type MetricVariant = glassnode.MetricVariant +type MetricDescriptors = glassnode.MetricDescriptors +type Timerange = glassnode.TimeRange +type Refs = glassnode.Refs // assetToMap returns a map of JSON field names to values for the given asset. // Used by PruneAssets to build objects with only requested fields. @@ -123,88 +60,48 @@ func PruneAssets(assets []Asset, fields []string) []map[string]interface{} { } func (c *Client) ListAssets(ctx context.Context, filter string) ([]Asset, error) { - params := map[string]string{} - if filter != "" { - params["filter"] = filter - } - body, err := c.Do(ctx, "GET", "/v1/metadata/assets", params) + client, err := c.sdk() if err != nil { - return nil, fmt.Errorf("listing assets: %w", err) - } - var resp AssetsResponse - if err := json.Unmarshal(body, &resp); err != nil { - return nil, fmt.Errorf("decoding assets response: %w", err) + return nil, err } - return resp.Data, nil + return client.ListAssets(ctx, filter) } -// ListNames lists the named values from a metadata list endpoint such as -// /v1/metadata/tags or /v1/metadata/assets/categories, optionally filtered by a CEL expression. -func (c *Client) ListNames(ctx context.Context, path string, filter string) ([]string, error) { - params := map[string]string{} - if filter != "" { - params["filter"] = filter - } - body, err := c.Do(ctx, "GET", path, params) +func (c *Client) ListNames(ctx context.Context, path, filter string) ([]string, error) { + client, err := c.sdk() if err != nil { - return nil, fmt.Errorf("listing %s: %w", path, err) - } - var resp NamesResponse - if err := json.Unmarshal(body, &resp); err != nil { - return nil, fmt.Errorf("decoding %s response: %w", path, err) + return nil, err } - names := make([]string, 0, len(resp.Data)) - for _, e := range resp.Data { - names = append(names, e.Name) - } - return names, nil + return client.ListNames(ctx, path, filter) } -// ListMetrics lists available metric paths, optionally filtered by query parameters. -// See https://docs.glassnode.com/basic-api/metadata#query-parameters-1 -// params: a (asset), c (currency), e (exchange), f (format), i (interval), -// from_exchange, to_exchange, miner, maturity, network, period, quote_symbol. -// repeatedParams: use key "a" for multiple assets (e.g. a=BTC&a=ETH). -func (c *Client) ListMetrics(ctx context.Context, params map[string]string, repeatedParams map[string][]string) ([]string, error) { - if params == nil { - params = map[string]string{} - } - if repeatedParams == nil { - repeatedParams = map[string][]string{} - } - body, err := c.DoWithRepeatedParams(ctx, "GET", "/v1/metadata/metrics", params, repeatedParams) +func (c *Client) ListMetrics(ctx context.Context, params map[string]string, repeated map[string][]string) ([]string, error) { + client, err := c.sdk() if err != nil { - return nil, fmt.Errorf("listing metrics: %w", err) - } - var metrics []string - if err := json.Unmarshal(body, &metrics); err != nil { - return nil, fmt.Errorf("decoding metrics response: %w", err) + return nil, err } - return metrics, nil + return client.ListMetrics(ctx, &glassnode.MetricParams{Extra: queryValues(params, repeated)}) } func (c *Client) DescribeMetric(ctx context.Context, path, asset string) (*MetricMetadata, error) { - params := map[string]string{"path": path} - if asset != "" { - params["a"] = asset - } - body, err := c.Do(ctx, "GET", "/v1/metadata/metric", params) + client, err := c.sdk() if err != nil { - return nil, fmt.Errorf("describing metric: %w", err) - } - var meta MetricMetadata - if err := json.Unmarshal(body, &meta); err != nil { - return nil, fmt.Errorf("decoding metric metadata: %w", err) + return nil, err } - return &meta, nil + return client.GetMetricMetadata(ctx, path, &glassnode.MetricParams{Asset: asset}) } // BuildURL constructs the full request URL without executing the request. func (c *Client) BuildURL(path string, params map[string]string, repeatedParams map[string][]string) (string, error) { - u, err := c.buildURL(path, params, repeatedParams) + u, err := url.Parse(c.baseURL + path) if err != nil { return "", err } + q := queryValues(params, repeatedParams) + if c.bearerToken == "" { + q.Set("api_key", c.apiKey) + } + u.RawQuery = q.Encode() return u.String(), nil } diff --git a/internal/api/metrics.go b/internal/api/metrics.go index d4c4dda..50701e3 100644 --- a/internal/api/metrics.go +++ b/internal/api/metrics.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "fmt" + + glassnode "github.com/glassnode/glassnode-api-go-client" ) type DataPoint struct { @@ -21,28 +23,48 @@ type BulkResponse struct { Data []BulkDataPoint `json:"data"` } +// UnmarshalJSON keeps the CLI's historical dynamic number/output behavior. +func (p *DataPoint) UnmarshalJSON(data []byte) error { + type wire DataPoint + return json.Unmarshal(data, (*wire)(p)) +} + func (c *Client) GetMetric(ctx context.Context, path string, params map[string]string) ([]DataPoint, error) { - normalized := NormalizePath(path) - body, err := c.Do(ctx, "GET", "/v1/metrics"+normalized, params) + client, err := c.sdk() if err != nil { - return nil, fmt.Errorf("getting metric: %w", err) + return nil, err } var points []DataPoint - if err := json.Unmarshal(body, &points); err != nil { - return nil, fmt.Errorf("decoding metric response: %w", err) + if err := client.GetMetric(ctx, path, &glassnode.MetricParams{Extra: queryValues(params, nil)}, &points); err != nil { + return nil, fmt.Errorf("getting metric: %w", err) } return points, nil } -func (c *Client) GetMetricBulk(ctx context.Context, path string, params map[string]string, repeatedParams map[string][]string) (*BulkResponse, error) { - normalized := NormalizePath(path) - body, err := c.DoWithRepeatedParams(ctx, "GET", "/v1/metrics"+normalized+"/bulk", params, repeatedParams) +func (c *Client) GetMetricBulk(ctx context.Context, path string, params map[string]string, repeated map[string][]string) (*BulkResponse, error) { + client, err := c.sdk() + if err != nil { + return nil, err + } + points, err := client.GetBulkMetric(ctx, path, &glassnode.MetricParams{Extra: queryValues(params, repeated)}) if err != nil { return nil, fmt.Errorf("getting bulk metric: %w", err) } - var resp BulkResponse - if err := json.Unmarshal(body, &resp); err != nil { - return nil, fmt.Errorf("decoding bulk metric response: %w", err) + response := &BulkResponse{Data: make([]BulkDataPoint, 0, len(points))} + for _, point := range points { + entries := make([]map[string]interface{}, 0, len(point.Bulk)) + for _, entry := range point.Bulk { + var value any + if entry.Value != nil { + value = *entry.Value + } + row := map[string]interface{}{"a": entry.Asset, "v": value} + if entry.Network != "" { + row["network"] = entry.Network + } + entries = append(entries, row) + } + response.Data = append(response.Data, BulkDataPoint{T: point.Timestamp, Bulk: entries}) } - return &resp, nil + return response, nil } diff --git a/internal/output/output.go b/internal/output/output.go index 7c1b363..bf34f2f 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -247,9 +247,9 @@ func PrintTable(w io.Writer, data interface{}, timestampFormat string) error { func printMetricMetadataTable(w io.Writer, m *api.MetricMetadata) error { fmt.Fprintf(w, "Path: %s\n", m.Path) - fmt.Fprintf(w, "Tier: %.0f\n", m.Tier) + fmt.Fprintf(w, "Tier: %d\n", m.Tier) fmt.Fprintf(w, "Bulk Supported: %t\n", m.BulkSupported) - fmt.Fprintf(w, "PIT: %t\n", m.IsPit) + fmt.Fprintf(w, "PIT: %t\n", m.IsPIT) if m.Descriptors != nil { if m.Descriptors.Name != "" { fmt.Fprintf(w, "Name: %s\n", m.Descriptors.Name) @@ -261,8 +261,8 @@ func printMetricMetadataTable(w io.Writer, m *api.MetricMetadata) error { fmt.Fprintf(w, "Tags: %s\n", strings.Join(m.Descriptors.Tags, ", ")) } } - if m.Timerange != nil { - fmt.Fprintf(w, "Timerange: %d - %d\n", m.Timerange.Min, m.Timerange.Max) + if m.TimeRange != nil { + fmt.Fprintf(w, "Timerange: %d - %d\n", m.TimeRange.Min, m.TimeRange.Max) } if len(m.Parameters) > 0 { fmt.Fprintln(w, "Parameters:") diff --git a/internal/output/output_test.go b/internal/output/output_test.go index 5b1a635..3f8df4b 100644 --- a/internal/output/output_test.go +++ b/internal/output/output_test.go @@ -305,7 +305,7 @@ func TestPrintTable_MetricMetadataWithOptionalFields(t *testing.T) { Group: "Market", Tags: []string{"price", "market"}, }, - Timerange: &api.Timerange{Min: 1609459200, Max: 1735689600}, + TimeRange: &api.Timerange{Min: 1609459200, Max: 1735689600}, Parameters: map[string][]string{"a": {"BTC", "ETH"}, "i": {"24h"}}, } if err := PrintTable(&buf, meta, ""); err != nil { From dd22b5a605eacf97a077c87767c80075709b0b7d Mon Sep 17 00:00:00 2001 From: planadecu Date: Mon, 5 Oct 2026 11:46:00 +0200 Subject: [PATCH 02/37] GN-159: pin reviewed SDK v0.1.0 and use OAuth token source --- docs/go-sdk-extraction.md | 1 + go.mod | 2 +- go.sum | 4 ++-- internal/api/client.go | 32 +++++++++++++++++++++++--------- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/docs/go-sdk-extraction.md b/docs/go-sdk-extraction.md index 37acd4a..082594c 100644 --- a/docs/go-sdk-extraction.md +++ b/docs/go-sdk-extraction.md @@ -10,6 +10,7 @@ OAuth refresh, dry-run URL rendering, asset pruning, credit presentation and historical output shapes. Endpoint HTTP handling, retries and decoding are provided by the SDK. The SDK defaults to header authentication; the CLI explicitly retains query API keys to preserve current dry-run and wire behavior. +OAuth uses the SDK token-source API alongside the CLI refresh-on-401 transport. SDK clients are initialized lazily once per CLI adapter and reused. GET calls now retry transport/read failures, 429 and 5xx responses twice with diff --git a/go.mod b/go.mod index 968e526..0461a05 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/glassnode/glassnode-cli go 1.24.0 require ( - github.com/glassnode/glassnode-api-go-client v0.0.0-20260930201957-f2813ef697f6 + github.com/glassnode/glassnode-api-go-client v0.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/spf13/cobra v1.10.2 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 81b9a27..3b40242 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuh github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/glassnode/glassnode-api-go-client v0.0.0-20260930201957-f2813ef697f6 h1:Ej0OlDt91S3UD2b9o3c6GtMAfEsKnry/rts9hsPk/v8= -github.com/glassnode/glassnode-api-go-client v0.0.0-20260930201957-f2813ef697f6/go.mod h1:Jwk/v/xgvMtTXLMV/WnnVa+3ycFVZiBR4EgU8/JdcbQ= +github.com/glassnode/glassnode-api-go-client v0.1.0 h1:bhPaVNe3MDgh/hr7XKgnh4OvVkKlobE4U4zGdxTWu5A= +github.com/glassnode/glassnode-api-go-client v0.1.0/go.mod h1:Jwk/v/xgvMtTXLMV/WnnVa+3ycFVZiBR4EgU8/JdcbQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= diff --git a/internal/api/client.go b/internal/api/client.go index 4611606..bd53e5e 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -45,21 +45,25 @@ func NewClient(apiKey, bearerToken string) *Client { func (c *Client) sdk() (*glassnode.Client, error) { c.init.Do(func() { httpClient := *c.httpClient + options := []glassnode.Option{ + glassnode.WithBaseURL(c.baseURL), + glassnode.WithUserAgent("glassnode-cli-" + version.Version), + } + key := c.apiKey if c.bearerToken != "" { transport := httpClient.Transport if transport == nil { transport = http.DefaultTransport } - httpClient.Transport = &oauthTransport{base: transport, token: c.bearerToken} - } - options := []glassnode.Option{ - glassnode.WithBaseURL(c.baseURL), glassnode.WithHTTPClient(&httpClient), - glassnode.WithAPIKeyInQuery(), glassnode.WithUserAgent("glassnode-cli-" + version.Version), - } - if c.bearerToken != "" { - options = append(options, glassnode.WithBearerToken(c.bearerToken)) + auth := &oauthTransport{base: transport, token: c.bearerToken} + httpClient.Transport = auth + options = append(options, glassnode.WithTokenSource(auth)) + key = "" + } else { + options = append(options, glassnode.WithAPIKeyInQuery()) } - c.sdkClient, c.initErr = glassnode.NewClient(c.apiKey, options...) + options = append(options, glassnode.WithHTTPClient(&httpClient)) + c.sdkClient, c.initErr = glassnode.NewClient(key, options...) }) return c.sdkClient, c.initErr } @@ -72,6 +76,16 @@ type oauthTransport struct { token string } +// Token supplies the current session token to the SDK on each attempt. +func (t *oauthTransport) Token(ctx context.Context) (string, error) { + if err := ctx.Err(); err != nil { + return "", err + } + t.mu.Lock() + defer t.mu.Unlock() + return t.token, nil +} + func (t *oauthTransport) RoundTrip(req *http.Request) (*http.Response, error) { t.mu.Lock() token := t.token From f9fd7c223429448871f853fa34661d4d4d421123 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Thu, 8 Oct 2026 09:20:05 +0200 Subject: [PATCH 03/37] GN-159: build against the SDK review branch Pin the SDK to the head of its review branch (fix/user-agent-validation) so this PR tracks the API changes under review; the released tag is substituted last. Adapt to the SDK changes: - BulkEntry keeps every selector in Params and Asset/Network are methods. Bulk rows now copy all selectors (e, category, network, ...) instead of only a and network, so they are no longer dropped from CLI output. - Map s, u, i, c, a and e onto the typed MetricParams fields so the SDK validates the time range and selectors; other parameters stay in Extra. - Bulk requests need s; the bulk tests pass one. --- go.mod | 2 +- go.sum | 2 ++ internal/api/client_test.go | 48 +++++++++++++++++++++++++-- internal/api/metrics.go | 66 +++++++++++++++++++++++++++++++++---- 4 files changed, 108 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 0461a05..1b54123 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/glassnode/glassnode-cli go 1.24.0 require ( - github.com/glassnode/glassnode-api-go-client v0.1.0 + github.com/glassnode/glassnode-api-go-client v0.1.1-0.20261007130113-3cf15fec37f3 github.com/olekukonko/tablewriter v0.0.5 github.com/spf13/cobra v1.10.2 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 3b40242..ad24620 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6N github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/glassnode/glassnode-api-go-client v0.1.0 h1:bhPaVNe3MDgh/hr7XKgnh4OvVkKlobE4U4zGdxTWu5A= github.com/glassnode/glassnode-api-go-client v0.1.0/go.mod h1:Jwk/v/xgvMtTXLMV/WnnVa+3ycFVZiBR4EgU8/JdcbQ= +github.com/glassnode/glassnode-api-go-client v0.1.1-0.20261007130113-3cf15fec37f3 h1:1D4TCb69qYrYwyQr3FglHOhQfaw2xqeybVUKkq7MJ6Y= +github.com/glassnode/glassnode-api-go-client v0.1.1-0.20261007130113-3cf15fec37f3/go.mod h1:Jwk/v/xgvMtTXLMV/WnnVa+3ycFVZiBR4EgU8/JdcbQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= diff --git a/internal/api/client_test.go b/internal/api/client_test.go index 17e6c5c..7fc0e8e 100644 --- a/internal/api/client_test.go +++ b/internal/api/client_test.go @@ -598,7 +598,7 @@ func TestGetMetricBulk(t *testing.T) { client.baseURL = server.URL client.httpClient = server.Client() - resp, err := client.GetMetricBulk(context.Background(), "/market/price_usd_close", nil, nil) + resp, err := client.GetMetricBulk(context.Background(), "/market/price_usd_close", map[string]string{"s": "1770076800"}, nil) if err != nil { t.Fatalf("GetMetricBulk: %v", err) } @@ -683,7 +683,7 @@ func TestGetMetricBulk_InvalidJSONReturnsError(t *testing.T) { client.baseURL = server.URL client.httpClient = server.Client() - _, err := client.GetMetricBulk(context.Background(), "/market/price", nil, nil) + _, err := client.GetMetricBulk(context.Background(), "/market/price", map[string]string{"s": "1"}, nil) if err == nil { t.Error("expected error for invalid JSON") } @@ -762,7 +762,7 @@ func TestGetMetricBulk_EmptyData(t *testing.T) { client.baseURL = server.URL client.httpClient = server.Client() - resp, err := client.GetMetricBulk(context.Background(), "/market/price", nil, nil) + resp, err := client.GetMetricBulk(context.Background(), "/market/price", map[string]string{"s": "1"}, nil) if err != nil { t.Fatalf("GetMetricBulk: %v", err) } @@ -829,3 +829,45 @@ func TestRedactAPIKeyFromURL_InvalidURL(t *testing.T) { t.Error("expected error for invalid URL") } } + +func TestGetMetricBulk_KeepsAllSelectors(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[{"t":1,"bulk":[ + {"a":"BTC","e":"binance","v":1}, + {"a":"ETH","category":"more_10y","v":null}]}]}`)) + })) + defer server.Close() + + client := NewClient("key", "") + client.baseURL = server.URL + client.httpClient = server.Client() + + resp, err := client.GetMetricBulk(context.Background(), "/distribution/balance_exchanges", map[string]string{"s": "1"}, nil) + if err != nil { + t.Fatalf("GetMetricBulk: %v", err) + } + rows := resp.Data[0].Bulk + if rows[0]["e"] != "binance" || rows[0]["a"] != "BTC" || rows[0]["v"] != 1.0 { + t.Errorf("exchange selector lost: %v", rows[0]) + } + if rows[1]["category"] != "more_10y" || rows[1]["v"] != nil { + t.Errorf("category selector lost: %v", rows[1]) + } +} + +func TestMetricParamsMapsTypedFields(t *testing.T) { + p, err := metricParams(map[string]string{"s": "100", "u": "200", "i": "24h", "c": "USD", "a": "BTC", "network": "eth"}, map[string][]string{"e": {"binance", "coinbase"}}) + if err != nil { + t.Fatal(err) + } + if p.Since.Unix() != 100 || p.Until.Unix() != 200 || p.Interval != "24h" || p.Currency != "USD" || p.Asset != "BTC" || len(p.Exchanges) != 2 { + t.Errorf("typed fields: %+v", p) + } + if p.Extra.Get("network") != "eth" || p.Extra.Has("s") || p.Extra.Has("a") || p.Extra.Has("e") { + t.Errorf("extra: %v", p.Extra) + } + if _, err := metricParams(map[string]string{"s": "yesterday"}, nil); err == nil { + t.Error("accepted non-numeric since") + } +} diff --git a/internal/api/metrics.go b/internal/api/metrics.go index 50701e3..1360bd0 100644 --- a/internal/api/metrics.go +++ b/internal/api/metrics.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "fmt" + "strconv" + "time" glassnode "github.com/glassnode/glassnode-api-go-client" ) @@ -29,13 +31,58 @@ func (p *DataPoint) UnmarshalJSON(data []byte) error { return json.Unmarshal(data, (*wire)(p)) } +// metricParams maps the CLI's flat query parameters onto the SDK's typed +// fields, so the SDK validates the time range and selectors; anything it has +// no field for stays in Extra. +func metricParams(params map[string]string, repeated map[string][]string) (*glassnode.MetricParams, error) { + p := &glassnode.MetricParams{Extra: queryValues(params, repeated)} + for key, value := range params { + switch key { + case "s", "u": + seconds, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, fmt.Errorf("parameter %s: expected Unix seconds, got %q", key, value) + } + if key == "s" { + p.Since = time.Unix(seconds, 0) + } else { + p.Until = time.Unix(seconds, 0) + } + case "i": + p.Interval = value + case "c": + p.Currency = value + case "a": + p.Asset = value + case "e": + p.Exchanges = []string{value} + default: + continue + } + p.Extra.Del(key) + } + if assets := repeated["a"]; len(assets) > 0 { + p.Assets = assets + p.Extra.Del("a") + } + if exchanges := repeated["e"]; len(exchanges) > 0 { + p.Exchanges = exchanges + p.Extra.Del("e") + } + return p, nil +} + func (c *Client) GetMetric(ctx context.Context, path string, params map[string]string) ([]DataPoint, error) { client, err := c.sdk() if err != nil { return nil, err } + p, err := metricParams(params, nil) + if err != nil { + return nil, err + } var points []DataPoint - if err := client.GetMetric(ctx, path, &glassnode.MetricParams{Extra: queryValues(params, nil)}, &points); err != nil { + if err := client.GetMetric(ctx, path, p, &points); err != nil { return nil, fmt.Errorf("getting metric: %w", err) } return points, nil @@ -46,7 +93,11 @@ func (c *Client) GetMetricBulk(ctx context.Context, path string, params map[stri if err != nil { return nil, err } - points, err := client.GetBulkMetric(ctx, path, &glassnode.MetricParams{Extra: queryValues(params, repeated)}) + p, err := metricParams(params, repeated) + if err != nil { + return nil, err + } + points, err := client.GetBulkMetric(ctx, path, p) if err != nil { return nil, fmt.Errorf("getting bulk metric: %w", err) } @@ -54,14 +105,17 @@ func (c *Client) GetMetricBulk(ctx context.Context, path string, params map[stri for _, point := range points { entries := make([]map[string]interface{}, 0, len(point.Bulk)) for _, entry := range point.Bulk { + // Every selector of the entry (a, e, network, category, ...) is + // kept, as the API returned it, next to the value. + row := make(map[string]interface{}, len(entry.Params)+1) + for key, selector := range entry.Params { + row[key] = selector + } var value any if entry.Value != nil { value = *entry.Value } - row := map[string]interface{}{"a": entry.Asset, "v": value} - if entry.Network != "" { - row["network"] = entry.Network - } + row["v"] = value entries = append(entries, row) } response.Data = append(response.Data, BulkDataPoint{T: point.Timestamp, Bulk: entries}) From 4d9d066b035c7bf36d674706693ea3e39db2a944 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Thu, 8 Oct 2026 12:15:58 +0200 Subject: [PATCH 04/37] GN-159: require --since for bulk metrics and add --split The API requires s for bulk requests and caps the range per request (10 days at 10m/1h, 31 at 24h, 93 at 1w/1month). Bulk metric get without --since never worked, it failed with HTTP 400, so the CLI now says so before sending anything. A range over the cap is rejected the same way, unless --split is passed: the range is then fetched in consecutive windows, with a note on stderr about the number of requests, and the points are merged. The API includes the interval containing s and only intervals ending by u, so consecutive windows neither overlap nor leave gaps; points are still deduplicated by timestamp as a guard. If the API still answers "too large time range", the window is halved for the rest of the range, up to three times. Any other failure aborts the fetch and names the window, so a script never gets a partial result that looks complete. --dry-run prints the URL of every window. --split is ignored for non-bulk paths. --- README.md | 12 ++- cmd/metric_get.go | 31 ++++-- internal/api/bulk_split.go | 172 +++++++++++++++++++++++++++++ internal/api/bulk_split_test.go | 184 ++++++++++++++++++++++++++++++++ 4 files changed, 390 insertions(+), 9 deletions(-) create mode 100644 internal/api/bulk_split.go create mode 100644 internal/api/bulk_split_test.go diff --git a/README.md b/README.md index d51de08..c32b1b9 100644 --- a/README.md +++ b/README.md @@ -232,8 +232,10 @@ gn metric get market/price_usd_close --asset BTC --interval 24h gn metric get market/price_usd_close --asset BTC --since 2024-01-01 --until 2024-02-01 gn metric get indicators/sopr --asset BTC --interval 24h --since 30d gn metric get distribution/balance_exchanges --asset BTC --exchange binance --currency usd -# Bulk: append /bulk to the path; use -a '*' for all assets or multiple -a for specific ones +# Bulk: append /bulk to the path; use -a '*' for all assets or multiple -a for specific ones. +# --since is required; longer ranges than the API allows per request need --split gn metric get market/marketcap_usd/bulk -a '*' --since 30d +gn metric get market/marketcap_usd/bulk -a BTC -a ETH --since 2024-01-01 --split ``` | Flag | Short | Description | @@ -245,9 +247,12 @@ gn metric get market/marketcap_usd/bulk -a '*' --since 30d | `--currency` | `-c` | Currency for the metric (`usd`, `native`) | | `--exchange` | `-e` | Exchange filter (repeatable for bulk) | | `--network` | `-n` | Network filter | +| `--split` | | Bulk only: fetch a range longer than the API allows per request in several requests | For **bulk metrics**, append `/bulk` to the path (e.g. `market/marketcap_usd/bulk`). To pass multiple assets, repeat the `-a` (or `--asset`) flag for each: `-a BTC -a ETH -a SOL`. Use `-a '*'` to request all assets. +The API requires `--since` for bulk metrics and accepts at most 10 days per request at `10m`/`1h`, 31 days at `24h` and 93 days at `1w`/`1month`. A longer range is rejected before any request is sent unless you pass `--split`: the CLI then fetches it window by window, prints how many requests that takes on stderr, and merges the points. Every request is billed per asset, so narrow `-a` for long ranges. The merged result is assembled in memory before it is printed. + ### `gn user credits` Show the current API usage for your account. @@ -332,8 +337,9 @@ gn metric get market/price_usd_close --asset BTC --since 30d --dry-run Append `/bulk` to the metric path. Specify multiple assets by repeating `-a` for each, or use `-a '*'` for all: ```bash -# Multiple specific assets (repeat -a for each) -gn metric get market/marketcap_usd/bulk -a BTC -a ETH -a SOL -s 2024-01-01 +# Multiple specific assets (repeat -a for each); a range over the API's +# per-request limit is fetched in several requests with --split +gn metric get market/marketcap_usd/bulk -a BTC -a ETH -a SOL -s 2024-01-01 -u 2024-04-01 --split # All assets (wildcard) gn metric get market/marketcap_usd/bulk -a '*' --interval 24h --since 30d diff --git a/cmd/metric_get.go b/cmd/metric_get.go index 86c7947..32d76cc 100644 --- a/cmd/metric_get.go +++ b/cmd/metric_get.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "time" "github.com/glassnode/glassnode-cli/internal/api" "github.com/glassnode/glassnode-cli/internal/output" @@ -70,18 +71,35 @@ var metricGetCmd = &cobra.Command{ repeatedParams["e"] = exchanges } + // The API requires s for bulk requests and limits the range per + // request; longer ranges are fetched in several requests with --split. + plan, err := api.PlanBulk(params, time.Now()) + if err != nil { + return err + } + split, _ := cmd.Flags().GetBool("split") + windows := plan.Windows() + if len(windows) > 1 && !split { + return plan.RangeError() + } + if len(windows) > 1 { + fmt.Fprintln(cmd.ErrOrStderr(), plan.Describe()) + } + dryRun, _ := cmd.Flags().GetBool("dry-run") if dryRun { - u, err := client.BuildURL("/v1/metrics"+path+"/bulk", params, repeatedParams) - if err != nil { - return err + for _, w := range windows { + u, err := client.BuildURL("/v1/metrics"+path+"/bulk", api.WindowParams(params, w), repeatedParams) + if err != nil { + return err + } + redacted, _ := api.RedactAPIKeyFromURL(u) + fmt.Println(redacted) } - redacted, _ := api.RedactAPIKeyFromURL(u) - fmt.Println(redacted) return nil } - resp, err := client.GetMetricBulk(cmd.Context(), path, params, repeatedParams) + resp, err := client.GetMetricBulkSplit(cmd.Context(), path, params, repeatedParams, plan) if err != nil { return err } @@ -127,4 +145,5 @@ func init() { metricGetCmd.Flags().StringP("currency", "c", "", "native or usd") metricGetCmd.Flags().StringArrayP("exchange", "e", nil, "exchange filter (repeatable for bulk)") metricGetCmd.Flags().StringP("network", "n", "", "network filter") + metricGetCmd.Flags().Bool("split", false, "bulk only: fetch a range longer than the API allows per request in several requests") } diff --git a/internal/api/bulk_split.go b/internal/api/bulk_split.go new file mode 100644 index 0000000..b12b6e7 --- /dev/null +++ b/internal/api/bulk_split.go @@ -0,0 +1,172 @@ +package api + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "time" + + glassnode "github.com/glassnode/glassnode-api-go-client" +) + +// Bulk requests are limited to a time range that depends on the resolution; +// see https://docs.glassnode.com/basic-api/bulk-metrics#timerange-constraints. +const ( + bulkWindowFine = 10 * 24 * time.Hour // 10m and 1h + bulkWindowDaily = 31 * 24 * time.Hour // 24h, the default resolution + bulkWindowCoarse = 93 * 24 * time.Hour // 1w and 1month + defaultResolution = "24h" + + // When the API still rejects a window as too large, the window is halved + // this many times before giving up. + maxWindowShrinks = 3 +) + +// BulkWindow returns the longest time range the API accepts in one bulk +// request at the given resolution. +func BulkWindow(interval string) time.Duration { + switch interval { + case "10m", "1h": + return bulkWindowFine + case "1w", "1month": + return bulkWindowCoarse + default: + return bulkWindowDaily + } +} + +// BulkPlan is the time range of a bulk request and the window it is fetched in. +type BulkPlan struct { + Since, Until time.Time + Interval string + Window time.Duration +} + +// Window is one request's time range: Since inclusive, Until exclusive. +type Window struct{ Since, Until time.Time } + +// PlanBulk reads the s, u and i parameters of a bulk request. s is required +// by the API; u defaults to now and is capped at now, since the API returns no +// future points. +func PlanBulk(params map[string]string, now time.Time) (BulkPlan, error) { + plan := BulkPlan{Interval: params["i"], Until: now} + if plan.Interval == "" { + plan.Interval = defaultResolution + } + plan.Window = BulkWindow(plan.Interval) + raw, ok := params["s"] + if !ok { + return plan, fmt.Errorf("bulk metrics require --since: the API accepts at most %s per request at %s (add --split to fetch a longer range in several requests)", + days(plan.Window), plan.Interval) + } + since, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + return plan, fmt.Errorf("parameter s: expected Unix seconds, got %q", raw) + } + plan.Since = time.Unix(since, 0).UTC() + if raw, ok := params["u"]; ok { + until, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + return plan, fmt.Errorf("parameter u: expected Unix seconds, got %q", raw) + } + if t := time.Unix(until, 0).UTC(); t.Before(plan.Until) { + plan.Until = t + } + } + plan.Until = plan.Until.UTC() + if !plan.Since.Before(plan.Until) { + return plan, fmt.Errorf("--since (%s) must be before --until (%s)", plan.Since.Format(time.RFC3339), plan.Until.Format(time.RFC3339)) + } + return plan, nil +} + +// Windows splits the range into consecutive windows of at most plan.Window. +// Each window's Until is the next window's Since: the API includes the +// interval containing s and only intervals that end by u, so consecutive +// windows neither overlap nor leave gaps. +func (plan BulkPlan) Windows() []Window { + var windows []Window + for since := plan.Since; since.Before(plan.Until); since = since.Add(plan.Window) { + until := since.Add(plan.Window) + if until.After(plan.Until) { + until = plan.Until + } + windows = append(windows, Window{since, until}) + } + return windows +} + +// Fits reports whether the whole range can be fetched in one request. +func (plan BulkPlan) Fits() bool { return len(plan.Windows()) == 1 } + +// RangeError explains why the range needs --split. +func (plan BulkPlan) RangeError() error { + return fmt.Errorf("range %s..%s exceeds the API limit of %s per bulk request at %s; shorten it or add --split to fetch it in %d requests", + plan.Since.Format(time.DateOnly), plan.Until.Format(time.DateOnly), days(plan.Window), plan.Interval, len(plan.Windows())) +} + +// Describe summarises the split for a warning to the user. +func (plan BulkPlan) Describe() string { + return fmt.Sprintf("splitting %s..%s into %d requests of up to %s (%s); the result is assembled in memory", + plan.Since.Format(time.DateOnly), plan.Until.Format(time.DateOnly), len(plan.Windows()), days(plan.Window), plan.Interval) +} + +// WindowParams returns params with s and u set to the window's bounds. +func WindowParams(params map[string]string, w Window) map[string]string { + out := make(map[string]string, len(params)+2) + for key, value := range params { + out[key] = value + } + out["s"] = strconv.FormatInt(w.Since.Unix(), 10) + out["u"] = strconv.FormatInt(w.Until.Unix(), 10) + return out +} + +// GetMetricBulkSplit fetches a bulk metric window by window and merges the +// points. If the API rejects a window as too large, the window is halved for +// the rest of the range, up to maxWindowShrinks times. Any other failure +// aborts the whole fetch: a partial result would look complete to a script. +func (c *Client) GetMetricBulkSplit(ctx context.Context, path string, params map[string]string, repeated map[string][]string, plan BulkPlan) (*BulkResponse, error) { + merged := &BulkResponse{} + seen := map[int64]bool{} + shrinks := 0 + since := plan.Since + for since.Before(plan.Until) { + windows := BulkPlan{Since: since, Until: plan.Until, Interval: plan.Interval, Window: plan.Window}.Windows() + w := windows[0] + resp, err := c.GetMetricBulk(ctx, path, WindowParams(params, w), repeated) + if err != nil { + if isRangeTooLarge(err) && shrinks < maxWindowShrinks { + shrinks++ + plan.Window /= 2 + continue + } + done := len(plan.Windows()) - len(windows) + return nil, fmt.Errorf("window %d/%d (%s..%s): %w; the fetch is aborted, retry it with --since %s", + done+1, len(plan.Windows()), w.Since.Format(time.DateOnly), w.Until.Format(time.DateOnly), err, w.Since.Format(time.DateOnly)) + } + for _, point := range resp.Data { + if seen[point.T] { + continue + } + seen[point.T] = true + merged.Data = append(merged.Data, point) + } + since = w.Until + } + if merged.Data == nil { + merged.Data = []BulkDataPoint{} + } + return merged, nil +} + +func isRangeTooLarge(err error) bool { + var apiErr *glassnode.APIError + return errors.As(err, &apiErr) && apiErr.StatusCode == 400 && strings.Contains(apiErr.Detail, "too large time range") +} + +func days(d time.Duration) string { + return fmt.Sprintf("%d days", int(d.Hours()/24)) +} diff --git a/internal/api/bulk_split_test.go b/internal/api/bulk_split_test.go new file mode 100644 index 0000000..0bb6ad1 --- /dev/null +++ b/internal/api/bulk_split_test.go @@ -0,0 +1,184 @@ +package api + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" +) + +var ( + day = 24 * time.Hour + planNow = time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC) + planSince = time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) +) + +func unix(t time.Time) string { return strconv.FormatInt(t.Unix(), 10) } + +func TestPlanBulk(t *testing.T) { + t.Run("requires since", func(t *testing.T) { + _, err := PlanBulk(map[string]string{}, planNow) + if err == nil || !strings.Contains(err.Error(), "--since") || !strings.Contains(err.Error(), "31 days") { + t.Fatalf("err = %v", err) + } + }) + t.Run("defaults and window by resolution", func(t *testing.T) { + for interval, want := range map[string]time.Duration{"": 31 * day, "24h": 31 * day, "10m": 10 * day, "1h": 10 * day, "1w": 93 * day, "1month": 93 * day} { + params := map[string]string{"s": unix(planSince)} + if interval != "" { + params["i"] = interval + } + plan, err := PlanBulk(params, planNow) + if err != nil || plan.Window != want || !plan.Until.Equal(planNow) { + t.Errorf("%q: plan=%+v err=%v", interval, plan, err) + } + } + }) + t.Run("until in the future is capped at now", func(t *testing.T) { + plan, err := PlanBulk(map[string]string{"s": unix(planSince), "u": unix(planNow.Add(400 * day))}, planNow) + if err != nil || !plan.Until.Equal(planNow) { + t.Fatalf("plan=%+v err=%v", plan, err) + } + }) + t.Run("since must precede until", func(t *testing.T) { + if _, err := PlanBulk(map[string]string{"s": unix(planNow), "u": unix(planSince)}, planNow); err == nil { + t.Fatal("accepted since >= until") + } + if _, err := PlanBulk(map[string]string{"s": unix(planNow.Add(day))}, planNow); err == nil { + t.Fatal("accepted since in the future") + } + }) +} + +func TestBulkPlanWindows(t *testing.T) { + plan := BulkPlan{Since: planSince, Until: planSince.Add(70 * day), Interval: "24h", Window: 31 * day} + windows := plan.Windows() + if len(windows) != 3 || plan.Fits() { + t.Fatalf("windows = %v", windows) + } + for i := 1; i < len(windows); i++ { + if !windows[i].Since.Equal(windows[i-1].Until) { + t.Errorf("gap or overlap between %v and %v", windows[i-1], windows[i]) + } + } + if !windows[0].Since.Equal(planSince) || !windows[2].Until.Equal(plan.Until) || windows[2].Until.Sub(windows[2].Since) != 8*day { + t.Errorf("bounds: %v", windows) + } + if !strings.Contains(plan.RangeError().Error(), "3 requests") || !strings.Contains(plan.Describe(), "3 requests of up to 31 days (24h)") { + t.Errorf("messages: %v / %s", plan.RangeError(), plan.Describe()) + } + if one := (BulkPlan{Since: planSince, Until: planSince.Add(day), Window: 31 * day}); !one.Fits() { + t.Error("one-day range should fit") + } +} + +// bulkServer answers one daily BTC point per day of the requested window and +// records the windows it was asked for. Ranges above maxRange get the API's +// "too large time range" error. +func bulkServer(t *testing.T, maxRange time.Duration, failAt int) (*httptest.Server, *atomic.Int32, *[]Window) { + t.Helper() + var calls atomic.Int32 + var windows []Window + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := int(calls.Add(1)) + s, _ := strconv.ParseInt(r.URL.Query().Get("s"), 10, 64) + u, _ := strconv.ParseInt(r.URL.Query().Get("u"), 10, 64) + since, until := time.Unix(s, 0).UTC(), time.Unix(u, 0).UTC() + windows = append(windows, Window{since, until}) + if until.Sub(since) > maxRange { + w.WriteHeader(http.StatusBadRequest) + fmt.Fprintf(w, `{"message":"too large time range (%s) requested (maximum allowed is %s)"}`, until.Sub(since), maxRange) + return + } + if n == failAt { + w.WriteHeader(http.StatusForbidden) + fmt.Fprint(w, `{"message":"metric not in your plan"}`) + return + } + var points []string + for day := since.Truncate(24 * time.Hour); day.Add(24*time.Hour).Compare(until) <= 0; day = day.Add(24 * time.Hour) { + points = append(points, fmt.Sprintf(`{"t":%d,"bulk":[{"a":"BTC","v":1}]}`, day.Unix())) + } + fmt.Fprintf(w, `{"data":[%s]}`, strings.Join(points, ",")) + })) + t.Cleanup(server.Close) + return server, &calls, &windows +} + +func newTestClient(server *httptest.Server) *Client { + client := NewClient("key", "") + client.baseURL = server.URL + client.httpClient = server.Client() + return client +} + +func TestGetMetricBulkSplit(t *testing.T) { + plan := BulkPlan{Since: planSince, Until: planSince.Add(70 * day), Interval: "24h", Window: 31 * day} + server, calls, _ := bulkServer(t, 31*day, 0) + resp, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan) + if err != nil { + t.Fatal(err) + } + if calls.Load() != 3 || len(resp.Data) != 70 { + t.Fatalf("calls=%d points=%d, want 3 and 70", calls.Load(), len(resp.Data)) + } + for i := 1; i < len(resp.Data); i++ { + if resp.Data[i].T-resp.Data[i-1].T != int64(day.Seconds()) { + t.Fatalf("gap or duplicate at %d: %d -> %d", i, resp.Data[i-1].T, resp.Data[i].T) + } + } +} + +func TestGetMetricBulkSplitShrinksWindow(t *testing.T) { + plan := BulkPlan{Since: planSince, Until: planSince.Add(31 * day), Interval: "24h", Window: 31 * day} + // The server allows only 10 days: 31 -> 15.5 -> 7.75 days, two shrinks. + server, calls, windows := bulkServer(t, 10*day, 0) + resp, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan) + if err != nil { + t.Fatal(err) + } + if len(resp.Data) != 31 { + t.Fatalf("points=%d, want 31", len(resp.Data)) + } + if (*windows)[0].Until.Sub((*windows)[0].Since) != 31*day || (*windows)[2].Until.Sub((*windows)[2].Since) > 10*day { + t.Errorf("window sizes: %v", *windows) + } + t.Logf("calls=%d", calls.Load()) + + // A limit below what three halvings reach is an error. + tiny, _, _ := bulkServer(t, 2*day, 0) + if _, err := newTestClient(tiny).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan); err == nil || !strings.Contains(err.Error(), "too large time range") { + t.Fatalf("err = %v", err) + } +} + +func TestGetMetricBulkSplitAbortsOnFailure(t *testing.T) { + plan := BulkPlan{Since: planSince, Until: planSince.Add(70 * day), Interval: "24h", Window: 31 * day} + server, calls, _ := bulkServer(t, 31*day, 2) + resp, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan) + if err == nil || resp != nil { + t.Fatalf("expected an error and no data, got %v %v", resp, err) + } + for _, want := range []string{"window 2/3", "2024-02-01..2024-03-03", "metric not in your plan", "--since 2024-02-01"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q lacks %q", err, want) + } + } + if calls.Load() != 2 { + t.Errorf("calls=%d, want 2 (no requests after the failure)", calls.Load()) + } +} + +func TestWindowParams(t *testing.T) { + params := map[string]string{"s": "1", "u": "2", "i": "24h"} + w := Window{planSince, planSince.Add(day)} + out := WindowParams(params, w) + if out["s"] != unix(planSince) || out["u"] != unix(planSince.Add(day)) || out["i"] != "24h" || params["s"] != "1" { + t.Errorf("out=%v params=%v", out, params) + } +} From e91d84c13a9ef00306240f259c9d534a21624c00 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Thu, 8 Oct 2026 13:05:01 +0200 Subject: [PATCH 05/37] GN-159: drop the dead --format filter from metric list The f filter on /v1/metadata/metrics returns an empty list for any value (checked live: 1819 metrics without it, 0 with f=csv or f=json), so the flag never produced results. With the SDK it would fail with an input error instead, as the SDK only decodes JSON; removing the flag is the honest option. Metric data in CSV is a separate feature (GetMetricCSV in the SDK), not a filter. --- README.md | 1 - cmd/metric_list.go | 4 ---- 2 files changed, 5 deletions(-) diff --git a/README.md b/README.md index c32b1b9..72f39e5 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,6 @@ gn metric list --from-exchange binance --to-exchange coinbase | `--assets` | | Filter by multiple assets (repeat for each, e.g. `--assets BTC --assets ETH`) | | `--currency` | `-c` | Filter by currency (e.g. `native`, `usd`) | | `--exchange` | `-e` | Filter by exchange (e.g. `binance`, `coinbase`) | -| `--format` | `-f` | Filter by response format (e.g. `json`, `csv`) | | `--interval` | `-i` | Filter by time interval (e.g. `1h`, `24h`) | | `--from-exchange` | | Source exchange for inter-exchange metrics | | `--to-exchange` | | Destination exchange for inter-exchange metrics | diff --git a/cmd/metric_list.go b/cmd/metric_list.go index b6ae1fa..5f482b9 100644 --- a/cmd/metric_list.go +++ b/cmd/metric_list.go @@ -64,9 +64,6 @@ func metricListParamsFromFlags(cmd *cobra.Command) (map[string]string, map[strin if v, _ := cmd.Flags().GetString("exchange"); v != "" { params["e"] = v } - if v, _ := cmd.Flags().GetString("format"); v != "" { - params["f"] = v - } if v, _ := cmd.Flags().GetString("interval"); v != "" { params["i"] = v } @@ -100,7 +97,6 @@ func init() { metricListCmd.Flags().StringArray("assets", nil, "filter by assets (multiple, e.g. --assets BTC --assets ETH)") metricListCmd.Flags().StringP("currency", "c", "", "filter by currency (e.g. native, usd)") metricListCmd.Flags().StringP("exchange", "e", "", "filter by exchange (e.g. binance, coinbase)") - metricListCmd.Flags().StringP("format", "f", "", "filter by response format (e.g. json, csv)") metricListCmd.Flags().StringP("interval", "i", "", "filter by time interval (e.g. 1h, 24h)") metricListCmd.Flags().String("from-exchange", "", "source exchange for inter-exchange metrics") metricListCmd.Flags().String("to-exchange", "", "destination exchange for inter-exchange metrics") From bb85344f35e654437648c2c5690a82c8a4025134 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Thu, 8 Oct 2026 13:12:21 +0200 Subject: [PATCH 06/37] GN-159: send the API key in the X-Api-Key header The adapter kept query authentication so --dry-run could print the same URL that was sent. That left the key in the URL of every real request, where proxies and access logs keep it, for the sake of a display detail. Use the SDK's default header authentication instead. --dry-run prints the URL without the key and notes on stderr which header carries the credential, so stdout stays a clean URL for scripts; the redaction helper is no longer needed. --- README.md | 2 +- cmd/asset_describe.go | 3 +- cmd/asset_list.go | 4 +-- cmd/credits.go | 5 +-- cmd/credits_test.go | 7 ++-- cmd/dryrun.go | 16 +++++++++ cmd/metric_describe.go | 5 +-- cmd/metric_get.go | 6 ++-- cmd/metric_get_test.go | 9 +++-- cmd/metric_list.go | 5 +-- cmd/names_list.go | 5 +-- docs/go-sdk-extraction.md | 5 +-- internal/api/api_usage_test.go | 4 +-- internal/api/client.go | 12 +++++-- internal/api/client_test.go | 63 ++++------------------------------ internal/api/metadata.go | 24 ++----------- 16 files changed, 60 insertions(+), 115 deletions(-) create mode 100644 cmd/dryrun.go diff --git a/README.md b/README.md index 72f39e5..d286d71 100644 --- a/README.md +++ b/README.md @@ -296,7 +296,7 @@ OAuth session fields (`oauth-access-token`, `oauth-refresh-token`, `oauth-expire |------|-------|-------------| | `--api-key` | | Glassnode API key | | `--output` | `-o` | Output format: `json` (default), `csv`, `table` | -| `--dry-run` | | Print the request URL without executing | +| `--dry-run` | | Print the request URL without executing; the API key is sent in the `X-Api-Key` header and is not part of the URL | | `--timestamp-format` | | Timestamp format in output | ## Output Formats diff --git a/cmd/asset_describe.go b/cmd/asset_describe.go index c0b8317..b71f8e3 100644 --- a/cmd/asset_describe.go +++ b/cmd/asset_describe.go @@ -33,8 +33,7 @@ var assetDescribeCmd = &cobra.Command{ if err != nil { return err } - redacted, _ := api.RedactAPIKeyFromURL(u) - _, _ = fmt.Println(redacted) + printDryRun(cmd, client, u) return nil } diff --git a/cmd/asset_list.go b/cmd/asset_list.go index 90e2679..7e427c8 100644 --- a/cmd/asset_list.go +++ b/cmd/asset_list.go @@ -1,7 +1,6 @@ package cmd import ( - "fmt" "regexp" "strings" @@ -51,8 +50,7 @@ var assetListCmd = &cobra.Command{ if err != nil { return err } - redacted, _ := api.RedactAPIKeyFromURL(u) - fmt.Println(redacted) + printDryRun(cmd, client, u) return nil } diff --git a/cmd/credits.go b/cmd/credits.go index fe1eeec..bcbbb18 100644 --- a/cmd/credits.go +++ b/cmd/credits.go @@ -1,8 +1,6 @@ package cmd import ( - "fmt" - "github.com/glassnode/glassnode-cli/internal/api" "github.com/glassnode/glassnode-cli/internal/output" "github.com/spf13/cobra" @@ -26,8 +24,7 @@ var creditsCmd = &cobra.Command{ if err != nil { return err } - redacted, _ := api.RedactAPIKeyFromURL(u) - fmt.Println(redacted) + printDryRun(cmd, client, u) return nil } diff --git a/cmd/credits_test.go b/cmd/credits_test.go index 489b661..775d604 100644 --- a/cmd/credits_test.go +++ b/cmd/credits_test.go @@ -19,7 +19,10 @@ func TestCredits_DryRun_RedactsKey(t *testing.T) { if !strings.Contains(stdout, "/v1/user/api_usage") { t.Errorf("stdout should contain path: %s", stdout) } - if !strings.Contains(stdout, "api_key=***") && !strings.Contains(stdout, "api_key=%2A%2A%2A") { - t.Errorf("stdout should contain redacted api_key: %s", stdout) + if strings.Contains(stdout, "api_key") { + t.Errorf("stdout must not contain an api_key parameter: %s", stdout) + } + if !strings.Contains(stderr, "X-Api-Key") { + t.Errorf("stderr should say the key is sent in the X-Api-Key header: %s", stderr) } } diff --git a/cmd/dryrun.go b/cmd/dryrun.go new file mode 100644 index 0000000..bcffdd5 --- /dev/null +++ b/cmd/dryrun.go @@ -0,0 +1,16 @@ +package cmd + +import ( + "fmt" + + "github.com/glassnode/glassnode-cli/internal/api" + "github.com/spf13/cobra" +) + +// printDryRun prints the request URL that --dry-run would send. The credential +// is sent in a header and is not part of the URL, which the note on stderr +// says, so the URL on stdout stays usable in scripts. +func printDryRun(cmd *cobra.Command, client *api.Client, url string) { + fmt.Fprintln(cmd.OutOrStdout(), url) + fmt.Fprintf(cmd.ErrOrStderr(), "# credentials are sent in the %s header, not in the URL\n", client.AuthHeader()) +} diff --git a/cmd/metric_describe.go b/cmd/metric_describe.go index 7ebe0f7..9a14491 100644 --- a/cmd/metric_describe.go +++ b/cmd/metric_describe.go @@ -1,8 +1,6 @@ package cmd import ( - "fmt" - "github.com/glassnode/glassnode-cli/internal/api" "github.com/glassnode/glassnode-cli/internal/output" "github.com/spf13/cobra" @@ -34,8 +32,7 @@ var metricDescribeCmd = &cobra.Command{ if err != nil { return err } - redacted, _ := api.RedactAPIKeyFromURL(u) - fmt.Println(redacted) + printDryRun(cmd, client, u) return nil } diff --git a/cmd/metric_get.go b/cmd/metric_get.go index 32d76cc..f590c51 100644 --- a/cmd/metric_get.go +++ b/cmd/metric_get.go @@ -93,8 +93,7 @@ var metricGetCmd = &cobra.Command{ if err != nil { return err } - redacted, _ := api.RedactAPIKeyFromURL(u) - fmt.Println(redacted) + printDryRun(cmd, client, u) } return nil } @@ -121,8 +120,7 @@ var metricGetCmd = &cobra.Command{ if err != nil { return err } - redacted, _ := api.RedactAPIKeyFromURL(u) - fmt.Println(redacted) + printDryRun(cmd, client, u) return nil } diff --git a/cmd/metric_get_test.go b/cmd/metric_get_test.go index 55dbc08..b55182a 100644 --- a/cmd/metric_get_test.go +++ b/cmd/metric_get_test.go @@ -75,7 +75,7 @@ func runCLI(t *testing.T, env []string, args ...string) (string, string, error) return outBuf.String(), errBuf.String(), runErr } -func TestMetricGet_DryRun_PrintsURLWithRedactedKey(t *testing.T) { +func TestMetricGet_DryRun_PrintsURLWithoutKey(t *testing.T) { stdout, stderr, err := runCLI(t, []string{"HOME=" + t.TempDir()}, "metric", "get", "/market/price_usd_close", "--api-key", "secret-key", @@ -87,8 +87,11 @@ func TestMetricGet_DryRun_PrintsURLWithRedactedKey(t *testing.T) { if strings.Contains(stdout, "secret-key") { t.Errorf("stdout must not contain API key: %s", stdout) } - if !strings.Contains(stdout, "api_key=***") && !strings.Contains(stdout, "api_key=%2A%2A%2A") { - t.Errorf("stdout should contain redacted api_key: %s", stdout) + if strings.Contains(stdout, "api_key") { + t.Errorf("stdout must not contain an api_key parameter: %s", stdout) + } + if !strings.Contains(stderr, "X-Api-Key") { + t.Errorf("stderr should say the key is sent in the X-Api-Key header: %s", stderr) } if !strings.Contains(stdout, "/v1/metrics") || !strings.Contains(stdout, "market/price_usd_close") { t.Errorf("stdout should contain metric path: %s", stdout) diff --git a/cmd/metric_list.go b/cmd/metric_list.go index 5f482b9..9093570 100644 --- a/cmd/metric_list.go +++ b/cmd/metric_list.go @@ -1,8 +1,6 @@ package cmd import ( - "fmt" - "github.com/glassnode/glassnode-cli/internal/api" "github.com/glassnode/glassnode-cli/internal/output" "github.com/spf13/cobra" @@ -27,8 +25,7 @@ var metricListCmd = &cobra.Command{ if err != nil { return err } - redacted, _ := api.RedactAPIKeyFromURL(u) - fmt.Println(redacted) + printDryRun(cmd, client, u) return nil } diff --git a/cmd/names_list.go b/cmd/names_list.go index c49c830..3a0bd28 100644 --- a/cmd/names_list.go +++ b/cmd/names_list.go @@ -1,8 +1,6 @@ package cmd import ( - "fmt" - "github.com/glassnode/glassnode-cli/internal/api" "github.com/glassnode/glassnode-cli/internal/output" "github.com/spf13/cobra" @@ -34,8 +32,7 @@ func runNamesListCommand(cmd *cobra.Command, endpoint string, withFilter bool) e if err != nil { return err } - redacted, _ := api.RedactAPIKeyFromURL(u) - fmt.Println(redacted) + printDryRun(cmd, client, u) return nil } diff --git a/docs/go-sdk-extraction.md b/docs/go-sdk-extraction.md index 082594c..0782018 100644 --- a/docs/go-sdk-extraction.md +++ b/docs/go-sdk-extraction.md @@ -8,8 +8,9 @@ module. The design and TS feature mapping live in that repository's `internal/api` is a CLI adapter. It retains environment/config resolution, OAuth refresh, dry-run URL rendering, asset pruning, credit presentation and historical output shapes. Endpoint HTTP handling, retries and decoding are -provided by the SDK. The SDK defaults to header authentication; the CLI -explicitly retains query API keys to preserve current dry-run and wire behavior. +provided by the SDK. The API key is sent in the +X-Api-Key header, as the SDK does by default, so it stays out of URLs and +access logs; --dry-run prints the URL without it and notes the header on stderr. OAuth uses the SDK token-source API alongside the CLI refresh-on-401 transport. SDK clients are initialized lazily once per CLI adapter and reused. diff --git a/internal/api/api_usage_test.go b/internal/api/api_usage_test.go index 566a4c9..f1a8af6 100644 --- a/internal/api/api_usage_test.go +++ b/internal/api/api_usage_test.go @@ -20,7 +20,7 @@ func TestGetAPIUsage(t *testing.T) { var gotPath, gotAPIKey string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { gotPath = r.URL.Path - gotAPIKey = r.URL.Query().Get("api_key") + gotAPIKey = r.Header.Get("X-Api-Key") w.Header().Set("Content-Type", "application/json") _, _ = w.Write(fixture) })) @@ -39,7 +39,7 @@ func TestGetAPIUsage(t *testing.T) { t.Errorf("path = %q, want /v1/user/api_usage", gotPath) } if gotAPIKey != "my-key" { - t.Errorf("api_key = %q, want my-key", gotAPIKey) + t.Errorf("X-Api-Key = %q, want my-key", gotAPIKey) } if out.CreditsUsed != 6 { t.Errorf("CreditsUsed = %d, want 6", out.CreditsUsed) diff --git a/internal/api/client.go b/internal/api/client.go index bd53e5e..9d49c3f 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -59,8 +59,6 @@ func (c *Client) sdk() (*glassnode.Client, error) { httpClient.Transport = auth options = append(options, glassnode.WithTokenSource(auth)) key = "" - } else { - options = append(options, glassnode.WithAPIKeyInQuery()) } options = append(options, glassnode.WithHTTPClient(&httpClient)) c.sdkClient, c.initErr = glassnode.NewClient(key, options...) @@ -68,6 +66,16 @@ func (c *Client) sdk() (*glassnode.Client, error) { return c.sdkClient, c.initErr } +// AuthHeader names the header that carries the credential. The SDK sends the +// API key in X-Api-Key rather than in the URL, so it stays out of proxy and +// access logs; dry-run output therefore shows the URL without it. +func (c *Client) AuthHeader() string { + if c.bearerToken != "" { + return "Authorization: Bearer" + } + return "X-Api-Key" +} + // oauthTransport preserves the CLI's refresh-on-401 behavior without teaching // the SDK about config files or OAuth session persistence. type oauthTransport struct { diff --git a/internal/api/client_test.go b/internal/api/client_test.go index 7fc0e8e..e7888a4 100644 --- a/internal/api/client_test.go +++ b/internal/api/client_test.go @@ -174,9 +174,10 @@ func TestResolveAuth_RefreshesExpiredAccessToken(t *testing.T) { } func TestDo_SendsCorrectURL(t *testing.T) { - var capturedURL string + var capturedURL, capturedKey string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { capturedURL = r.URL.String() + capturedKey = r.Header.Get("X-Api-Key") w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("{}")) })) @@ -190,8 +191,8 @@ func TestDo_SendsCorrectURL(t *testing.T) { if err != nil { t.Fatalf("Do: %v", err) } - if !strings.Contains(capturedURL, "api_key=my-api-key") { - t.Errorf("URL %q missing api_key param", capturedURL) + if capturedKey != "my-api-key" || strings.Contains(capturedURL, "api_key") { + t.Errorf("URL %q header %q: the key must travel in X-Api-Key only", capturedURL, capturedKey) } if !strings.Contains(capturedURL, "a=b") { t.Errorf("URL %q missing a=b param", capturedURL) @@ -376,8 +377,8 @@ func TestBuildURL(t *testing.T) { if err != nil { t.Fatalf("BuildURL: %v", err) } - if !strings.Contains(got, "api_key=test-key") { - t.Errorf("URL %q missing api_key", got) + if strings.Contains(got, "api_key") || strings.Contains(got, "test-key") { + t.Errorf("URL %q must not contain the API key", got) } if !strings.Contains(got, "p=v") { t.Errorf("URL %q missing p=v", got) @@ -387,24 +388,6 @@ func TestBuildURL(t *testing.T) { } } -func TestRedactAPIKeyFromURL(t *testing.T) { - raw := "https://api.example.com/v1/path?api_key=secret123&a=b" - redacted, err := RedactAPIKeyFromURL(raw) - if err != nil { - t.Fatalf("RedactAPIKeyFromURL: %v", err) - } - if strings.Contains(redacted, "secret123") { - t.Errorf("redacted URL should not contain secret: %q", redacted) - } - // Placeholder may be URL-encoded as %2A%2A%2A - if !strings.Contains(redacted, "api_key=***") && !strings.Contains(redacted, "api_key=%2A%2A%2A") { - t.Errorf("redacted URL should contain api_key redaction: %q", redacted) - } - if !strings.Contains(redacted, "a=b") { - t.Errorf("redacted URL should preserve other params: %q", redacted) - } -} - func TestListAssets(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -796,40 +779,6 @@ func TestDo_4xxReturnsErrorWithBody(t *testing.T) { } } -// RedactAPIKeyFromURL edge cases (point 7) - -func TestRedactAPIKeyFromURL_NoAPIKeyParam(t *testing.T) { - raw := "https://api.example.com/v1/path?a=b" - redacted, err := RedactAPIKeyFromURL(raw) - if err != nil { - t.Fatalf("RedactAPIKeyFromURL: %v", err) - } - if redacted != raw { - t.Errorf("URL without api_key should be unchanged: got %q", redacted) - } -} - -func TestRedactAPIKeyFromURL_EmptyAPIKey(t *testing.T) { - raw := "https://api.example.com/v1/path?api_key=&a=b" - redacted, err := RedactAPIKeyFromURL(raw) - if err != nil { - t.Fatalf("RedactAPIKeyFromURL: %v", err) - } - if !strings.Contains(redacted, "a=b") { - t.Errorf("should preserve other params: %q", redacted) - } - if !strings.Contains(redacted, "api_key=") { - t.Errorf("should still have api_key param: %q", redacted) - } -} - -func TestRedactAPIKeyFromURL_InvalidURL(t *testing.T) { - _, err := RedactAPIKeyFromURL("://invalid") - if err == nil { - t.Error("expected error for invalid URL") - } -} - func TestGetMetricBulk_KeepsAllSelectors(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/internal/api/metadata.go b/internal/api/metadata.go index 267a328..11bd94f 100644 --- a/internal/api/metadata.go +++ b/internal/api/metadata.go @@ -91,32 +91,14 @@ func (c *Client) DescribeMetric(ctx context.Context, path, asset string) (*Metri return client.GetMetricMetadata(ctx, path, &glassnode.MetricParams{Asset: asset}) } -// BuildURL constructs the full request URL without executing the request. +// BuildURL constructs the request URL without executing the request. The +// credential is not part of it: it travels in the header named by AuthHeader. func (c *Client) BuildURL(path string, params map[string]string, repeatedParams map[string][]string) (string, error) { u, err := url.Parse(c.baseURL + path) if err != nil { return "", err } - q := queryValues(params, repeatedParams) - if c.bearerToken == "" { - q.Set("api_key", c.apiKey) - } - u.RawQuery = q.Encode() - return u.String(), nil -} - -// RedactAPIKeyFromURL returns a copy of the URL with the api_key query parameter -// replaced by a placeholder, for safe display (e.g. dry-run output). -func RedactAPIKeyFromURL(raw string) (string, error) { - u, err := url.Parse(raw) - if err != nil { - return "", err - } - q := u.Query() - if q.Has("api_key") { - q.Set("api_key", "***") - u.RawQuery = q.Encode() - } + u.RawQuery = queryValues(params, repeatedParams).Encode() return u.String(), nil } From 78b276fad0655660c7bd0e312b2bce19eacfda01 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Thu, 8 Oct 2026 13:23:02 +0200 Subject: [PATCH 07/37] GN-159: refresh OAuth tokens through the SDK's TokenRefresher The CLI wrapped the HTTP transport to catch 401, refresh the session and resend the request itself. Two layers set Authorization, the repeat bypassed the SDK's timeout and size limits, and the SDK never learned the new token, so it could not redact it. The SDK now asks a TokenRefresher for a new token once after a 401 and repeats the request itself; the transport, its redaction helper and their tests go away, and sessionTokens is a Token/Refresh pair over the OAuth session. Pin the SDK to the review branch head that adds TokenRefresher. --- docs/go-sdk-extraction.md | 3 +- go.mod | 2 +- go.sum | 6 +-- internal/api/client.go | 87 ++++++++++--------------------------- internal/api/client_test.go | 10 ----- 5 files changed, 27 insertions(+), 81 deletions(-) diff --git a/docs/go-sdk-extraction.md b/docs/go-sdk-extraction.md index 0782018..623bdaf 100644 --- a/docs/go-sdk-extraction.md +++ b/docs/go-sdk-extraction.md @@ -11,7 +11,8 @@ historical output shapes. Endpoint HTTP handling, retries and decoding are provided by the SDK. The API key is sent in the X-Api-Key header, as the SDK does by default, so it stays out of URLs and access logs; --dry-run prints the URL without it and notes the header on stderr. -OAuth uses the SDK token-source API alongside the CLI refresh-on-401 transport. +OAuth sessions reach the SDK through its TokenRefresher interface: the CLI +supplies the current token and refreshes it once when the API answers 401. SDK clients are initialized lazily once per CLI adapter and reused. GET calls now retry transport/read failures, 429 and 5xx responses twice with diff --git a/go.mod b/go.mod index 1b54123..559327f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/glassnode/glassnode-cli go 1.24.0 require ( - github.com/glassnode/glassnode-api-go-client v0.1.1-0.20261007130113-3cf15fec37f3 + github.com/glassnode/glassnode-api-go-client v0.1.1-0.20261008112157-0d4f7d416871 github.com/olekukonko/tablewriter v0.0.5 github.com/spf13/cobra v1.10.2 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index ad24620..c0bc2ae 100644 --- a/go.sum +++ b/go.sum @@ -4,10 +4,8 @@ github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuh github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/glassnode/glassnode-api-go-client v0.1.0 h1:bhPaVNe3MDgh/hr7XKgnh4OvVkKlobE4U4zGdxTWu5A= -github.com/glassnode/glassnode-api-go-client v0.1.0/go.mod h1:Jwk/v/xgvMtTXLMV/WnnVa+3ycFVZiBR4EgU8/JdcbQ= -github.com/glassnode/glassnode-api-go-client v0.1.1-0.20261007130113-3cf15fec37f3 h1:1D4TCb69qYrYwyQr3FglHOhQfaw2xqeybVUKkq7MJ6Y= -github.com/glassnode/glassnode-api-go-client v0.1.1-0.20261007130113-3cf15fec37f3/go.mod h1:Jwk/v/xgvMtTXLMV/WnnVa+3ycFVZiBR4EgU8/JdcbQ= +github.com/glassnode/glassnode-api-go-client v0.1.1-0.20261008112157-0d4f7d416871 h1:AQ3WDvtC6BGK8EU4+ouEXtwfp+CXfYg9AZ1V66g6gho= +github.com/glassnode/glassnode-api-go-client v0.1.1-0.20261008112157-0d4f7d416871/go.mod h1:Jwk/v/xgvMtTXLMV/WnnVa+3ycFVZiBR4EgU8/JdcbQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= diff --git a/internal/api/client.go b/internal/api/client.go index 9d49c3f..37c00fa 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -1,7 +1,6 @@ package api import ( - "bytes" "context" "errors" "fmt" @@ -9,7 +8,6 @@ import ( "net/http" "net/url" "os" - "strings" "sync" "time" @@ -51,13 +49,7 @@ func (c *Client) sdk() (*glassnode.Client, error) { } key := c.apiKey if c.bearerToken != "" { - transport := httpClient.Transport - if transport == nil { - transport = http.DefaultTransport - } - auth := &oauthTransport{base: transport, token: c.bearerToken} - httpClient.Transport = auth - options = append(options, glassnode.WithTokenSource(auth)) + options = append(options, glassnode.WithTokenSource(&sessionTokens{token: c.bearerToken})) key = "" } options = append(options, glassnode.WithHTTPClient(&httpClient)) @@ -76,74 +68,39 @@ func (c *Client) AuthHeader() string { return "X-Api-Key" } -// oauthTransport preserves the CLI's refresh-on-401 behavior without teaching -// the SDK about config files or OAuth session persistence. -type oauthTransport struct { - base http.RoundTripper +// sessionTokens hands the CLI's OAuth session token to the SDK and refreshes +// it when the API rejects it, so a token revoked before its expiry is replaced +// once without teaching the SDK about config files or OAuth persistence. +type sessionTokens struct { mu sync.Mutex token string } -// Token supplies the current session token to the SDK on each attempt. -func (t *oauthTransport) Token(ctx context.Context) (string, error) { +// Token returns the current session token. +func (s *sessionTokens) Token(ctx context.Context) (string, error) { if err := ctx.Err(); err != nil { return "", err } - t.mu.Lock() - defer t.mu.Unlock() - return t.token, nil + s.mu.Lock() + defer s.mu.Unlock() + return s.token, nil } -func (t *oauthTransport) RoundTrip(req *http.Request) (*http.Response, error) { - t.mu.Lock() - token := t.token - t.mu.Unlock() - request := req.Clone(req.Context()) - request.Header.Set("Authorization", "Bearer "+token) - response, err := t.base.RoundTrip(request) - if err != nil || response.StatusCode != http.StatusUnauthorized { - return redactOAuthErrorResponse(response, token), err - } - t.mu.Lock() - fresh := t.token - var refreshErr error - if fresh == token { - fresh, refreshErr = oauth.ForceRefreshAccessToken(req.Context()) - if refreshErr == nil && fresh != "" { - t.token = fresh - } - } - t.mu.Unlock() - if refreshErr != nil || fresh == "" || fresh == token { - if refreshErr != nil && !errors.Is(refreshErr, oauth.ErrSessionExpired) { +// Refresh exchanges the stored refresh token for a new access token after a +// 401 and keeps it for the following requests. An expired session is reported +// to the user through the SDK's AuthError. +func (s *sessionTokens) Refresh(ctx context.Context) (string, error) { + fresh, err := oauth.ForceRefreshAccessToken(ctx) + if err != nil { + if !errors.Is(err, oauth.ErrSessionExpired) { _, _ = fmt.Fprintln(stderr, "warning: OAuth refresh after HTTP 401 failed") } - return response, nil - } - _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096)) - _ = response.Body.Close() - request = req.Clone(req.Context()) - request.Header.Set("Authorization", "Bearer "+fresh) - response, err = t.base.RoundTrip(request) - return redactOAuthErrorResponse(response, fresh), err -} - -// The SDK knows the original token; the CLI transport also redacts a rotated -// token if an error body echoes it after refresh. -func redactOAuthErrorResponse(response *http.Response, token string) *http.Response { - if response == nil || response.StatusCode < 300 || token == "" { - return response - } - body, err := io.ReadAll(io.LimitReader(response.Body, 4096)) - _ = response.Body.Close() - if err != nil { - body = []byte("failed to read OAuth API error response") + return "", err } - text := strings.ReplaceAll(string(body), token, "[redacted]") - text = strings.ReplaceAll(text, url.QueryEscape(token), "[redacted]") - response.Body = io.NopCloser(bytes.NewBufferString(text)) - response.ContentLength = int64(len(text)) - return response + s.mu.Lock() + s.token = fresh + s.mu.Unlock() + return fresh, nil } func queryValues(params map[string]string, repeated map[string][]string) url.Values { diff --git a/internal/api/client_test.go b/internal/api/client_test.go index e7888a4..7934ade 100644 --- a/internal/api/client_test.go +++ b/internal/api/client_test.go @@ -3,7 +3,6 @@ package api import ( "context" _ "embed" - "io" "net/http" "net/http/httptest" "os" @@ -288,15 +287,6 @@ func TestDo_401TriggersRefreshAndRetry(t *testing.T) { }) } -func TestOAuthErrorResponseRedactsRotatedToken(t *testing.T) { - response := &http.Response{StatusCode: 401, Body: io.NopCloser(strings.NewReader("rotated-secret"))} - response = redactOAuthErrorResponse(response, "rotated-secret") - body, err := io.ReadAll(response.Body) - if err != nil || strings.Contains(string(body), "rotated-secret") { - t.Fatalf("unredacted response: %s %v", body, err) - } -} - func TestDo_401WithAPIKey_NoRetry(t *testing.T) { // When using an API key (not a bearer), a 401 must NOT trigger a refresh attempt; we just // surface the error. This guards against silently attempting OAuth refreshes for users who From e627c961af862138220fa3029122a935e08402f6 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Thu, 8 Oct 2026 13:32:00 +0200 Subject: [PATCH 08/37] GN-159: show computed_at and plain decimal numbers in output Point-in-time metrics report computed_at for every value; the CLI dropped it. DataPoint carries it now, JSON includes it and CSV/table add a trailing column when any point has it. CSV and table rendered large values with %v, which prints 1.7382414210210708e+12 for a market cap. Format floats in plain decimal notation so they load into spreadsheets as numbers; JSON output is unchanged. --- README.md | 2 + internal/api/metrics.go | 2 + internal/output/output.go | 84 ++++++++++++++++++++++++++++------ internal/output/output_test.go | 46 +++++++++++++++++++ 4 files changed, 121 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index d286d71..3dff46d 100644 --- a/README.md +++ b/README.md @@ -250,6 +250,8 @@ gn metric get market/marketcap_usd/bulk -a BTC -a ETH --since 2024-01-01 --split For **bulk metrics**, append `/bulk` to the path (e.g. `market/marketcap_usd/bulk`). To pass multiple assets, repeat the `-a` (or `--asset`) flag for each: `-a BTC -a ETH -a SOL`. Use `-a '*'` to request all assets. +Point-in-time metrics (`*_pit`) include `computed_at`, the time the value was computed, as a field in JSON and as a trailing column in CSV and table output. + The API requires `--since` for bulk metrics and accepts at most 10 days per request at `10m`/`1h`, 31 days at `24h` and 93 days at `1w`/`1month`. A longer range is rejected before any request is sent unless you pass `--split`: the CLI then fetches it window by window, prints how many requests that takes on stderr, and merges the points. Every request is billed per asset, so narrow `-a` for long ranges. The merged result is assembled in memory before it is printed. ### `gn user credits` diff --git a/internal/api/metrics.go b/internal/api/metrics.go index 1360bd0..b2e2c0c 100644 --- a/internal/api/metrics.go +++ b/internal/api/metrics.go @@ -14,6 +14,8 @@ type DataPoint struct { T int64 `json:"t"` V interface{} `json:"v,omitempty"` O map[string]interface{} `json:"o,omitempty"` + // ComputedAt is set by point-in-time metrics: when the value was computed. + ComputedAt *int64 `json:"computed_at,omitempty"` } type BulkDataPoint struct { diff --git a/internal/output/output.go b/internal/output/output.go index bf34f2f..45d6bf9 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -7,6 +7,7 @@ import ( "io" "os" "sort" + "strconv" "strings" "time" @@ -110,27 +111,42 @@ func PrintCSV(w io.Writer, data interface{}, timestampFormat string) error { cw.Flush() return cw.Error() } + computed := hasComputedAt(v) if v[0].O != nil { keys := sortedKeys(v[0].O) header := append([]string{"t"}, keys...) + if computed { + header = append(header, "computed_at") + } if err := write(header); err != nil { return err } for _, dp := range v { row := []string{formatTimestamp(dp.T, timestampFormat)} for _, k := range keys { - row = append(row, fmt.Sprintf("%v", dp.O[k])) + row = append(row, valueStr(dp.O[k])) + } + if computed { + row = append(row, computedAtStr(dp, timestampFormat)) } if err := write(row); err != nil { return err } } } else { - if err := write([]string{"t", "v"}); err != nil { + header := []string{"t", "v"} + if computed { + header = append(header, "computed_at") + } + if err := write(header); err != nil { return err } for _, dp := range v { - if err := write([]string{formatTimestamp(dp.T, timestampFormat), fmt.Sprintf("%v", dp.V)}); err != nil { + row := []string{formatTimestamp(dp.T, timestampFormat), valueStr(dp.V)} + if computed { + row = append(row, computedAtStr(dp, timestampFormat)) + } + if err := write(row); err != nil { return err } } @@ -152,7 +168,7 @@ func PrintCSV(w io.Writer, data interface{}, timestampFormat string) error { for _, entry := range dp.Bulk { row := []string{formatTimestamp(dp.T, timestampFormat)} for _, k := range keys { - row = append(row, fmt.Sprintf("%v", entry[k])) + row = append(row, valueStr(entry[k])) } if err := write(row); err != nil { return err @@ -201,21 +217,36 @@ func PrintTable(w io.Writer, data interface{}, timestampFormat string) error { if len(v) == 0 { return nil } + computed := hasComputedAt(v) if v[0].O != nil { keys := sortedKeys(v[0].O) header := append([]string{"TIME"}, upperAll(keys)...) + if computed { + header = append(header, "COMPUTED AT") + } table.SetHeader(header) for _, dp := range v { row := []string{formatTimestamp(dp.T, timestampFormat)} for _, k := range keys { - row = append(row, fmt.Sprintf("%v", dp.O[k])) + row = append(row, valueStr(dp.O[k])) + } + if computed { + row = append(row, computedAtStr(dp, timestampFormat)) } table.Append(row) } } else { - table.SetHeader([]string{"TIME", "VALUE"}) + header := []string{"TIME", "VALUE"} + if computed { + header = append(header, "COMPUTED AT") + } + table.SetHeader(header) for _, dp := range v { - table.Append([]string{formatTimestamp(dp.T, timestampFormat), fmt.Sprintf("%v", dp.V)}) + row := []string{formatTimestamp(dp.T, timestampFormat), valueStr(dp.V)} + if computed { + row = append(row, computedAtStr(dp, timestampFormat)) + } + table.Append(row) } } case *api.BulkResponse: @@ -232,7 +263,7 @@ func PrintTable(w io.Writer, data interface{}, timestampFormat string) error { for _, entry := range dp.Bulk { row := []string{formatTimestamp(dp.T, timestampFormat)} for _, k := range keys { - row = append(row, fmt.Sprintf("%v", entry[k])) + row = append(row, valueStr(entry[k])) } table.Append(row) } @@ -282,15 +313,42 @@ func sortedKeys(m map[string]interface{}) []string { return keys } -// valueStr formats a value for CSV/table. Slices of strings are joined with ";". +// valueStr formats a value for CSV/table. Numbers are written in plain +// decimal notation (1738241421021.0708, not 1.7382414210210708e+12), so they +// load into spreadsheets as numbers; slices of strings are joined with ";". func valueStr(v interface{}) string { - if v == nil { + switch x := v.(type) { + case nil: return "" + case []string: + return strings.Join(x, ";") + case float64: + return strconv.FormatFloat(x, 'f', -1, 64) + case float32: + return strconv.FormatFloat(float64(x), 'f', -1, 32) + case json.Number: + return x.String() + default: + return fmt.Sprintf("%v", v) + } +} + +// hasComputedAt reports whether any point carries computed_at, in which case +// the column is shown for all of them. +func hasComputedAt(points []api.DataPoint) bool { + for _, dp := range points { + if dp.ComputedAt != nil { + return true + } } - if ss, ok := v.([]string); ok { - return strings.Join(ss, ";") + return false +} + +func computedAtStr(dp api.DataPoint, timestampFormat string) string { + if dp.ComputedAt == nil { + return "" } - return fmt.Sprintf("%v", v) + return formatTimestamp(*dp.ComputedAt, timestampFormat) } func upperAll(ss []string) []string { diff --git a/internal/output/output_test.go b/internal/output/output_test.go index 3f8df4b..a0399c4 100644 --- a/internal/output/output_test.go +++ b/internal/output/output_test.go @@ -325,3 +325,49 @@ func TestPrintTable_MetricMetadataWithOptionalFields(t *testing.T) { t.Errorf("table should include Parameters: %q", out) } } + +func TestPrintCSV_NumbersArePlainDecimal(t *testing.T) { + var buf bytes.Buffer + data := []api.DataPoint{{T: 1, V: 1738241421021.0708}, {T: 2, V: 0.000001}} + if err := PrintCSV(&buf, data, "unix"); err != nil { + t.Fatal(err) + } + want := "t,v\n1,1738241421021.0708\n2,0.000001\n" + if got := buf.String(); got != want { + t.Errorf("got %q, want %q", got, want) + } + var table bytes.Buffer + if err := PrintTable(&table, data, "unix"); err != nil { + t.Fatal(err) + } + if strings.Contains(table.String(), "e+") { + t.Errorf("table uses scientific notation: %s", table.String()) + } +} + +func TestPrintCSV_ComputedAtColumn(t *testing.T) { + computed := int64(1791244932) + var buf bytes.Buffer + data := []api.DataPoint{{T: 1791158400, V: 664280.0, ComputedAt: &computed}, {T: 1791244800, V: 640332.0}} + if err := PrintCSV(&buf, data, "unix"); err != nil { + t.Fatal(err) + } + want := "t,v,computed_at\n1791158400,664280,1791244932\n1791244800,640332,\n" + if got := buf.String(); got != want { + t.Errorf("got %q, want %q", got, want) + } + var table bytes.Buffer + if err := PrintTable(&table, data, "unix"); err != nil { + t.Fatal(err) + } + if !strings.Contains(table.String(), "COMPUTED AT") || !strings.Contains(table.String(), "1791244932") { + t.Errorf("table lacks computed_at: %s", table.String()) + } + var plain bytes.Buffer + if err := PrintCSV(&plain, []api.DataPoint{{T: 1, V: 2.0}}, "unix"); err != nil { + t.Fatal(err) + } + if strings.Contains(plain.String(), "computed_at") { + t.Errorf("computed_at column shown without PIT data: %q", plain.String()) + } +} From 6510e1c53df018d8301199343d149774e791aac6 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Thu, 8 Oct 2026 16:02:19 +0200 Subject: [PATCH 09/37] GN-159: pin SDK v0.1.0 and drop the private-dependency setup The SDK's first release is tagged and the repository is being made public, so the CLI no longer needs GOPRIVATE, the GLASSNODE_SDK_READ_TOKEN secret or the authentication steps in CI and release workflows; the README section about private source builds and the feature-branch CI trigger go with them. Pin the released v0.1.0 instead of the review branch pseudo-version. The tag replaces a private v0.1.0 that was never published; a module cache that holds the old one reports a checksum mismatch and needs `go clean -modcache`. --- .github/workflows/release.yml | 14 ----------- .github/workflows/test.yml | 21 +--------------- README.md | 13 ---------- docs/go-sdk-extraction.md | 45 ++++++++++++++++------------------- go.mod | 2 +- go.sum | 4 ++-- 6 files changed, 24 insertions(+), 75 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 80948c8..665f59b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,8 +11,6 @@ permissions: jobs: release: runs-on: ubuntu-latest - env: - GOPRIVATE: github.com/glassnode/glassnode-api-go-client steps: - uses: actions/checkout@v4 with: @@ -22,21 +20,9 @@ jobs: with: go-version: '1.24' - - name: Authenticate private SDK dependency - env: - GH_TOKEN: ${{ secrets.GLASSNODE_SDK_READ_TOKEN }} - run: | - if [ -z "$GH_TOKEN" ]; then - echo "GLASSNODE_SDK_READ_TOKEN must grant read access to glassnode/glassnode-api-go-client." - exit 1 - fi - gh auth setup-git - go mod download - - uses: goreleaser/goreleaser-action@v6 with: version: '~> v2' args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_TOKEN: ${{ secrets.GLASSNODE_SDK_READ_TOKEN }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 67bd7e3..544b36d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,14 +2,12 @@ name: CI on: push: - branches: [main, 'feat/GN-159-extract-go-api-client'] + branches: [main] pull_request: jobs: test: runs-on: ubuntu-latest - env: - GOPRIVATE: github.com/glassnode/glassnode-api-go-client steps: - uses: actions/checkout@v4 @@ -17,28 +15,11 @@ jobs: with: go-version: '1.24' - - name: Authenticate private SDK dependency - env: - GH_TOKEN: ${{ secrets.GLASSNODE_SDK_READ_TOKEN }} - run: | - if [ -z "$GH_TOKEN" ]; then - echo "GLASSNODE_SDK_READ_TOKEN must grant read access to glassnode/glassnode-api-go-client. Fork PRs require a trusted maintainer build." - exit 1 - fi - gh auth setup-git - go mod download - - name: Vet - env: - GH_TOKEN: ${{ secrets.GLASSNODE_SDK_READ_TOKEN }} run: go vet ./... - name: Build - env: - GH_TOKEN: ${{ secrets.GLASSNODE_SDK_READ_TOKEN }} run: go build ./... - name: Test - env: - GH_TOKEN: ${{ secrets.GLASSNODE_SDK_READ_TOKEN }} run: go test -race ./... diff --git a/README.md b/README.md index 3dff46d..8dda5b9 100644 --- a/README.md +++ b/README.md @@ -34,19 +34,6 @@ Download the archive for your OS and architecture from [Releases](https://github ### From source -Source builds use the private `glassnode/glassnode-api-go-client` module. Your -GitHub account must have read access. Authenticate Git with GitHub (for example, -`gh auth setup-git`) and include the module in your existing `GOPRIVATE` list: - -```bash -go env -w GOPRIVATE=github.com/glassnode/glassnode-api-go-client -``` - -Preserve any existing entries in `GOPRIVATE`. Published CLI binaries do not -require GitHub access to run. CI and source releases require the repository -secret `GLASSNODE_SDK_READ_TOKEN`, scoped to read the private SDK repository; -fork pull requests require a trusted maintainer build. - ```bash go install github.com/glassnode/glassnode-cli@latest ``` diff --git a/docs/go-sdk-extraction.md b/docs/go-sdk-extraction.md index 623bdaf..2c54854 100644 --- a/docs/go-sdk-extraction.md +++ b/docs/go-sdk-extraction.md @@ -1,31 +1,26 @@ -# Go SDK extraction — GN-159 +# Go SDK extraction -The reusable API implementation is now in the private +The reusable API implementation lives in the [`glassnode-api-go-client`](https://github.com/glassnode/glassnode-api-go-client) -module. The design and TS feature mapping live in that repository's -`docs/design.md`. Tracking: https://glassnode.atlassian.net/browse/GN-159 +module; its design notes are in that repository's `docs/design.md`. `internal/api` is a CLI adapter. It retains environment/config resolution, -OAuth refresh, dry-run URL rendering, asset pruning, credit presentation and -historical output shapes. Endpoint HTTP handling, retries and decoding are -provided by the SDK. The API key is sent in the -X-Api-Key header, as the SDK does by default, so it stays out of URLs and -access logs; --dry-run prints the URL without it and notes the header on stderr. -OAuth sessions reach the SDK through its TokenRefresher interface: the CLI -supplies the current token and refreshes it once when the API answers 401. -SDK clients are initialized lazily once per CLI adapter and reused. +OAuth login and refresh, dry-run URL rendering, asset pruning, credit +presentation and the historical output shapes. Endpoint HTTP handling, +retries and decoding are provided by the SDK. -GET calls now retry transport/read failures, 429 and 5xx responses twice with -jitter. Context cancellation interrupts calls and backoff. Errors expose SDK -types and redact credentials. Redirects are no longer followed. +- The API key is sent in the `X-Api-Key` header, as the SDK does by default, so + it stays out of URLs and access logs; `--dry-run` prints the URL without it + and notes the header on stderr. +- OAuth sessions reach the SDK through its `TokenRefresher` interface: the CLI + supplies the current token and refreshes it once when the API answers 401. +- SDK clients are initialized lazily once per CLI adapter and reused. +- GET calls retry transport failures, 429 and 5xx responses with jitter and + honour the API's rate-limit reset. Context cancellation interrupts calls and + backoff. Errors expose SDK types and redact credentials. Redirects are not + followed. +- Bulk metrics require `--since`; ranges over the API's per-request limit are + fetched in windows with `--split`. -The CLI remains public while the SDK is private, so source builds need GitHub -read access. Set `GOPRIVATE` and authenticate Git as described in README.md. -CI/release jobs use `GLASSNODE_SDK_READ_TOKEN`; no credential is committed. -The token must be configured before merging this branch. Do not use -`pull_request_target` to expose it to untrusted fork code. - -For local SDK development, create a temporary Go workspace outside either repo -and use both checkouts. If module graph resolution tries to download the -version in the CLI's go.mod, add a version-specific replace to that temporary -workspace. Never commit an absolute filesystem replace into the CLI module. +For local SDK development, use a temporary `go.work` outside both repositories +with both checkouts; never commit a filesystem `replace` into the CLI module. diff --git a/go.mod b/go.mod index 559327f..0461a05 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/glassnode/glassnode-cli go 1.24.0 require ( - github.com/glassnode/glassnode-api-go-client v0.1.1-0.20261008112157-0d4f7d416871 + github.com/glassnode/glassnode-api-go-client v0.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/spf13/cobra v1.10.2 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index c0bc2ae..c6b60e4 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuh github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/glassnode/glassnode-api-go-client v0.1.1-0.20261008112157-0d4f7d416871 h1:AQ3WDvtC6BGK8EU4+ouEXtwfp+CXfYg9AZ1V66g6gho= -github.com/glassnode/glassnode-api-go-client v0.1.1-0.20261008112157-0d4f7d416871/go.mod h1:Jwk/v/xgvMtTXLMV/WnnVa+3ycFVZiBR4EgU8/JdcbQ= +github.com/glassnode/glassnode-api-go-client v0.1.0 h1:qm0XGrdAZJsGYbn7KVoiEjUkGS8YUA+6f0t6CD7C3xM= +github.com/glassnode/glassnode-api-go-client v0.1.0/go.mod h1:Jwk/v/xgvMtTXLMV/WnnVa+3ycFVZiBR4EgU8/JdcbQ= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= From bdead95729e8da55e9375686a28bd0d134ce50ab Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Mon, 12 Oct 2026 19:52:11 +0200 Subject: [PATCH 10/37] GN-159: make the missing --since hint say --split is in addition The hint read as if --split could replace --since, so a user following it got the same error again. --- internal/api/bulk_split.go | 2 +- internal/api/bulk_split_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/api/bulk_split.go b/internal/api/bulk_split.go index b12b6e7..a6f3ff4 100644 --- a/internal/api/bulk_split.go +++ b/internal/api/bulk_split.go @@ -58,7 +58,7 @@ func PlanBulk(params map[string]string, now time.Time) (BulkPlan, error) { plan.Window = BulkWindow(plan.Interval) raw, ok := params["s"] if !ok { - return plan, fmt.Errorf("bulk metrics require --since: the API accepts at most %s per request at %s (add --split to fetch a longer range in several requests)", + return plan, fmt.Errorf("bulk metrics require --since (at most %s per request at %s; for a longer range add --split as well)", days(plan.Window), plan.Interval) } since, err := strconv.ParseInt(raw, 10, 64) diff --git a/internal/api/bulk_split_test.go b/internal/api/bulk_split_test.go index 0bb6ad1..6122159 100644 --- a/internal/api/bulk_split_test.go +++ b/internal/api/bulk_split_test.go @@ -23,7 +23,7 @@ func unix(t time.Time) string { return strconv.FormatInt(t.Unix(), 10) } func TestPlanBulk(t *testing.T) { t.Run("requires since", func(t *testing.T) { _, err := PlanBulk(map[string]string{}, planNow) - if err == nil || !strings.Contains(err.Error(), "--since") || !strings.Contains(err.Error(), "31 days") { + if err == nil || !strings.Contains(err.Error(), "--since") || !strings.Contains(err.Error(), "31 days") || !strings.Contains(err.Error(), "add --split as well") { t.Fatalf("err = %v", err) } }) From 59841c23af8b0d2de40738eded65e14f7dec4272 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Mon, 12 Oct 2026 19:54:02 +0200 Subject: [PATCH 11/37] GN-159: keep the later point if split windows ever repeat a timestamp Adjacent windows share no timestamp, so the dedup is a guard, but should it fire it now keeps the later response, which is at least as fresh, instead of dropping it. --- internal/api/bulk_split.go | 10 +++++++--- internal/api/bulk_split_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/internal/api/bulk_split.go b/internal/api/bulk_split.go index a6f3ff4..ec5cc97 100644 --- a/internal/api/bulk_split.go +++ b/internal/api/bulk_split.go @@ -130,7 +130,10 @@ func WindowParams(params map[string]string, w Window) map[string]string { // aborts the whole fetch: a partial result would look complete to a script. func (c *Client) GetMetricBulkSplit(ctx context.Context, path string, params map[string]string, repeated map[string][]string, plan BulkPlan) (*BulkResponse, error) { merged := &BulkResponse{} - seen := map[int64]bool{} + // Adjacent windows share no timestamp (the API rounds s down and includes + // only intervals ending by u), so index is a guard: should a timestamp + // repeat, the later response, which is at least as fresh, replaces it. + index := map[int64]int{} shrinks := 0 since := plan.Since for since.Before(plan.Until) { @@ -148,10 +151,11 @@ func (c *Client) GetMetricBulkSplit(ctx context.Context, path string, params map done+1, len(plan.Windows()), w.Since.Format(time.DateOnly), w.Until.Format(time.DateOnly), err, w.Since.Format(time.DateOnly)) } for _, point := range resp.Data { - if seen[point.T] { + if i, ok := index[point.T]; ok { + merged.Data[i] = point continue } - seen[point.T] = true + index[point.T] = len(merged.Data) merged.Data = append(merged.Data, point) } since = w.Until diff --git a/internal/api/bulk_split_test.go b/internal/api/bulk_split_test.go index 6122159..751fb49 100644 --- a/internal/api/bulk_split_test.go +++ b/internal/api/bulk_split_test.go @@ -182,3 +182,27 @@ func TestWindowParams(t *testing.T) { t.Errorf("out=%v params=%v", out, params) } } + +// Should windows ever overlap, the later window's point replaces the earlier +// one instead of being dropped. +func TestGetMetricBulkSplitKeepsLaterPointOnOverlap(t *testing.T) { + plan := BulkPlan{Since: planSince, Until: planSince.Add(2 * day), Interval: "24h", Window: day} + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + boundary := planSince.Add(day).Unix() + if calls.Add(1) == 1 { + // The first window wrongly repeats the boundary point with one entry. + fmt.Fprintf(w, `{"data":[{"t":%d,"bulk":[{"a":"BTC","v":1}]},{"t":%d,"bulk":[{"a":"BTC","v":2}]}]}`, planSince.Unix(), boundary) + return + } + fmt.Fprintf(w, `{"data":[{"t":%d,"bulk":[{"a":"BTC","v":2},{"a":"ETH","v":3}]}]}`, boundary) + })) + t.Cleanup(server.Close) + resp, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan) + if err != nil { + t.Fatal(err) + } + if len(resp.Data) != 2 || len(resp.Data[1].Bulk) != 2 { + t.Fatalf("points=%d boundary entries=%d, want 2 and 2 (from the later window)", len(resp.Data), len(resp.Data[1].Bulk)) + } +} From 39e18b62200335689195122fa3deeabc301cc671 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Mon, 12 Oct 2026 19:57:19 +0200 Subject: [PATCH 12/37] GN-159: match the too-large window error case-insensitively Match 'time range' without regard to case so small rewordings of the API message keep the halving fallback working; other 400s still abort with the API's own error. --- internal/api/bulk_split.go | 6 +++++- internal/api/bulk_split_test.go | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/internal/api/bulk_split.go b/internal/api/bulk_split.go index ec5cc97..48e4f29 100644 --- a/internal/api/bulk_split.go +++ b/internal/api/bulk_split.go @@ -166,9 +166,13 @@ func (c *Client) GetMetricBulkSplit(ctx context.Context, path string, params map return merged, nil } +// isRangeTooLarge reports the API's rejection of a window as too long. Other +// 400s (an invalid asset, an unsupported parameter combination) are not +// treated as a reason to halve: that would spend requests and replace the +// API's message with one about window size. func isRangeTooLarge(err error) bool { var apiErr *glassnode.APIError - return errors.As(err, &apiErr) && apiErr.StatusCode == 400 && strings.Contains(apiErr.Detail, "too large time range") + return errors.As(err, &apiErr) && apiErr.StatusCode == 400 && strings.Contains(strings.ToLower(apiErr.Detail), "time range") } func days(d time.Duration) string { diff --git a/internal/api/bulk_split_test.go b/internal/api/bulk_split_test.go index 751fb49..4294e89 100644 --- a/internal/api/bulk_split_test.go +++ b/internal/api/bulk_split_test.go @@ -10,6 +10,8 @@ import ( "sync/atomic" "testing" "time" + + glassnode "github.com/glassnode/glassnode-api-go-client" ) var ( @@ -206,3 +208,20 @@ func TestGetMetricBulkSplitKeepsLaterPointOnOverlap(t *testing.T) { t.Fatalf("points=%d boundary entries=%d, want 2 and 2 (from the later window)", len(resp.Data), len(resp.Data[1].Bulk)) } } + +func TestIsRangeTooLarge(t *testing.T) { + for _, tt := range []struct { + status int + detail string + want bool + }{ + {400, "too large time range (155664h0m0s) requested (maximum allowed is 744h0m0s)", true}, + {400, "Requested Time Range is too large", true}, + {400, "parameter a=NOPE is invalid", false}, + {403, "time range not in your plan", false}, + } { + if got := isRangeTooLarge(&glassnode.APIError{StatusCode: tt.status, Detail: tt.detail}); got != tt.want { + t.Errorf("isRangeTooLarge(%d %q) = %v, want %v", tt.status, tt.detail, got, tt.want) + } + } +} From 3f7a1fb9f737c3a0ba194d7a933acc4595690c2d Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Mon, 12 Oct 2026 20:10:56 +0200 Subject: [PATCH 13/37] GN-159: count fetched windows and keep the time of day in split errors After a halving, window N/M was computed over the shrunk plan and no longer matched the requests made; count fetched windows and estimate the rest. Show bounds as RFC 3339 when they are not at midnight UTC, so the retry hint resumes an hourly or 10-minute fetch exactly. --- internal/api/bulk_split.go | 20 ++++++++++++++++---- internal/api/bulk_split_test.go | 29 ++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/internal/api/bulk_split.go b/internal/api/bulk_split.go index 48e4f29..f42059b 100644 --- a/internal/api/bulk_split.go +++ b/internal/api/bulk_split.go @@ -134,7 +134,7 @@ func (c *Client) GetMetricBulkSplit(ctx context.Context, path string, params map // only intervals ending by u), so index is a guard: should a timestamp // repeat, the later response, which is at least as fresh, replaces it. index := map[int64]int{} - shrinks := 0 + shrinks, fetched := 0, 0 since := plan.Since for since.Before(plan.Until) { windows := BulkPlan{Since: since, Until: plan.Until, Interval: plan.Interval, Window: plan.Window}.Windows() @@ -146,10 +146,11 @@ func (c *Client) GetMetricBulkSplit(ctx context.Context, path string, params map plan.Window /= 2 continue } - done := len(plan.Windows()) - len(windows) - return nil, fmt.Errorf("window %d/%d (%s..%s): %w; the fetch is aborted, retry it with --since %s", - done+1, len(plan.Windows()), w.Since.Format(time.DateOnly), w.Until.Format(time.DateOnly), err, w.Since.Format(time.DateOnly)) + // The total is an estimate: a halved window changes how many remain. + return nil, fmt.Errorf("window %d of about %d (%s..%s): %w; the fetch is aborted, retry it with --since %s", + fetched+1, fetched+len(windows), formatBound(w.Since), formatBound(w.Until), err, formatBound(w.Since)) } + fetched++ for _, point := range resp.Data { if i, ok := index[point.T]; ok { merged.Data[i] = point @@ -175,6 +176,17 @@ func isRangeTooLarge(err error) bool { return errors.As(err, &apiErr) && apiErr.StatusCode == 400 && strings.Contains(strings.ToLower(apiErr.Detail), "time range") } +// formatBound shows a window bound as a date when it is at midnight UTC and +// as an RFC 3339 timestamp otherwise, both of which --since accepts, so a +// retry of an hourly or 10-minute fetch resumes exactly at the bound. +func formatBound(t time.Time) string { + t = t.UTC() + if t.Equal(t.Truncate(24 * time.Hour)) { + return t.Format(time.DateOnly) + } + return t.Format(time.RFC3339) +} + func days(d time.Duration) string { return fmt.Sprintf("%d days", int(d.Hours()/24)) } diff --git a/internal/api/bulk_split_test.go b/internal/api/bulk_split_test.go index 4294e89..3bfe2b0 100644 --- a/internal/api/bulk_split_test.go +++ b/internal/api/bulk_split_test.go @@ -166,7 +166,7 @@ func TestGetMetricBulkSplitAbortsOnFailure(t *testing.T) { if err == nil || resp != nil { t.Fatalf("expected an error and no data, got %v %v", resp, err) } - for _, want := range []string{"window 2/3", "2024-02-01..2024-03-03", "metric not in your plan", "--since 2024-02-01"} { + for _, want := range []string{"window 2 of about 3", "2024-02-01..2024-03-03", "metric not in your plan", "--since 2024-02-01"} { if !strings.Contains(err.Error(), want) { t.Errorf("error %q lacks %q", err, want) } @@ -225,3 +225,30 @@ func TestIsRangeTooLarge(t *testing.T) { } } } + +func TestGetMetricBulkSplitErrorAfterHalving(t *testing.T) { + // Hourly data from 06:00: 20 days need two 10-day windows. The server + // accepts only 5 days, so the window is halved once, then the third + // request fails. + since := planSince.Add(6 * time.Hour) + plan := BulkPlan{Since: since, Until: since.Add(20 * day), Interval: "1h", Window: 10 * day} + server, _, _ := bulkServer(t, 5*day, 4) // call 1 too large; calls 2-3 succeed; call 4 fails + _, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan) + if err == nil { + t.Fatal("expected an error") + } + for _, want := range []string{"window 3 of about 4", "2024-01-11T06:00:00Z..2024-01-16T06:00:00Z", "--since 2024-01-11T06:00:00Z"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q lacks %q", err, want) + } + } +} + +func TestFormatBound(t *testing.T) { + if got := formatBound(planSince); got != "2024-01-01" { + t.Errorf("midnight: %s", got) + } + if got := formatBound(planSince.Add(90 * time.Minute)); got != "2024-01-01T01:30:00Z" { + t.Errorf("intraday: %s", got) + } +} From c34d7e70fd6060c7fc13472b9ee2e18ae79083b9 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Mon, 12 Oct 2026 20:20:57 +0200 Subject: [PATCH 14/37] GN-159: do not name --until when it was not passed Without --until the upper bound is now, so a future --since is reported as not in the past instead of not before an --until the user never gave. --- internal/api/bulk_split.go | 3 +++ internal/api/bulk_split_test.go | 10 ++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/internal/api/bulk_split.go b/internal/api/bulk_split.go index f42059b..d372a3d 100644 --- a/internal/api/bulk_split.go +++ b/internal/api/bulk_split.go @@ -77,6 +77,9 @@ func PlanBulk(params map[string]string, now time.Time) (BulkPlan, error) { } plan.Until = plan.Until.UTC() if !plan.Since.Before(plan.Until) { + if _, given := params["u"]; !given { + return plan, fmt.Errorf("--since (%s) is not in the past", plan.Since.Format(time.RFC3339)) + } return plan, fmt.Errorf("--since (%s) must be before --until (%s)", plan.Since.Format(time.RFC3339), plan.Until.Format(time.RFC3339)) } return plan, nil diff --git a/internal/api/bulk_split_test.go b/internal/api/bulk_split_test.go index 3bfe2b0..78ef9fb 100644 --- a/internal/api/bulk_split_test.go +++ b/internal/api/bulk_split_test.go @@ -48,11 +48,13 @@ func TestPlanBulk(t *testing.T) { } }) t.Run("since must precede until", func(t *testing.T) { - if _, err := PlanBulk(map[string]string{"s": unix(planNow), "u": unix(planSince)}, planNow); err == nil { - t.Fatal("accepted since >= until") + if _, err := PlanBulk(map[string]string{"s": unix(planNow), "u": unix(planSince)}, planNow); err == nil || !strings.Contains(err.Error(), "must be before --until") { + t.Fatalf("since >= until: %v", err) } - if _, err := PlanBulk(map[string]string{"s": unix(planNow.Add(day))}, planNow); err == nil { - t.Fatal("accepted since in the future") + // Without --until the bound is now, and the message must not name --until. + _, err := PlanBulk(map[string]string{"s": unix(planNow.Add(day))}, planNow) + if err == nil || !strings.Contains(err.Error(), "is not in the past") || strings.Contains(err.Error(), "--until") { + t.Fatalf("since in the future: %v", err) } }) } From e59453ea8468ca2828ae683918170a2c784b24cc Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Mon, 12 Oct 2026 20:28:52 +0200 Subject: [PATCH 15/37] GN-159: pass bulk entries through instead of typing them The CLI only prints bulk entries, but decoded them through the SDK's BulkEntry, so a non-numeric v failed the whole request, numeric selectors became strings and large integers lost precision. Decode the /bulk response through the SDK's GetMetric into the CLI's own rows, as before the migration: the SDK decodes with UseNumber, so numbers stay numbers and any value passes through, while retries, limits and redaction still come from the SDK. --- internal/api/client_test.go | 31 +++++++++++++++++++++++++++++-- internal/api/metrics.go | 32 +++++++++++--------------------- 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/internal/api/client_test.go b/internal/api/client_test.go index 7934ade..7338938 100644 --- a/internal/api/client_test.go +++ b/internal/api/client_test.go @@ -3,6 +3,7 @@ package api import ( "context" _ "embed" + "encoding/json" "net/http" "net/http/httptest" "os" @@ -587,7 +588,7 @@ func TestGetMetricBulk(t *testing.T) { if resp.Data[0].Bulk[0]["a"] != "BTC" { t.Errorf("got a=%v", resp.Data[0].Bulk[0]["a"]) } - if resp.Data[0].Bulk[0]["v"] != 1513125006987.5164 { + if resp.Data[0].Bulk[0]["v"] != json.Number("1513125006987.5164") { t.Errorf("got v=%v", resp.Data[0].Bulk[0]["v"]) } } @@ -787,7 +788,7 @@ func TestGetMetricBulk_KeepsAllSelectors(t *testing.T) { t.Fatalf("GetMetricBulk: %v", err) } rows := resp.Data[0].Bulk - if rows[0]["e"] != "binance" || rows[0]["a"] != "BTC" || rows[0]["v"] != 1.0 { + if rows[0]["e"] != "binance" || rows[0]["a"] != "BTC" || rows[0]["v"] != json.Number("1") { t.Errorf("exchange selector lost: %v", rows[0]) } if rows[1]["category"] != "more_10y" || rows[1]["v"] != nil { @@ -810,3 +811,29 @@ func TestMetricParamsMapsTypedFields(t *testing.T) { t.Error("accepted non-numeric since") } } + +func TestGetMetricBulk_PassesEntriesThrough(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/metrics/market/price/bulk" { + t.Errorf("path %s", r.URL.Path) + } + _, _ = w.Write([]byte(`{"data":[{"t":1,"bulk":[{"a":"BTC","period":7,"v":12345678901234567890},{"a":"ETH","v":{"o":1,"c":2}}]}]}`)) + })) + defer server.Close() + + client := NewClient("key", "") + client.baseURL = server.URL + client.httpClient = server.Client() + + resp, err := client.GetMetricBulk(context.Background(), "/market/price", map[string]string{"s": "1"}, nil) + if err != nil { + t.Fatalf("GetMetricBulk: %v", err) + } + encoded, err := json.Marshal(resp.Data[0].Bulk) + if err != nil { + t.Fatal(err) + } + if got, want := string(encoded), `[{"a":"BTC","period":7,"v":12345678901234567890},{"a":"ETH","v":{"c":2,"o":1}}]`; got != want { + t.Errorf("bulk entries\n got %s\nwant %s", got, want) + } +} diff --git a/internal/api/metrics.go b/internal/api/metrics.go index b2e2c0c..baec3f6 100644 --- a/internal/api/metrics.go +++ b/internal/api/metrics.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "strconv" + "strings" "time" glassnode "github.com/glassnode/glassnode-api-go-client" @@ -90,6 +91,11 @@ func (c *Client) GetMetric(ctx context.Context, path string, params map[string]s return points, nil } +// GetMetricBulk fetches the bulk variant of a metric and returns its entries as +// the API sent them. The CLI only prints them, so it decodes into its own rows +// rather than the SDK's typed BulkEntry: numbers stay numbers (the SDK decodes +// with UseNumber), large integers keep their precision and selectors or values +// of any type pass through. func (c *Client) GetMetricBulk(ctx context.Context, path string, params map[string]string, repeated map[string][]string) (*BulkResponse, error) { client, err := c.sdk() if err != nil { @@ -99,28 +105,12 @@ func (c *Client) GetMetricBulk(ctx context.Context, path string, params map[stri if err != nil { return nil, err } - points, err := client.GetBulkMetric(ctx, path, p) - if err != nil { + var response BulkResponse + if err := client.GetMetric(ctx, strings.TrimSuffix(NormalizePath(path), "/")+"/bulk", p, &response); err != nil { return nil, fmt.Errorf("getting bulk metric: %w", err) } - response := &BulkResponse{Data: make([]BulkDataPoint, 0, len(points))} - for _, point := range points { - entries := make([]map[string]interface{}, 0, len(point.Bulk)) - for _, entry := range point.Bulk { - // Every selector of the entry (a, e, network, category, ...) is - // kept, as the API returned it, next to the value. - row := make(map[string]interface{}, len(entry.Params)+1) - for key, selector := range entry.Params { - row[key] = selector - } - var value any - if entry.Value != nil { - value = *entry.Value - } - row["v"] = value - entries = append(entries, row) - } - response.Data = append(response.Data, BulkDataPoint{T: point.Timestamp, Bulk: entries}) + if response.Data == nil { + response.Data = []BulkDataPoint{} } - return response, nil + return &response, nil } From d3993f6c7d2d88594f7937bb2e69ba5406227188 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Mon, 12 Oct 2026 20:32:26 +0200 Subject: [PATCH 16/37] GN-159: keep the SDK's json.Number in data points DataPoint.UnmarshalJSON re-decoded every point into float64, undoing the SDK's UseNumber: -o json rewrote the API's numbers and integers above 2^53 were rounded, and valueStr's json.Number branch never ran. Drop the wrapper so points keep json.Number end to end, and drop the unused BulkPlan.Fits. --- internal/api/bulk_split.go | 3 --- internal/api/bulk_split_test.go | 6 +++--- internal/api/client_test.go | 4 ++-- internal/api/metrics.go | 10 +++------- internal/output/output_test.go | 20 ++++++++++++++++++++ 5 files changed, 28 insertions(+), 15 deletions(-) diff --git a/internal/api/bulk_split.go b/internal/api/bulk_split.go index d372a3d..edff647 100644 --- a/internal/api/bulk_split.go +++ b/internal/api/bulk_split.go @@ -101,9 +101,6 @@ func (plan BulkPlan) Windows() []Window { return windows } -// Fits reports whether the whole range can be fetched in one request. -func (plan BulkPlan) Fits() bool { return len(plan.Windows()) == 1 } - // RangeError explains why the range needs --split. func (plan BulkPlan) RangeError() error { return fmt.Errorf("range %s..%s exceeds the API limit of %s per bulk request at %s; shorten it or add --split to fetch it in %d requests", diff --git a/internal/api/bulk_split_test.go b/internal/api/bulk_split_test.go index 78ef9fb..9248608 100644 --- a/internal/api/bulk_split_test.go +++ b/internal/api/bulk_split_test.go @@ -62,7 +62,7 @@ func TestPlanBulk(t *testing.T) { func TestBulkPlanWindows(t *testing.T) { plan := BulkPlan{Since: planSince, Until: planSince.Add(70 * day), Interval: "24h", Window: 31 * day} windows := plan.Windows() - if len(windows) != 3 || plan.Fits() { + if len(windows) != 3 { t.Fatalf("windows = %v", windows) } for i := 1; i < len(windows); i++ { @@ -76,8 +76,8 @@ func TestBulkPlanWindows(t *testing.T) { if !strings.Contains(plan.RangeError().Error(), "3 requests") || !strings.Contains(plan.Describe(), "3 requests of up to 31 days (24h)") { t.Errorf("messages: %v / %s", plan.RangeError(), plan.Describe()) } - if one := (BulkPlan{Since: planSince, Until: planSince.Add(day), Window: 31 * day}); !one.Fits() { - t.Error("one-day range should fit") + if one := (BulkPlan{Since: planSince, Until: planSince.Add(day), Window: 31 * day}).Windows(); len(one) != 1 { + t.Errorf("one-day range should be one window, got %d", len(one)) } } diff --git a/internal/api/client_test.go b/internal/api/client_test.go index 7338938..479551e 100644 --- a/internal/api/client_test.go +++ b/internal/api/client_test.go @@ -556,8 +556,8 @@ func TestGetMetric(t *testing.T) { if points[0].T != 1230940800 { t.Errorf("got T %d, want 1230940800", points[0].T) } - if points[0].V != 2.4755000000000003 { - t.Errorf("got V %v, want 2.4755000000000003", points[0].V) + if points[0].V != json.Number("2.4755000000000003") { + t.Errorf("got V %v, want the API's number 2.4755000000000003 unchanged", points[0].V) } } diff --git a/internal/api/metrics.go b/internal/api/metrics.go index baec3f6..641be3a 100644 --- a/internal/api/metrics.go +++ b/internal/api/metrics.go @@ -2,7 +2,6 @@ package api import ( "context" - "encoding/json" "fmt" "strconv" "strings" @@ -11,6 +10,9 @@ import ( glassnode "github.com/glassnode/glassnode-api-go-client" ) +// DataPoint is a metric point as the API sent it. Numbers are json.Number, +// since the SDK decodes with UseNumber: -o json prints them unchanged and +// integers above 2^53 keep their precision. type DataPoint struct { T int64 `json:"t"` V interface{} `json:"v,omitempty"` @@ -28,12 +30,6 @@ type BulkResponse struct { Data []BulkDataPoint `json:"data"` } -// UnmarshalJSON keeps the CLI's historical dynamic number/output behavior. -func (p *DataPoint) UnmarshalJSON(data []byte) error { - type wire DataPoint - return json.Unmarshal(data, (*wire)(p)) -} - // metricParams maps the CLI's flat query parameters onto the SDK's typed // fields, so the SDK validates the time range and selectors; anything it has // no field for stays in Extra. diff --git a/internal/output/output_test.go b/internal/output/output_test.go index a0399c4..0f09da3 100644 --- a/internal/output/output_test.go +++ b/internal/output/output_test.go @@ -371,3 +371,23 @@ func TestPrintCSV_ComputedAtColumn(t *testing.T) { t.Errorf("computed_at column shown without PIT data: %q", plain.String()) } } + +// Values arrive as json.Number from the SDK; neither JSON nor CSV output may +// round them through float64. +func TestLargeIntegersKeepPrecision(t *testing.T) { + data := []api.DataPoint{{T: 1, V: json.Number("12345678901234567890")}} + var jsonOut bytes.Buffer + if err := PrintJSON(&jsonOut, data); err != nil { + t.Fatal(err) + } + if !strings.Contains(jsonOut.String(), `"v": 12345678901234567890`) { + t.Errorf("JSON rounded the value: %s", jsonOut.String()) + } + var csvOut bytes.Buffer + if err := PrintCSV(&csvOut, data, "unix"); err != nil { + t.Fatal(err) + } + if got, want := csvOut.String(), "t,v\n1,12345678901234567890\n"; got != want { + t.Errorf("CSV got %q, want %q", got, want) + } +} From ed3567d314f607b16eae8a5f76525439fc40da92 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Mon, 12 Oct 2026 20:33:41 +0200 Subject: [PATCH 17/37] GN-159: drop the refresh warning that lost its cause The warning printed after a failed refresh no longer carried the error, and the SDK already returns that failure to the user as an AuthError with its cause. Remove the duplicate line; a test checks the IdP's error text reaches the caller. --- internal/api/client.go | 9 +++------ internal/api/client_test.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/internal/api/client.go b/internal/api/client.go index 37c00fa..517e224 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -2,7 +2,6 @@ package api import ( "context" - "errors" "fmt" "io" "net/http" @@ -87,14 +86,12 @@ func (s *sessionTokens) Token(ctx context.Context) (string, error) { } // Refresh exchanges the stored refresh token for a new access token after a -// 401 and keeps it for the following requests. An expired session is reported -// to the user through the SDK's AuthError. +// 401 and keeps it for the following requests. A failure, including an +// expired session, reaches the user with its cause through the SDK's +// AuthError, so nothing is printed here. func (s *sessionTokens) Refresh(ctx context.Context) (string, error) { fresh, err := oauth.ForceRefreshAccessToken(ctx) if err != nil { - if !errors.Is(err, oauth.ErrSessionExpired) { - _, _ = fmt.Fprintln(stderr, "warning: OAuth refresh after HTTP 401 failed") - } return "", err } s.mu.Lock() diff --git a/internal/api/client_test.go b/internal/api/client_test.go index 479551e..0cabebb 100644 --- a/internal/api/client_test.go +++ b/internal/api/client_test.go @@ -4,6 +4,7 @@ import ( "context" _ "embed" "encoding/json" + "errors" "net/http" "net/http/httptest" "os" @@ -12,6 +13,7 @@ import ( "testing" "time" + glassnode "github.com/glassnode/glassnode-api-go-client" "github.com/glassnode/glassnode-cli/internal/config" "github.com/glassnode/glassnode-cli/internal/oauth" "github.com/glassnode/glassnode-cli/internal/testhelper" @@ -837,3 +839,34 @@ func TestGetMetricBulk_PassesEntriesThrough(t *testing.T) { t.Errorf("bulk entries\n got %s\nwant %s", got, want) } } + +// A failed refresh after a 401 surfaces its cause in the returned error. +func TestDo_401RefreshFailureReportsCause(t *testing.T) { + withTempHome(t, func() { + api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte(`{"error":"unauthorized"}`)) + })) + defer api.Close() + idp := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"invalid_grant","error_description":"refresh token revoked"}`)) + })) + defer idp.Close() + restore := oauth.OverrideDefaultsForTesting(idp.URL, "https://api.example", "cid") + defer restore() + if err := config.SaveOAuthSession(config.OAuthSession{AccessToken: "stale", RefreshToken: "rt1", ExpiresAt: time.Now().UTC().Add(time.Hour)}); err != nil { + t.Fatal(err) + } + + client := NewClient("", "stale") + client.baseURL = api.URL + client.httpClient = api.Client() + + _, err := client.Do(context.Background(), "GET", "/v1/anything", nil) + var authErr *glassnode.AuthError + if !errors.As(err, &authErr) || !strings.Contains(err.Error(), "invalid_grant") || !strings.Contains(err.Error(), "refresh token revoked") { + t.Fatalf("err = %v, want an AuthError carrying the IdP's cause", err) + } + }) +} From a3453295c892db20b21843267a32f0a7e4d033d9 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Mon, 12 Oct 2026 21:10:37 +0200 Subject: [PATCH 18/37] GN-159: apply metric list --format on the client side The API's f filter on /v1/metadata/metrics returns no metrics for any value, yet metrics do differ in the response formats they support (some are JSON only). Keep --format/-f and make it work: list the metrics, then read each one's metadata and keep those whose parameters.f includes the requested format. Metadata requests spend no credits; a progress note goes to stderr and metrics without metadata are reported, not guessed. --- README.md | 7 +++ cmd/metric_list.go | 29 ++++++++++++- internal/api/metadata.go | 80 +++++++++++++++++++++++++++++++++++ internal/api/metadata_test.go | 63 +++++++++++++++++++++++++++ skills/glassnode-cli/SKILL.md | 2 +- 5 files changed, 178 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0bd8a7a..91a55f3 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,7 @@ gn metric list --from-exchange binance --to-exchange coinbase | `--currency` | `-c` | Filter by currency (e.g. `native`, `usd`) | | `--exchange` | `-e` | Filter by exchange (e.g. `binance`, `coinbase`) | | `--interval` | `-i` | Filter by time interval (e.g. `1h`, `24h`) | +| `--format` | `-f` | Keep only metrics available in this response format (`csv` or `json`) | | `--from-exchange` | | Source exchange for inter-exchange metrics | | `--to-exchange` | | Destination exchange for inter-exchange metrics | | `--miner` | | Miner identifier for mining-related metrics | @@ -200,6 +201,12 @@ gn metric list --from-exchange binance --to-exchange coinbase | `--period` | | Time period for aggregation | | `--quote-symbol` | | Quote currency symbol for trading pairs | +`--format` is applied by the CLI: the API's own format filter returns no +metrics, so the CLI reads each listed metric's metadata and keeps those whose +supported formats include the one asked for. That is one metadata request per +metric (no credits; a progress note goes to stderr), so narrow the list with +the other filters where you can. + ### `gn metric describe ` Show metadata for a metric: supported assets, intervals, exchanges, currencies, and parameters. diff --git a/cmd/metric_list.go b/cmd/metric_list.go index 9093570..74d4c29 100644 --- a/cmd/metric_list.go +++ b/cmd/metric_list.go @@ -1,6 +1,9 @@ package cmd import ( + "fmt" + "strings" + "github.com/glassnode/glassnode-cli/internal/api" "github.com/glassnode/glassnode-cli/internal/output" "github.com/spf13/cobra" @@ -18,6 +21,11 @@ var metricListCmd = &cobra.Command{ client := api.NewClient(apiKey, bearer) params, repeatedParams := metricListParamsFromFlags(cmd) + format, _ := cmd.Flags().GetString("format") + format = strings.ToLower(format) + if format != "" && format != "csv" && format != "json" { + return fmt.Errorf("--format must be csv or json, got %q", format) + } dryRun, _ := cmd.Flags().GetBool("dry-run") if dryRun { @@ -26,6 +34,9 @@ var metricListCmd = &cobra.Command{ return err } printDryRun(cmd, client, u) + if format != "" { + fmt.Fprintln(cmd.ErrOrStderr(), "# --format then reads each listed metric's metadata (/v1/metadata/metric?path=...)") + } return nil } @@ -33,9 +44,22 @@ var metricListCmd = &cobra.Command{ if err != nil { return err } + if format != "" { + // The API's f filter returns nothing; check each metric's metadata + // instead. Metadata requests cost no credits. + fmt.Fprintf(cmd.ErrOrStderr(), "# checking the metadata of %d metrics for %s support\n", len(metrics), format) + kept, skipped, err := client.FilterMetricsByFormat(cmd.Context(), metrics, format) + if err != nil { + return err + } + if len(skipped) > 0 { + fmt.Fprintf(cmd.ErrOrStderr(), "# %d metrics have no metadata and were left out\n", len(skipped)) + } + metrics = kept + } - format, _ := cmd.Flags().GetString("output") - return output.Print(output.Options{Format: format, Data: metrics}) + outputFormat, _ := cmd.Flags().GetString("output") + return output.Print(output.Options{Format: outputFormat, Data: metrics}) }, } @@ -94,6 +118,7 @@ func init() { metricListCmd.Flags().StringArray("assets", nil, "filter by assets (multiple, e.g. --assets BTC --assets ETH)") metricListCmd.Flags().StringP("currency", "c", "", "filter by currency (e.g. native, usd)") metricListCmd.Flags().StringP("exchange", "e", "", "filter by exchange (e.g. binance, coinbase)") + metricListCmd.Flags().StringP("format", "f", "", "keep only metrics available in this response format (csv or json); reads each metric's metadata") metricListCmd.Flags().StringP("interval", "i", "", "filter by time interval (e.g. 1h, 24h)") metricListCmd.Flags().String("from-exchange", "", "source exchange for inter-exchange metrics") metricListCmd.Flags().String("to-exchange", "", "destination exchange for inter-exchange metrics") diff --git a/internal/api/metadata.go b/internal/api/metadata.go index 11bd94f..28c5d15 100644 --- a/internal/api/metadata.go +++ b/internal/api/metadata.go @@ -2,8 +2,13 @@ package api import ( "context" + "errors" + "fmt" + "net/http" "net/url" + "slices" "strings" + "sync" glassnode "github.com/glassnode/glassnode-api-go-client" ) @@ -83,6 +88,81 @@ func (c *Client) ListMetrics(ctx context.Context, params map[string]string, repe return client.ListMetrics(ctx, &glassnode.MetricParams{Extra: queryValues(params, repeated)}) } +// metadataWorkers bounds concurrent metadata requests; the metadata endpoints +// have their own rate limit of 1200 requests per minute. +const metadataWorkers = 8 + +// FilterMetricsByFormat keeps the metrics whose metadata lists format among +// the response formats they support (parameters.f). The API's own f filter on +// the metric list returns no metrics for any value, so the CLI reads each +// metric's metadata instead; metadata requests spend no credits. Metrics the +// API has no metadata for are returned in skipped rather than guessed. +func (c *Client) FilterMetricsByFormat(ctx context.Context, paths []string, format string) (kept, skipped []string, err error) { + client, err := c.sdk() + if err != nil { + return nil, nil, err + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + type verdict struct { + supported bool + missing bool + err error + } + verdicts := make([]verdict, len(paths)) + jobs := make(chan int) + var group sync.WaitGroup + for range metadataWorkers { + group.Add(1) + go func() { + defer group.Done() + for i := range jobs { + meta, err := client.GetMetricMetadata(ctx, paths[i], nil) + var apiErr *glassnode.APIError + switch { + case errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusNotFound: + verdicts[i] = verdict{missing: true} + case err != nil: + verdicts[i] = verdict{err: err} + cancel() // no point finishing the others + default: + verdicts[i] = verdict{supported: slices.ContainsFunc(meta.Parameters["f"], func(f string) bool { return strings.EqualFold(f, format) })} + } + } + }() + } + for i := range paths { + select { + case jobs <- i: + case <-ctx.Done(): + } + if ctx.Err() != nil { + break + } + } + close(jobs) + group.Wait() + + for i, v := range verdicts { + switch { + case v.err != nil: + return nil, nil, fmt.Errorf("checking metadata of %s: %w", paths[i], v.err) + case v.missing: + skipped = append(skipped, paths[i]) + case v.supported: + kept = append(kept, paths[i]) + } + } + if ctx.Err() != nil { + return nil, nil, ctx.Err() + } + if kept == nil { + kept = []string{} + } + return kept, skipped, nil +} + func (c *Client) DescribeMetric(ctx context.Context, path, asset string) (*MetricMetadata, error) { client, err := c.sdk() if err != nil { diff --git a/internal/api/metadata_test.go b/internal/api/metadata_test.go index 7c21f97..997aae3 100644 --- a/internal/api/metadata_test.go +++ b/internal/api/metadata_test.go @@ -1,7 +1,14 @@ package api import ( + "context" "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync/atomic" "testing" ) @@ -137,3 +144,59 @@ func TestTrimBulkSuffix(t *testing.T) { } } } + +func TestFilterMetricsByFormat(t *testing.T) { + formats := map[string][]string{ + "/market/price": {"csv", "json"}, "/derivatives/term_structure": {"json"}, + "/market/hodl_cave": {"CSV", "json"}, "/indicators/urpd": {"json"}, + } + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + path := r.URL.Query().Get("path") + f, ok := formats[path] + if !ok { + w.WriteHeader(http.StatusNotFound) + fmt.Fprintf(w, "metadata not found: path=%s", path) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{"path": path, "parameters": map[string]any{"f": f, "a": []string{"BTC"}}}) + })) + defer server.Close() + client := NewClient("key", "") + client.baseURL = server.URL + client.httpClient = server.Client() + + paths := []string{"/market/price", "/derivatives/term_structure", "/options/smile", "/market/hodl_cave", "/indicators/urpd"} + kept, skipped, err := client.FilterMetricsByFormat(context.Background(), paths, "csv") + if err != nil { + t.Fatal(err) + } + if want := []string{"/market/price", "/market/hodl_cave"}; !slices.Equal(kept, want) { + t.Errorf("kept = %v, want %v (in list order)", kept, want) + } + if want := []string{"/options/smile"}; !slices.Equal(skipped, want) { + t.Errorf("skipped = %v, want %v", skipped, want) + } + if calls.Load() != int32(len(paths)) { + t.Errorf("metadata calls = %d, want %d", calls.Load(), len(paths)) + } + kept, _, err = client.FilterMetricsByFormat(context.Background(), paths, "json") + if err != nil || len(kept) != 4 { + t.Errorf("json: kept=%v err=%v", kept, err) + } +} + +func TestFilterMetricsByFormat_AbortsOnOtherErrors(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + fmt.Fprint(w, `{"message":"no access"}`) + })) + defer server.Close() + client := NewClient("key", "") + client.baseURL = server.URL + client.httpClient = server.Client() + if _, _, err := client.FilterMetricsByFormat(context.Background(), []string{"/a", "/b"}, "csv"); err == nil || !strings.Contains(err.Error(), "no access") { + t.Fatalf("err = %v, want the API error", err) + } +} diff --git a/skills/glassnode-cli/SKILL.md b/skills/glassnode-cli/SKILL.md index 9c98d00..4fb7e1e 100644 --- a/skills/glassnode-cli/SKILL.md +++ b/skills/glassnode-cli/SKILL.md @@ -42,7 +42,7 @@ gn metric list -a BTC -e binance -i 24h gn metric list --assets BTC --assets ETH gn metric list --from-exchange binance --to-exchange coinbase ``` -Filter by any [metadata query parameters](https://docs.glassnode.com/basic-api/metadata#query-parameters-1): `--currency`, `--exchange`, `--interval`, `--from-exchange`, `--to-exchange`, `--miner`, `--maturity`, `--network`, `--period`, `--quote-symbol`, or multiple `--assets`. +Filter by any [metadata query parameters](https://docs.glassnode.com/basic-api/metadata#query-parameters-1): `--currency`, `--exchange`, `--interval`, `--from-exchange`, `--to-exchange`, `--miner`, `--maturity`, `--network`, `--period`, `--quote-symbol`, or multiple `--assets`. `--format csv|json` keeps only metrics available in that response format; it reads each listed metric's metadata (no credits, slower), so combine it with other filters. ### Describe a metric (discover valid parameters) ```bash From f8efe69445d56d8e657813599fbe9a5efeae415f Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Mon, 12 Oct 2026 21:15:14 +0200 Subject: [PATCH 19/37] Revert "GN-159: apply metric list --format on the client side" This reverts commit a3453295c892db20b21843267a32f0a7e4d033d9. --- README.md | 7 --- cmd/metric_list.go | 29 +------------ internal/api/metadata.go | 80 ----------------------------------- internal/api/metadata_test.go | 63 --------------------------- skills/glassnode-cli/SKILL.md | 2 +- 5 files changed, 3 insertions(+), 178 deletions(-) diff --git a/README.md b/README.md index 91a55f3..0bd8a7a 100644 --- a/README.md +++ b/README.md @@ -192,7 +192,6 @@ gn metric list --from-exchange binance --to-exchange coinbase | `--currency` | `-c` | Filter by currency (e.g. `native`, `usd`) | | `--exchange` | `-e` | Filter by exchange (e.g. `binance`, `coinbase`) | | `--interval` | `-i` | Filter by time interval (e.g. `1h`, `24h`) | -| `--format` | `-f` | Keep only metrics available in this response format (`csv` or `json`) | | `--from-exchange` | | Source exchange for inter-exchange metrics | | `--to-exchange` | | Destination exchange for inter-exchange metrics | | `--miner` | | Miner identifier for mining-related metrics | @@ -201,12 +200,6 @@ gn metric list --from-exchange binance --to-exchange coinbase | `--period` | | Time period for aggregation | | `--quote-symbol` | | Quote currency symbol for trading pairs | -`--format` is applied by the CLI: the API's own format filter returns no -metrics, so the CLI reads each listed metric's metadata and keeps those whose -supported formats include the one asked for. That is one metadata request per -metric (no credits; a progress note goes to stderr), so narrow the list with -the other filters where you can. - ### `gn metric describe ` Show metadata for a metric: supported assets, intervals, exchanges, currencies, and parameters. diff --git a/cmd/metric_list.go b/cmd/metric_list.go index 74d4c29..9093570 100644 --- a/cmd/metric_list.go +++ b/cmd/metric_list.go @@ -1,9 +1,6 @@ package cmd import ( - "fmt" - "strings" - "github.com/glassnode/glassnode-cli/internal/api" "github.com/glassnode/glassnode-cli/internal/output" "github.com/spf13/cobra" @@ -21,11 +18,6 @@ var metricListCmd = &cobra.Command{ client := api.NewClient(apiKey, bearer) params, repeatedParams := metricListParamsFromFlags(cmd) - format, _ := cmd.Flags().GetString("format") - format = strings.ToLower(format) - if format != "" && format != "csv" && format != "json" { - return fmt.Errorf("--format must be csv or json, got %q", format) - } dryRun, _ := cmd.Flags().GetBool("dry-run") if dryRun { @@ -34,9 +26,6 @@ var metricListCmd = &cobra.Command{ return err } printDryRun(cmd, client, u) - if format != "" { - fmt.Fprintln(cmd.ErrOrStderr(), "# --format then reads each listed metric's metadata (/v1/metadata/metric?path=...)") - } return nil } @@ -44,22 +33,9 @@ var metricListCmd = &cobra.Command{ if err != nil { return err } - if format != "" { - // The API's f filter returns nothing; check each metric's metadata - // instead. Metadata requests cost no credits. - fmt.Fprintf(cmd.ErrOrStderr(), "# checking the metadata of %d metrics for %s support\n", len(metrics), format) - kept, skipped, err := client.FilterMetricsByFormat(cmd.Context(), metrics, format) - if err != nil { - return err - } - if len(skipped) > 0 { - fmt.Fprintf(cmd.ErrOrStderr(), "# %d metrics have no metadata and were left out\n", len(skipped)) - } - metrics = kept - } - outputFormat, _ := cmd.Flags().GetString("output") - return output.Print(output.Options{Format: outputFormat, Data: metrics}) + format, _ := cmd.Flags().GetString("output") + return output.Print(output.Options{Format: format, Data: metrics}) }, } @@ -118,7 +94,6 @@ func init() { metricListCmd.Flags().StringArray("assets", nil, "filter by assets (multiple, e.g. --assets BTC --assets ETH)") metricListCmd.Flags().StringP("currency", "c", "", "filter by currency (e.g. native, usd)") metricListCmd.Flags().StringP("exchange", "e", "", "filter by exchange (e.g. binance, coinbase)") - metricListCmd.Flags().StringP("format", "f", "", "keep only metrics available in this response format (csv or json); reads each metric's metadata") metricListCmd.Flags().StringP("interval", "i", "", "filter by time interval (e.g. 1h, 24h)") metricListCmd.Flags().String("from-exchange", "", "source exchange for inter-exchange metrics") metricListCmd.Flags().String("to-exchange", "", "destination exchange for inter-exchange metrics") diff --git a/internal/api/metadata.go b/internal/api/metadata.go index 28c5d15..11bd94f 100644 --- a/internal/api/metadata.go +++ b/internal/api/metadata.go @@ -2,13 +2,8 @@ package api import ( "context" - "errors" - "fmt" - "net/http" "net/url" - "slices" "strings" - "sync" glassnode "github.com/glassnode/glassnode-api-go-client" ) @@ -88,81 +83,6 @@ func (c *Client) ListMetrics(ctx context.Context, params map[string]string, repe return client.ListMetrics(ctx, &glassnode.MetricParams{Extra: queryValues(params, repeated)}) } -// metadataWorkers bounds concurrent metadata requests; the metadata endpoints -// have their own rate limit of 1200 requests per minute. -const metadataWorkers = 8 - -// FilterMetricsByFormat keeps the metrics whose metadata lists format among -// the response formats they support (parameters.f). The API's own f filter on -// the metric list returns no metrics for any value, so the CLI reads each -// metric's metadata instead; metadata requests spend no credits. Metrics the -// API has no metadata for are returned in skipped rather than guessed. -func (c *Client) FilterMetricsByFormat(ctx context.Context, paths []string, format string) (kept, skipped []string, err error) { - client, err := c.sdk() - if err != nil { - return nil, nil, err - } - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - type verdict struct { - supported bool - missing bool - err error - } - verdicts := make([]verdict, len(paths)) - jobs := make(chan int) - var group sync.WaitGroup - for range metadataWorkers { - group.Add(1) - go func() { - defer group.Done() - for i := range jobs { - meta, err := client.GetMetricMetadata(ctx, paths[i], nil) - var apiErr *glassnode.APIError - switch { - case errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusNotFound: - verdicts[i] = verdict{missing: true} - case err != nil: - verdicts[i] = verdict{err: err} - cancel() // no point finishing the others - default: - verdicts[i] = verdict{supported: slices.ContainsFunc(meta.Parameters["f"], func(f string) bool { return strings.EqualFold(f, format) })} - } - } - }() - } - for i := range paths { - select { - case jobs <- i: - case <-ctx.Done(): - } - if ctx.Err() != nil { - break - } - } - close(jobs) - group.Wait() - - for i, v := range verdicts { - switch { - case v.err != nil: - return nil, nil, fmt.Errorf("checking metadata of %s: %w", paths[i], v.err) - case v.missing: - skipped = append(skipped, paths[i]) - case v.supported: - kept = append(kept, paths[i]) - } - } - if ctx.Err() != nil { - return nil, nil, ctx.Err() - } - if kept == nil { - kept = []string{} - } - return kept, skipped, nil -} - func (c *Client) DescribeMetric(ctx context.Context, path, asset string) (*MetricMetadata, error) { client, err := c.sdk() if err != nil { diff --git a/internal/api/metadata_test.go b/internal/api/metadata_test.go index 997aae3..7c21f97 100644 --- a/internal/api/metadata_test.go +++ b/internal/api/metadata_test.go @@ -1,14 +1,7 @@ package api import ( - "context" "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "slices" - "strings" - "sync/atomic" "testing" ) @@ -144,59 +137,3 @@ func TestTrimBulkSuffix(t *testing.T) { } } } - -func TestFilterMetricsByFormat(t *testing.T) { - formats := map[string][]string{ - "/market/price": {"csv", "json"}, "/derivatives/term_structure": {"json"}, - "/market/hodl_cave": {"CSV", "json"}, "/indicators/urpd": {"json"}, - } - var calls atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - calls.Add(1) - path := r.URL.Query().Get("path") - f, ok := formats[path] - if !ok { - w.WriteHeader(http.StatusNotFound) - fmt.Fprintf(w, "metadata not found: path=%s", path) - return - } - _ = json.NewEncoder(w).Encode(map[string]any{"path": path, "parameters": map[string]any{"f": f, "a": []string{"BTC"}}}) - })) - defer server.Close() - client := NewClient("key", "") - client.baseURL = server.URL - client.httpClient = server.Client() - - paths := []string{"/market/price", "/derivatives/term_structure", "/options/smile", "/market/hodl_cave", "/indicators/urpd"} - kept, skipped, err := client.FilterMetricsByFormat(context.Background(), paths, "csv") - if err != nil { - t.Fatal(err) - } - if want := []string{"/market/price", "/market/hodl_cave"}; !slices.Equal(kept, want) { - t.Errorf("kept = %v, want %v (in list order)", kept, want) - } - if want := []string{"/options/smile"}; !slices.Equal(skipped, want) { - t.Errorf("skipped = %v, want %v", skipped, want) - } - if calls.Load() != int32(len(paths)) { - t.Errorf("metadata calls = %d, want %d", calls.Load(), len(paths)) - } - kept, _, err = client.FilterMetricsByFormat(context.Background(), paths, "json") - if err != nil || len(kept) != 4 { - t.Errorf("json: kept=%v err=%v", kept, err) - } -} - -func TestFilterMetricsByFormat_AbortsOnOtherErrors(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusForbidden) - fmt.Fprint(w, `{"message":"no access"}`) - })) - defer server.Close() - client := NewClient("key", "") - client.baseURL = server.URL - client.httpClient = server.Client() - if _, _, err := client.FilterMetricsByFormat(context.Background(), []string{"/a", "/b"}, "csv"); err == nil || !strings.Contains(err.Error(), "no access") { - t.Fatalf("err = %v, want the API error", err) - } -} diff --git a/skills/glassnode-cli/SKILL.md b/skills/glassnode-cli/SKILL.md index 4fb7e1e..9c98d00 100644 --- a/skills/glassnode-cli/SKILL.md +++ b/skills/glassnode-cli/SKILL.md @@ -42,7 +42,7 @@ gn metric list -a BTC -e binance -i 24h gn metric list --assets BTC --assets ETH gn metric list --from-exchange binance --to-exchange coinbase ``` -Filter by any [metadata query parameters](https://docs.glassnode.com/basic-api/metadata#query-parameters-1): `--currency`, `--exchange`, `--interval`, `--from-exchange`, `--to-exchange`, `--miner`, `--maturity`, `--network`, `--period`, `--quote-symbol`, or multiple `--assets`. `--format csv|json` keeps only metrics available in that response format; it reads each listed metric's metadata (no credits, slower), so combine it with other filters. +Filter by any [metadata query parameters](https://docs.glassnode.com/basic-api/metadata#query-parameters-1): `--currency`, `--exchange`, `--interval`, `--from-exchange`, `--to-exchange`, `--miner`, `--maturity`, `--network`, `--period`, `--quote-symbol`, or multiple `--assets`. ### Describe a metric (discover valid parameters) ```bash From 4111319920206e45a2c0a3d6c82ceb8118090fed Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Mon, 12 Oct 2026 21:16:21 +0200 Subject: [PATCH 20/37] GN-159: keep metric list --format hidden for one release with an explanation Scripts passing -f csv got cobra's "unknown shorthand flag". The flag is registered but hidden and fails with the reason the filter was removed (the API's f filter returns no metrics for any value) and the replacement for CSV data. It goes away in the next release. --- cmd/metric_list.go | 12 ++++++++++++ cmd/metric_list_test.go | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 cmd/metric_list_test.go diff --git a/cmd/metric_list.go b/cmd/metric_list.go index 9093570..088af6a 100644 --- a/cmd/metric_list.go +++ b/cmd/metric_list.go @@ -1,6 +1,8 @@ package cmd import ( + "errors" + "github.com/glassnode/glassnode-cli/internal/api" "github.com/glassnode/glassnode-cli/internal/output" "github.com/spf13/cobra" @@ -10,6 +12,9 @@ var metricListCmd = &cobra.Command{ Use: "list", Short: "List available metrics", RunE: func(cmd *cobra.Command, args []string) error { + if cmd.Flags().Changed("format") { + return errors.New(formatRemoved) + } apiKeyFlag, _ := cmd.Flags().GetString("api-key") apiKey, bearer, err := api.RequireAuth(cmd.Context(), apiKeyFlag) if err != nil { @@ -89,7 +94,14 @@ func metricListParamsFromFlags(cmd *cobra.Command) (map[string]string, map[strin return params, repeatedParams } +// formatRemoved explains the removed --format filter to scripts that still +// pass it; the hidden flag and this message go away in the next release. +const formatRemoved = `--format was removed: the API's response-format filter on the metric list returns no metrics for any value; ` + + `for metric data in CSV use "gn metric get ... -o csv"` + func init() { + metricListCmd.Flags().StringP("format", "f", "", "removed") + _ = metricListCmd.Flags().MarkHidden("format") metricListCmd.Flags().StringP("asset", "a", "", "filter by asset (single)") metricListCmd.Flags().StringArray("assets", nil, "filter by assets (multiple, e.g. --assets BTC --assets ETH)") metricListCmd.Flags().StringP("currency", "c", "", "filter by currency (e.g. native, usd)") diff --git a/cmd/metric_list_test.go b/cmd/metric_list_test.go new file mode 100644 index 0000000..2edaa1b --- /dev/null +++ b/cmd/metric_list_test.go @@ -0,0 +1,20 @@ +package cmd + +import ( + "strings" + "testing" +) + +// The removed --format filter fails with the reason and the replacement for +// one release, instead of cobra's "unknown shorthand flag". +func TestMetricList_RemovedFormatFlagExplains(t *testing.T) { + for _, args := range [][]string{{"-f", "csv"}, {"--format", "json"}} { + _, stderr, err := runCLI(t, []string{"HOME=" + t.TempDir()}, append([]string{"metric", "list", "--api-key", "k"}, args...)...) + if err == nil { + t.Fatalf("%v: expected a failure", args) + } + if !strings.Contains(stderr, "--format was removed") || !strings.Contains(stderr, "-o csv") || strings.Contains(stderr, "unknown shorthand") { + t.Errorf("%v: stderr = %s", args, stderr) + } + } +} From 06869aaa8953e6a660da39dffbdde6b99519f07e Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Mon, 12 Oct 2026 21:31:01 +0200 Subject: [PATCH 21/37] GN-159: stop halving below one interval and keep intraday bounds in messages Self-review of the branch. Halving a bulk window below the resolution's interval (93 days twice at 1month) would make the API return no points and the rest of the fetch silent gaps; halving now stops at one interval and the API's error is returned. The range and split messages showed dates only, dropping the time of day for hourly and 10-minute fetches; they now use the same bound formatting as the abort error. --- internal/api/bulk_split.go | 25 ++++++++++++++++++++++--- internal/api/bulk_split_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/internal/api/bulk_split.go b/internal/api/bulk_split.go index edff647..c5f53fd 100644 --- a/internal/api/bulk_split.go +++ b/internal/api/bulk_split.go @@ -24,6 +24,23 @@ const ( maxWindowShrinks = 3 ) +// intervalDuration is the length of one data interval at the resolution; a +// window shorter than it returns no points, so halving stops there. +func intervalDuration(interval string) time.Duration { + switch interval { + case "10m": + return 10 * time.Minute + case "1h": + return time.Hour + case "1w": + return 7 * 24 * time.Hour + case "1month": + return 31 * 24 * time.Hour + default: + return 24 * time.Hour + } +} + // BulkWindow returns the longest time range the API accepts in one bulk // request at the given resolution. func BulkWindow(interval string) time.Duration { @@ -104,13 +121,13 @@ func (plan BulkPlan) Windows() []Window { // RangeError explains why the range needs --split. func (plan BulkPlan) RangeError() error { return fmt.Errorf("range %s..%s exceeds the API limit of %s per bulk request at %s; shorten it or add --split to fetch it in %d requests", - plan.Since.Format(time.DateOnly), plan.Until.Format(time.DateOnly), days(plan.Window), plan.Interval, len(plan.Windows())) + formatBound(plan.Since), formatBound(plan.Until), days(plan.Window), plan.Interval, len(plan.Windows())) } // Describe summarises the split for a warning to the user. func (plan BulkPlan) Describe() string { return fmt.Sprintf("splitting %s..%s into %d requests of up to %s (%s); the result is assembled in memory", - plan.Since.Format(time.DateOnly), plan.Until.Format(time.DateOnly), len(plan.Windows()), days(plan.Window), plan.Interval) + formatBound(plan.Since), formatBound(plan.Until), len(plan.Windows()), days(plan.Window), plan.Interval) } // WindowParams returns params with s and u set to the window's bounds. @@ -141,7 +158,9 @@ func (c *Client) GetMetricBulkSplit(ctx context.Context, path string, params map w := windows[0] resp, err := c.GetMetricBulk(ctx, path, WindowParams(params, w), repeated) if err != nil { - if isRangeTooLarge(err) && shrinks < maxWindowShrinks { + // A window shorter than one interval would return no points and + // turn the rest of the fetch into silent gaps, so halving stops there. + if isRangeTooLarge(err) && shrinks < maxWindowShrinks && plan.Window/2 >= intervalDuration(plan.Interval) { shrinks++ plan.Window /= 2 continue diff --git a/internal/api/bulk_split_test.go b/internal/api/bulk_split_test.go index 9248608..d7c1014 100644 --- a/internal/api/bulk_split_test.go +++ b/internal/api/bulk_split_test.go @@ -254,3 +254,27 @@ func TestFormatBound(t *testing.T) { t.Errorf("intraday: %s", got) } } + +// At monthly resolution a 93-day window may be halved once (46 days, still +// over one interval) but not twice (23 days would return no points). +func TestGetMetricBulkSplitDoesNotHalveBelowAnInterval(t *testing.T) { + plan := BulkPlan{Since: planSince, Until: planSince.Add(93 * day), Interval: "1month", Window: 93 * day} + server, calls, windows := bulkServer(t, 40*day, 0) // even 46 days is rejected + _, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan) + if err == nil || !strings.Contains(err.Error(), "time range") { + t.Fatalf("err = %v, want the API's too-large error once halving is exhausted", err) + } + if calls.Load() != 2 || (*windows)[1].Until.Sub((*windows)[1].Since) != 46*day+12*time.Hour { + t.Errorf("calls=%d windows=%v, want 93d then 46.5d and no 23d window", calls.Load(), *windows) + } +} + +func TestRangeMessagesKeepIntradayBounds(t *testing.T) { + since := planSince.Add(6 * time.Hour) + plan := BulkPlan{Since: since, Until: since.Add(19 * day), Interval: "1h", Window: 10 * day} + for _, msg := range []string{plan.RangeError().Error(), plan.Describe()} { + if !strings.Contains(msg, "2024-01-01T06:00:00Z..2024-01-20T06:00:00Z") { + t.Errorf("message drops the time of day: %s", msg) + } + } +} From 520a42e1ab753306dfdaac8b73c0fdd1fac5ebe1 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Mon, 12 Oct 2026 21:36:11 +0200 Subject: [PATCH 22/37] GN-159: render gn user credits as a table and as CSV -o table and -o csv fell through to JSON for the credits summary, as they did before this branch. Print labelled lines (period, limit with its unit, used, left) for the table and one CSV row; an unknown period shows the limit and says the usage is unknown instead of a number from another period's counter. --- internal/output/output.go | 39 ++++++++++++++++++++++++++++++++++ internal/output/output_test.go | 37 ++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/internal/output/output.go b/internal/output/output.go index 45d6bf9..90e96e7 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -77,6 +77,13 @@ func PrintCSV(w io.Writer, data interface{}, timestampFormat string) error { return err } } + case api.CreditsSummary: + if err := write([]string{"limit", "period", "used", "left"}); err != nil { + return err + } + if err := write([]string{strconv.Itoa(v.CreditsLimit), string(v.CreditsPeriod), optionalInt(v.CreditsUsed), optionalInt(v.CreditsLeft)}); err != nil { + return err + } case []api.Asset: if err := write([]string{"id", "symbol", "name", "asset_type", "categories"}); err != nil { return err @@ -193,6 +200,8 @@ func PrintTable(w io.Writer, data interface{}, timestampFormat string) error { for _, s := range v { table.Append([]string{s}) } + case api.CreditsSummary: + return printCreditsTable(w, v) case []api.Asset: table.SetHeader([]string{"ID", "Symbol", "Name", "Type", "Categories"}) for _, a := range v { @@ -276,6 +285,36 @@ func PrintTable(w io.Writer, data interface{}, timestampFormat string) error { return nil } +// printCreditsTable lays the allowance out as labelled lines, like the metric +// metadata table. A daily allowance counts requests, a monthly one credits; +// usage is unknown when the period is not one the CLI knows. +func printCreditsTable(w io.Writer, s api.CreditsSummary) error { + unit := "credits" + switch s.CreditsPeriod { + case api.CreditsPeriodDaily: + unit = "requests" + case api.CreditsPeriodMonthly: + default: + unit = "" + } + fmt.Fprintf(w, "Period: %s\n", s.CreditsPeriod) + fmt.Fprintf(w, "Limit: %s\n", strings.TrimSpace(fmt.Sprintf("%d %s", s.CreditsLimit, unit))) + if s.CreditsUsed == nil || s.CreditsLeft == nil { + fmt.Fprintln(w, "Used: unknown for this period") + return nil + } + fmt.Fprintf(w, "Used: %d\n", *s.CreditsUsed) + fmt.Fprintf(w, "Left: %d\n", *s.CreditsLeft) + return nil +} + +func optionalInt(n *int) string { + if n == nil { + return "" + } + return strconv.Itoa(*n) +} + func printMetricMetadataTable(w io.Writer, m *api.MetricMetadata) error { fmt.Fprintf(w, "Path: %s\n", m.Path) fmt.Fprintf(w, "Tier: %d\n", m.Tier) diff --git a/internal/output/output_test.go b/internal/output/output_test.go index 0f09da3..351414c 100644 --- a/internal/output/output_test.go +++ b/internal/output/output_test.go @@ -391,3 +391,40 @@ func TestLargeIntegersKeepPrecision(t *testing.T) { t.Errorf("CSV got %q, want %q", got, want) } } + +func TestCreditsSummary_TableAndCSV(t *testing.T) { + used, left := 2, 48 + daily := api.CreditsSummary{CreditsLimit: 50, CreditsPeriod: api.CreditsPeriodDaily, CreditsUsed: &used, CreditsLeft: &left} + var table bytes.Buffer + if err := PrintTable(&table, daily, ""); err != nil { + t.Fatal(err) + } + for _, want := range []string{"Period: daily", "Limit: 50 requests", "Used: 2", "Left: 48"} { + if !strings.Contains(table.String(), want) { + t.Errorf("table lacks %q:\n%s", want, table.String()) + } + } + var csvOut bytes.Buffer + if err := PrintCSV(&csvOut, daily, ""); err != nil { + t.Fatal(err) + } + if got, want := csvOut.String(), "limit,period,used,left\n50,daily,2,48\n"; got != want { + t.Errorf("csv got %q want %q", got, want) + } + + unknown := api.CreditsSummary{CreditsLimit: 300, CreditsPeriod: "weekly"} + table.Reset() + if err := PrintTable(&table, unknown, ""); err != nil { + t.Fatal(err) + } + if !strings.Contains(table.String(), "Limit: 300\n") || !strings.Contains(table.String(), "unknown for this period") { + t.Errorf("unknown period table:\n%s", table.String()) + } + csvOut.Reset() + if err := PrintCSV(&csvOut, unknown, ""); err != nil { + t.Fatal(err) + } + if got, want := csvOut.String(), "limit,period,used,left\n300,weekly,,\n"; got != want { + t.Errorf("unknown period csv got %q want %q", got, want) + } +} From 143b834d62bca80de8ad6c97bb4f14afcc83bf54 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Tue, 13 Oct 2026 08:37:27 +0200 Subject: [PATCH 23/37] GN-159: halve the rejected bulk range and report future --until honestly Halving the nominal window when the API rejected the shorter tail of a range repeated the identical request and spent one of the three shrinks on it; the range the API actually rejected is halved instead. PlanBulk caps a future --until at now before comparing it with --since, so the error named a value the user never gave; it now says --since is not in the past, as when --until is absent. The --since error no longer suggests adding --split, which may already be set. An absent add-on period is read as monthly in the mixed-period check too. --- internal/api/api_usage.go | 13 +++++++++++-- internal/api/api_usage_test.go | 2 ++ internal/api/bulk_split.go | 20 ++++++++++++++------ internal/api/bulk_split_test.go | 26 +++++++++++++++++++++++++- 4 files changed, 52 insertions(+), 9 deletions(-) diff --git a/internal/api/api_usage.go b/internal/api/api_usage.go index 236454e..da63820 100644 --- a/internal/api/api_usage.go +++ b/internal/api/api_usage.go @@ -48,16 +48,25 @@ func (a *APIUsageResponse) Summary() CreditsSummary { } // An unknown or mixed period: report what the API sent rather than // measuring it against a counter of another period. - summary := CreditsSummary{CreditsPeriod: a.APIAddons[0].Period} + summary := CreditsSummary{CreditsPeriod: addonPeriod(a.APIAddons[0])} for _, addon := range a.APIAddons { summary.CreditsLimit = max(summary.CreditsLimit, addon.Value) - if addon.Period != summary.CreditsPeriod { + if addonPeriod(addon) != summary.CreditsPeriod { summary.CreditsPeriod = "mixed" } } return summary } +// addonPeriod is the add-on's period with the API's default, an absent +// period, read as monthly, as the SDK does. +func addonPeriod(addon APIAddon) CreditsPeriod { + if addon.Period == "" { + return CreditsPeriodMonthly + } + return addon.Period +} + // GetAPIUsage fetches the current API usage for the authenticated user func (c *Client) GetAPIUsage(ctx context.Context) (*APIUsageResponse, error) { client, err := c.sdk() diff --git a/internal/api/api_usage_test.go b/internal/api/api_usage_test.go index 9f086b5..53f0795 100644 --- a/internal/api/api_usage_test.go +++ b/internal/api/api_usage_test.go @@ -74,6 +74,8 @@ func TestCreditsPeriod_ComesFromTheAddon(t *testing.T) { {"no addons is monthly", nil, CreditsPeriodMonthly}, {"a period outside the enum is reported as sent", []APIAddon{{Value: 50, Period: "weekly"}}, "weekly"}, {"mixed periods are reported as mixed", []APIAddon{{Value: 50, Period: "daily"}, {Value: 5000000, Period: "monthly"}}, "mixed"}, + {"an absent period counts as monthly when mixed", []APIAddon{{Value: 50}, {Value: 5000000, Period: "monthly"}, {Value: 7, Period: "weekly"}}, "mixed"}, + {"an absent period never surfaces as empty", []APIAddon{{Value: 50}, {Value: 60, Period: "weekly"}}, "mixed"}, } for _, tt := range tests { diff --git a/internal/api/bulk_split.go b/internal/api/bulk_split.go index c5f53fd..5863074 100644 --- a/internal/api/bulk_split.go +++ b/internal/api/bulk_split.go @@ -75,7 +75,7 @@ func PlanBulk(params map[string]string, now time.Time) (BulkPlan, error) { plan.Window = BulkWindow(plan.Interval) raw, ok := params["s"] if !ok { - return plan, fmt.Errorf("bulk metrics require --since (at most %s per request at %s; for a longer range add --split as well)", + return plan, fmt.Errorf("bulk metrics require --since (at most %s per request at %s; --split fetches a longer range in several requests)", days(plan.Window), plan.Interval) } since, err := strconv.ParseInt(raw, 10, 64) @@ -83,6 +83,10 @@ func PlanBulk(params map[string]string, now time.Time) (BulkPlan, error) { return plan, fmt.Errorf("parameter s: expected Unix seconds, got %q", raw) } plan.Since = time.Unix(since, 0).UTC() + // boundedByNow: the range ends at now, either because --until was not + // given or because it lay in the future; the error then talks about now, + // not about an --until the user never saw. + boundedByNow := true if raw, ok := params["u"]; ok { until, err := strconv.ParseInt(raw, 10, 64) if err != nil { @@ -90,11 +94,12 @@ func PlanBulk(params map[string]string, now time.Time) (BulkPlan, error) { } if t := time.Unix(until, 0).UTC(); t.Before(plan.Until) { plan.Until = t + boundedByNow = false } } plan.Until = plan.Until.UTC() if !plan.Since.Before(plan.Until) { - if _, given := params["u"]; !given { + if boundedByNow { return plan, fmt.Errorf("--since (%s) is not in the past", plan.Since.Format(time.RFC3339)) } return plan, fmt.Errorf("--since (%s) must be before --until (%s)", plan.Since.Format(time.RFC3339), plan.Until.Format(time.RFC3339)) @@ -158,11 +163,14 @@ func (c *Client) GetMetricBulkSplit(ctx context.Context, path string, params map w := windows[0] resp, err := c.GetMetricBulk(ctx, path, WindowParams(params, w), repeated) if err != nil { - // A window shorter than one interval would return no points and - // turn the rest of the fetch into silent gaps, so halving stops there. - if isRangeTooLarge(err) && shrinks < maxWindowShrinks && plan.Window/2 >= intervalDuration(plan.Interval) { + // Halve the range the API actually rejected: near the end of the + // range it is shorter than plan.Window, and halving the nominal + // window could repeat the same request. A window shorter than one + // interval would return no points and turn the rest of the fetch + // into silent gaps, so halving stops there. + if halved := w.Until.Sub(w.Since) / 2; isRangeTooLarge(err) && shrinks < maxWindowShrinks && halved >= intervalDuration(plan.Interval) { shrinks++ - plan.Window /= 2 + plan.Window = halved continue } // The total is an estimate: a halved window changes how many remain. diff --git a/internal/api/bulk_split_test.go b/internal/api/bulk_split_test.go index d7c1014..a08ebb9 100644 --- a/internal/api/bulk_split_test.go +++ b/internal/api/bulk_split_test.go @@ -25,7 +25,7 @@ func unix(t time.Time) string { return strconv.FormatInt(t.Unix(), 10) } func TestPlanBulk(t *testing.T) { t.Run("requires since", func(t *testing.T) { _, err := PlanBulk(map[string]string{}, planNow) - if err == nil || !strings.Contains(err.Error(), "--since") || !strings.Contains(err.Error(), "31 days") || !strings.Contains(err.Error(), "add --split as well") { + if err == nil || !strings.Contains(err.Error(), "--since") || !strings.Contains(err.Error(), "31 days") || !strings.Contains(err.Error(), "--split fetches") { t.Fatalf("err = %v", err) } }) @@ -56,9 +56,33 @@ func TestPlanBulk(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "is not in the past") || strings.Contains(err.Error(), "--until") { t.Fatalf("since in the future: %v", err) } + // A future --until is capped at now; the error must not report the + // capped value as if the user had given it. + _, err = PlanBulk(map[string]string{"s": unix(planNow.Add(day)), "u": unix(planNow.Add(2 * day))}, planNow) + if err == nil || !strings.Contains(err.Error(), "is not in the past") || strings.Contains(err.Error(), "--until") { + t.Fatalf("since and until in the future: %v", err) + } }) } +// When the rejected window is the shorter tail of the range, halving the +// nominal window would repeat the identical request; the rejected range +// itself is halved instead. +func TestGetMetricBulkSplitHalvesTheRejectedRange(t *testing.T) { + plan := BulkPlan{Since: planSince, Until: planSince.Add(12 * day), Interval: "24h", Window: 31 * day} + server, calls, windows := bulkServer(t, 10*day, 0) + resp, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan) + if err != nil { + t.Fatal(err) + } + if calls.Load() != 3 || len(resp.Data) != 12 { + t.Fatalf("calls=%d points=%d, want 3 (12d rejected, then 6d + 6d) and 12", calls.Load(), len(resp.Data)) + } + if (*windows)[1].Until.Sub((*windows)[1].Since) != 6*day { + t.Errorf("windows=%v, want the second one to be 6 days", *windows) + } +} + func TestBulkPlanWindows(t *testing.T) { plan := BulkPlan{Since: planSince, Until: planSince.Add(70 * day), Interval: "24h", Window: 31 * day} windows := plan.Windows() From 8b1c2a09882f46e3cab182ddf97ae0424e654741 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Tue, 13 Oct 2026 09:52:13 +0200 Subject: [PATCH 24/37] GN-159: measure --since and the bulk plan against one clock --since 31d was parsed against one time.Now() and the bulk plan built against another a moment later, so the range came out a fraction of a second over the per-request window: without --split a range exactly at the limit was rejected, with --split a billed sub-second window followed. The command now reads the clock once, truncated to seconds, for parsing and planning, and PlanBulk truncates now itself. --- cmd/metric_get.go | 9 ++++++--- internal/api/bulk_split.go | 5 +++-- internal/api/bulk_split_test.go | 9 +++++++++ internal/timeparse/timeparse.go | 11 ++++++++++- internal/timeparse/timeparse_test.go | 11 +++++++++++ 5 files changed, 39 insertions(+), 6 deletions(-) diff --git a/cmd/metric_get.go b/cmd/metric_get.go index f590c51..081cdd6 100644 --- a/cmd/metric_get.go +++ b/cmd/metric_get.go @@ -38,15 +38,18 @@ var metricGetCmd = &cobra.Command{ params := map[string]string{} + // One clock for the whole command: --since 31d then ends exactly 31 + // days before the planned range's end, not a fraction of a second more. + now := time.Now().Truncate(time.Second) if since != "" { - ts, err := timeparse.Parse(since) + ts, err := timeparse.ParseAt(since, now) if err != nil { return fmt.Errorf("parsing --since: %w", err) } params["s"] = fmt.Sprintf("%d", ts) } if until != "" { - ts, err := timeparse.Parse(until) + ts, err := timeparse.ParseAt(until, now) if err != nil { return fmt.Errorf("parsing --until: %w", err) } @@ -73,7 +76,7 @@ var metricGetCmd = &cobra.Command{ // The API requires s for bulk requests and limits the range per // request; longer ranges are fetched in several requests with --split. - plan, err := api.PlanBulk(params, time.Now()) + plan, err := api.PlanBulk(params, now) if err != nil { return err } diff --git a/internal/api/bulk_split.go b/internal/api/bulk_split.go index 5863074..7f4c00a 100644 --- a/internal/api/bulk_split.go +++ b/internal/api/bulk_split.go @@ -66,9 +66,10 @@ type Window struct{ Since, Until time.Time } // PlanBulk reads the s, u and i parameters of a bulk request. s is required // by the API; u defaults to now and is capped at now, since the API returns no -// future points. +// future points. now is truncated to seconds, the API's resolution, so that a +// range ending now is never a fraction of a second over the window. func PlanBulk(params map[string]string, now time.Time) (BulkPlan, error) { - plan := BulkPlan{Interval: params["i"], Until: now} + plan := BulkPlan{Interval: params["i"], Until: now.Truncate(time.Second)} if plan.Interval == "" { plan.Interval = defaultResolution } diff --git a/internal/api/bulk_split_test.go b/internal/api/bulk_split_test.go index a08ebb9..d1b2f3c 100644 --- a/internal/api/bulk_split_test.go +++ b/internal/api/bulk_split_test.go @@ -41,6 +41,15 @@ func TestPlanBulk(t *testing.T) { } } }) + t.Run("a range exactly one window long is one request", func(t *testing.T) { + for interval, window := range map[string]time.Duration{"24h": 31 * day, "1h": 10 * day, "1month": 93 * day} { + now := planNow.Add(123456789 * time.Nanosecond) // a clock read after --since was parsed + plan, err := PlanBulk(map[string]string{"s": unix(planNow.Add(-window)), "i": interval}, now) + if err != nil || len(plan.Windows()) != 1 || !plan.Until.Equal(planNow) { + t.Errorf("%s: windows=%v err=%v, want one window ending at %v", interval, plan.Windows(), err, planNow) + } + } + }) t.Run("until in the future is capped at now", func(t *testing.T) { plan, err := PlanBulk(map[string]string{"s": unix(planSince), "u": unix(planNow.Add(400 * day))}, planNow) if err != nil || !plan.Until.Equal(planNow) { diff --git a/internal/timeparse/timeparse.go b/internal/timeparse/timeparse.go index 15d706a..a791840 100644 --- a/internal/timeparse/timeparse.go +++ b/internal/timeparse/timeparse.go @@ -8,13 +8,22 @@ import ( "unicode" ) +// Parse reads a time as Unix seconds, a date, an RFC 3339 timestamp or a +// duration back from now, such as 30d or 1h. func Parse(input string) (int64, error) { + return ParseAt(input, time.Now()) +} + +// ParseAt is Parse with relative durations measured back from now. A command +// that parses several times and plans requests from them should use one now, +// so that a range such as --since 31d is exactly 31 days long. +func ParseAt(input string, now time.Time) (int64, error) { if isAllDigits(input) { return strconv.ParseInt(input, 10, 64) } if dur, ok := parseRelativeDuration(input); ok { - return time.Now().Add(-dur).Unix(), nil + return now.Add(-dur).Unix(), nil } if t, err := time.Parse("2006-01-02", input); err == nil { diff --git a/internal/timeparse/timeparse_test.go b/internal/timeparse/timeparse_test.go index 577a5c1..2430ce8 100644 --- a/internal/timeparse/timeparse_test.go +++ b/internal/timeparse/timeparse_test.go @@ -85,3 +85,14 @@ func TestParse(t *testing.T) { } }) } + +func TestParseAt(t *testing.T) { + now := time.Date(2026, 10, 8, 12, 0, 0, 500, time.UTC) + got, err := ParseAt("31d", now) + if err != nil || got != now.Add(-31*24*time.Hour).Unix() { + t.Fatalf("ParseAt(31d) = %d, %v; want exactly 31 days before now", got, err) + } + if got, _ := ParseAt("2024-01-01", now); got != 1704067200 { + t.Errorf("absolute dates ignore now: %d", got) + } +} From 365edc451df251f25149b8d09eaefa36233fa341 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Tue, 13 Oct 2026 09:57:53 +0200 Subject: [PATCH 25/37] GN-159: halve bulk windows only with --split Every bulk request went through the split path, so when the API rejected a range inside the documented limit the CLI halved the window and sent several billed requests without the user's --split. Without --split the one planned request is sent and the API's rejection is reported, with a hint to add --split; IsRangeTooLarge is exported for that. --- cmd/metric_get.go | 14 +++++++++++++- cmd/metric_get_test.go | 31 +++++++++++++++++++++++++++++++ internal/api/bulk_split.go | 6 +++--- internal/api/bulk_split_test.go | 4 ++-- 4 files changed, 49 insertions(+), 6 deletions(-) diff --git a/cmd/metric_get.go b/cmd/metric_get.go index 081cdd6..3630204 100644 --- a/cmd/metric_get.go +++ b/cmd/metric_get.go @@ -101,7 +101,19 @@ var metricGetCmd = &cobra.Command{ return nil } - resp, err := client.GetMetricBulkSplit(cmd.Context(), path, params, repeatedParams, plan) + // --split is the only consent to more than one request: without it + // the single request goes out as planned, and a rejection of its + // range is reported rather than answered with halved windows. + var resp *api.BulkResponse + if split { + resp, err = client.GetMetricBulkSplit(cmd.Context(), path, params, repeatedParams, plan) + } else { + resp, err = client.GetMetricBulk(cmd.Context(), path, params, repeatedParams) + if api.IsRangeTooLarge(err) { + err = fmt.Errorf("%w; the range is within the documented limit of %d days at %s, add --split to retry it in shorter windows", + err, int(plan.Window.Hours()/24), plan.Interval) + } + } if err != nil { return err } diff --git a/cmd/metric_get_test.go b/cmd/metric_get_test.go index b55182a..93021bf 100644 --- a/cmd/metric_get_test.go +++ b/cmd/metric_get_test.go @@ -2,12 +2,16 @@ package cmd import ( "encoding/json" + "fmt" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" "runtime" "strings" "sync" + "sync/atomic" "testing" ) @@ -167,3 +171,30 @@ func TestMetricGet_InvalidUntil_Fails(t *testing.T) { t.Errorf("stderr should mention until/parsing: %q", stderr) } } + +// Without --split the CLI sends the one planned request and reports the API's +// rejection; it does not halve the window and multiply billed requests. +func TestMetricGet_Bulk_WithoutSplitSendsOneRequest(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.WriteHeader(http.StatusBadRequest) + fmt.Fprint(w, `{"message":"too large time range (744h0m0s) requested (maximum allowed is 240h0m0s)"}`) + })) + t.Cleanup(server.Close) + _, stderr, err := runCLI(t, []string{"HOME=" + t.TempDir(), "GLASSNODE_BASE_URL=" + server.URL}, + "metric", "get", "/market/price_usd_close/bulk", + "--api-key", "k", "--asset", "BTC", "--since", "31d", + ) + if err == nil { + t.Fatal("expected non-zero exit when the API rejects the range") + } + if calls.Load() != 1 { + t.Errorf("requests = %d, want exactly 1 without --split", calls.Load()) + } + for _, want := range []string{"too large time range", "add --split"} { + if !strings.Contains(stderr, want) { + t.Errorf("stderr %q lacks %q", stderr, want) + } + } +} diff --git a/internal/api/bulk_split.go b/internal/api/bulk_split.go index 7f4c00a..872aec0 100644 --- a/internal/api/bulk_split.go +++ b/internal/api/bulk_split.go @@ -169,7 +169,7 @@ func (c *Client) GetMetricBulkSplit(ctx context.Context, path string, params map // window could repeat the same request. A window shorter than one // interval would return no points and turn the rest of the fetch // into silent gaps, so halving stops there. - if halved := w.Until.Sub(w.Since) / 2; isRangeTooLarge(err) && shrinks < maxWindowShrinks && halved >= intervalDuration(plan.Interval) { + if halved := w.Until.Sub(w.Since) / 2; IsRangeTooLarge(err) && shrinks < maxWindowShrinks && halved >= intervalDuration(plan.Interval) { shrinks++ plan.Window = halved continue @@ -195,11 +195,11 @@ func (c *Client) GetMetricBulkSplit(ctx context.Context, path string, params map return merged, nil } -// isRangeTooLarge reports the API's rejection of a window as too long. Other +// IsRangeTooLarge reports the API's rejection of a window as too long. Other // 400s (an invalid asset, an unsupported parameter combination) are not // treated as a reason to halve: that would spend requests and replace the // API's message with one about window size. -func isRangeTooLarge(err error) bool { +func IsRangeTooLarge(err error) bool { var apiErr *glassnode.APIError return errors.As(err, &apiErr) && apiErr.StatusCode == 400 && strings.Contains(strings.ToLower(apiErr.Detail), "time range") } diff --git a/internal/api/bulk_split_test.go b/internal/api/bulk_split_test.go index d1b2f3c..e9ab84c 100644 --- a/internal/api/bulk_split_test.go +++ b/internal/api/bulk_split_test.go @@ -255,8 +255,8 @@ func TestIsRangeTooLarge(t *testing.T) { {400, "parameter a=NOPE is invalid", false}, {403, "time range not in your plan", false}, } { - if got := isRangeTooLarge(&glassnode.APIError{StatusCode: tt.status, Detail: tt.detail}); got != tt.want { - t.Errorf("isRangeTooLarge(%d %q) = %v, want %v", tt.status, tt.detail, got, tt.want) + if got := IsRangeTooLarge(&glassnode.APIError{StatusCode: tt.status, Detail: tt.detail}); got != tt.want { + t.Errorf("IsRangeTooLarge(%d %q) = %v, want %v", tt.status, tt.detail, got, tt.want) } } } From 84fd7933e0fe81711b0c442c5e2014005e9910cf Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Tue, 13 Oct 2026 10:12:19 +0200 Subject: [PATCH 26/37] GN-159: drop Client.Do and name the bulk window through the plan Do and DoWithRepeatedParams had no production callers since the SDK took over requests; the adapter tests that used them (headers, bearer refresh on 401, error bodies, repeated selectors) now go through GetMetric and GetMetricBulk. The --split hint reuses BulkPlan.WindowDays instead of recomputing the day count. --- cmd/metric_get.go | 4 +-- internal/api/bulk_split.go | 9 ++++--- internal/api/client.go | 15 ----------- internal/api/client_test.go | 54 ++++++++++++++++++------------------- 4 files changed, 35 insertions(+), 47 deletions(-) diff --git a/cmd/metric_get.go b/cmd/metric_get.go index 3630204..45dc87d 100644 --- a/cmd/metric_get.go +++ b/cmd/metric_get.go @@ -110,8 +110,8 @@ var metricGetCmd = &cobra.Command{ } else { resp, err = client.GetMetricBulk(cmd.Context(), path, params, repeatedParams) if api.IsRangeTooLarge(err) { - err = fmt.Errorf("%w; the range is within the documented limit of %d days at %s, add --split to retry it in shorter windows", - err, int(plan.Window.Hours()/24), plan.Interval) + err = fmt.Errorf("%w; the range is within the documented limit of %s at %s, add --split to retry it in shorter windows", + err, plan.WindowDays(), plan.Interval) } } if err != nil { diff --git a/internal/api/bulk_split.go b/internal/api/bulk_split.go index 872aec0..06cb4e4 100644 --- a/internal/api/bulk_split.go +++ b/internal/api/bulk_split.go @@ -77,7 +77,7 @@ func PlanBulk(params map[string]string, now time.Time) (BulkPlan, error) { raw, ok := params["s"] if !ok { return plan, fmt.Errorf("bulk metrics require --since (at most %s per request at %s; --split fetches a longer range in several requests)", - days(plan.Window), plan.Interval) + plan.WindowDays(), plan.Interval) } since, err := strconv.ParseInt(raw, 10, 64) if err != nil { @@ -124,16 +124,19 @@ func (plan BulkPlan) Windows() []Window { return windows } +// WindowDays is the window's length as shown to the user, such as "31 days". +func (plan BulkPlan) WindowDays() string { return days(plan.Window) } + // RangeError explains why the range needs --split. func (plan BulkPlan) RangeError() error { return fmt.Errorf("range %s..%s exceeds the API limit of %s per bulk request at %s; shorten it or add --split to fetch it in %d requests", - formatBound(plan.Since), formatBound(plan.Until), days(plan.Window), plan.Interval, len(plan.Windows())) + formatBound(plan.Since), formatBound(plan.Until), plan.WindowDays(), plan.Interval, len(plan.Windows())) } // Describe summarises the split for a warning to the user. func (plan BulkPlan) Describe() string { return fmt.Sprintf("splitting %s..%s into %d requests of up to %s (%s); the result is assembled in memory", - formatBound(plan.Since), formatBound(plan.Until), len(plan.Windows()), days(plan.Window), plan.Interval) + formatBound(plan.Since), formatBound(plan.Until), len(plan.Windows()), plan.WindowDays(), plan.Interval) } // WindowParams returns params with s and u set to the window's bounds. diff --git a/internal/api/client.go b/internal/api/client.go index 517e224..0a0967a 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -177,18 +177,3 @@ func RequireAPIKey(flagValue string) (string, error) { } return key, nil } - -func (c *Client) Do(ctx context.Context, method, path string, params map[string]string) ([]byte, error) { - return c.DoWithRepeatedParams(ctx, method, path, params, nil) -} - -func (c *Client) DoWithRepeatedParams(ctx context.Context, method, path string, params map[string]string, repeated map[string][]string) ([]byte, error) { - if method != http.MethodGet { - return nil, fmt.Errorf("unsupported HTTP method %s", method) - } - client, err := c.sdk() - if err != nil { - return nil, err - } - return client.Raw(ctx, path, queryValues(params, repeated)) -} diff --git a/internal/api/client_test.go b/internal/api/client_test.go index 0cabebb..2857b57 100644 --- a/internal/api/client_test.go +++ b/internal/api/client_test.go @@ -175,13 +175,13 @@ func TestResolveAuth_RefreshesExpiredAccessToken(t *testing.T) { }) } -func TestDo_SendsCorrectURL(t *testing.T) { +func TestRequest_SendsCorrectURL(t *testing.T) { var capturedURL, capturedKey string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { capturedURL = r.URL.String() capturedKey = r.Header.Get("X-Api-Key") w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("{}")) + _, _ = w.Write([]byte("[]")) })) defer server.Close() @@ -189,9 +189,9 @@ func TestDo_SendsCorrectURL(t *testing.T) { client.baseURL = server.URL client.httpClient = server.Client() - _, err := client.Do(context.Background(), "GET", "/v1/test", map[string]string{"a": "b"}) + _, err := client.GetMetric(context.Background(), "/test", map[string]string{"a": "b"}) if err != nil { - t.Fatalf("Do: %v", err) + t.Fatalf("GetMetric: %v", err) } if capturedKey != "my-api-key" || strings.Contains(capturedURL, "api_key") { t.Errorf("URL %q header %q: the key must travel in X-Api-Key only", capturedURL, capturedKey) @@ -201,13 +201,13 @@ func TestDo_SendsCorrectURL(t *testing.T) { } } -func TestDo_SendsBearerAuthorization(t *testing.T) { +func TestRequest_SendsBearerAuthorization(t *testing.T) { var authHdr, capturedURL string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { authHdr = r.Header.Get("Authorization") capturedURL = r.URL.String() w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("{}")) + _, _ = w.Write([]byte("[]")) })) defer server.Close() @@ -215,9 +215,9 @@ func TestDo_SendsBearerAuthorization(t *testing.T) { client.baseURL = server.URL client.httpClient = server.Client() - _, err := client.Do(context.Background(), "GET", "/v1/test", map[string]string{"a": "b"}) + _, err := client.GetMetric(context.Background(), "/test", map[string]string{"a": "b"}) if err != nil { - t.Fatalf("Do: %v", err) + t.Fatalf("GetMetric: %v", err) } if authHdr != "Bearer secret-token" { t.Errorf("Authorization header %q, want Bearer secret-token", authHdr) @@ -227,7 +227,7 @@ func TestDo_SendsBearerAuthorization(t *testing.T) { } } -func TestDo_401TriggersRefreshAndRetry(t *testing.T) { +func TestRequest_401TriggersRefreshAndRetry(t *testing.T) { withTempHome(t, func() { var mu sync.Mutex var authHdrs []string @@ -244,7 +244,7 @@ func TestDo_401TriggersRefreshAndRetry(t *testing.T) { return } w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(`{"ok":true}`)) + _, _ = w.Write([]byte(`[{"t":1,"v":1}]`)) })) defer api.Close() @@ -268,12 +268,12 @@ func TestDo_401TriggersRefreshAndRetry(t *testing.T) { client.baseURL = api.URL client.httpClient = api.Client() - body, err := client.Do(context.Background(), "GET", "/v1/anything", nil) + points, err := client.GetMetric(context.Background(), "/anything", nil) if err != nil { - t.Fatalf("Do: %v", err) + t.Fatalf("GetMetric: %v", err) } - if !strings.Contains(string(body), "ok") { - t.Errorf("body %q, want contents including 'ok'", string(body)) + if len(points) != 1 { + t.Errorf("points = %v, want the retried response", points) } if len(authHdrs) != 2 { t.Fatalf("expected 2 API attempts, got %d", len(authHdrs)) @@ -281,7 +281,7 @@ func TestDo_401TriggersRefreshAndRetry(t *testing.T) { if authHdrs[0] != "Bearer stale" || authHdrs[1] != "Bearer fresh" { t.Errorf("auth headers %v", authHdrs) } - if _, err := client.Do(context.Background(), "GET", "/v1/anything", nil); err != nil { + if _, err := client.GetMetric(context.Background(), "/anything", nil); err != nil { t.Fatal(err) } if len(authHdrs) != 3 || authHdrs[2] != "Bearer fresh" { @@ -290,7 +290,7 @@ func TestDo_401TriggersRefreshAndRetry(t *testing.T) { }) } -func TestDo_401WithAPIKey_NoRetry(t *testing.T) { +func TestRequest_401WithAPIKey_NoRetry(t *testing.T) { // When using an API key (not a bearer), a 401 must NOT trigger a refresh attempt; we just // surface the error. This guards against silently attempting OAuth refreshes for users who // never ran gn login. @@ -310,7 +310,7 @@ func TestDo_401WithAPIKey_NoRetry(t *testing.T) { client.baseURL = server.URL client.httpClient = server.Client() - _, err := client.Do(context.Background(), "GET", "/v1/x", nil) + _, err := client.GetMetric(context.Background(), "/x", nil) if err == nil { t.Fatal("expected error") } @@ -323,7 +323,7 @@ func TestDo_401WithAPIKey_NoRetry(t *testing.T) { }) } -func TestDo_Non2xxReturnsError(t *testing.T) { +func TestRequest_Non2xxReturnsError(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte("server error")) @@ -334,18 +334,18 @@ func TestDo_Non2xxReturnsError(t *testing.T) { client.baseURL = server.URL client.httpClient = server.Client() - _, err := client.Do(context.Background(), "GET", "/v1/test", nil) + _, err := client.GetMetric(context.Background(), "/test", nil) if err == nil { t.Error("expected error for 500 status") } } -func TestDoWithRepeatedParams(t *testing.T) { +func TestRequest_RepeatedParams(t *testing.T) { var capturedURL string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { capturedURL = r.URL.String() w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte("{}")) + _, _ = w.Write([]byte(`{"data":[]}`)) })) defer server.Close() @@ -354,9 +354,9 @@ func TestDoWithRepeatedParams(t *testing.T) { client.httpClient = server.Client() repeated := map[string][]string{"a": {"x", "y"}} - _, err := client.DoWithRepeatedParams(context.Background(), "GET", "/v1/test", nil, repeated) + _, err := client.GetMetricBulk(context.Background(), "/test", map[string]string{"s": "1"}, repeated) if err != nil { - t.Fatalf("DoWithRepeatedParams: %v", err) + t.Fatalf("GetMetricBulk: %v", err) } if !strings.Contains(capturedURL, "a=x") || !strings.Contains(capturedURL, "a=y") { t.Errorf("URL %q should contain a=x and a=y", capturedURL) @@ -749,7 +749,7 @@ func TestGetMetricBulk_EmptyData(t *testing.T) { // 4xx with body (point 7) -func TestDo_4xxReturnsErrorWithBody(t *testing.T) { +func TestRequest_4xxReturnsErrorWithBody(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusUnauthorized) _, _ = w.Write([]byte(`{"error":"Invalid API key"}`)) @@ -760,7 +760,7 @@ func TestDo_4xxReturnsErrorWithBody(t *testing.T) { client.baseURL = server.URL client.httpClient = server.Client() - _, err := client.Do(context.Background(), "GET", "/v1/test", nil) + _, err := client.GetMetric(context.Background(), "/test", nil) if err == nil { t.Fatal("expected error for 401 status") } @@ -841,7 +841,7 @@ func TestGetMetricBulk_PassesEntriesThrough(t *testing.T) { } // A failed refresh after a 401 surfaces its cause in the returned error. -func TestDo_401RefreshFailureReportsCause(t *testing.T) { +func TestRequest_401RefreshFailureReportsCause(t *testing.T) { withTempHome(t, func() { api := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusUnauthorized) @@ -863,7 +863,7 @@ func TestDo_401RefreshFailureReportsCause(t *testing.T) { client.baseURL = api.URL client.httpClient = api.Client() - _, err := client.Do(context.Background(), "GET", "/v1/anything", nil) + _, err := client.GetMetric(context.Background(), "/anything", nil) var authErr *glassnode.AuthError if !errors.As(err, &authErr) || !strings.Contains(err.Error(), "invalid_grant") || !strings.Contains(err.Error(), "refresh token revoked") { t.Fatalf("err = %v, want an AuthError carrying the IdP's cause", err) From 62f28058d45ca66c01c609973d228b95ef1228eb Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Tue, 13 Oct 2026 10:22:17 +0200 Subject: [PATCH 27/37] GN-159: send the planned bounds in the single bulk request Without --split the request carried the user's raw s and u, not the bounds PlanBulk validated and --dry-run prints: a future --until reached the API uncapped, could exceed the per-request limit, and the hint that the range was within the limit was then false. The single request now uses the planned window, like the split and dry-run paths. --- cmd/metric_get.go | 7 ++++--- cmd/metric_get_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/cmd/metric_get.go b/cmd/metric_get.go index 45dc87d..25058d2 100644 --- a/cmd/metric_get.go +++ b/cmd/metric_get.go @@ -102,13 +102,14 @@ var metricGetCmd = &cobra.Command{ } // --split is the only consent to more than one request: without it - // the single request goes out as planned, and a rejection of its - // range is reported rather than answered with halved windows. + // the single request goes out as planned, with the bounds PlanBulk + // validated and --dry-run prints, and a rejection of its range is + // reported rather than answered with halved windows. var resp *api.BulkResponse if split { resp, err = client.GetMetricBulkSplit(cmd.Context(), path, params, repeatedParams, plan) } else { - resp, err = client.GetMetricBulk(cmd.Context(), path, params, repeatedParams) + resp, err = client.GetMetricBulk(cmd.Context(), path, api.WindowParams(params, windows[0]), repeatedParams) if api.IsRangeTooLarge(err) { err = fmt.Errorf("%w; the range is within the documented limit of %s at %s, add --split to retry it in shorter windows", err, plan.WindowDays(), plan.Interval) diff --git a/cmd/metric_get_test.go b/cmd/metric_get_test.go index 93021bf..237db95 100644 --- a/cmd/metric_get_test.go +++ b/cmd/metric_get_test.go @@ -9,10 +9,12 @@ import ( "os/exec" "path/filepath" "runtime" + "strconv" "strings" "sync" "sync/atomic" "testing" + "time" ) // moduleRoot returns the path to the repository root (parent of cmd/). @@ -198,3 +200,29 @@ func TestMetricGet_Bulk_WithoutSplitSendsOneRequest(t *testing.T) { } } } + +// The one request sent without --split carries the planned bounds, the same +// ones --dry-run prints: a future --until is capped at now, so the API never +// sees a range the plan did not validate. +func TestMetricGet_Bulk_WithoutSplitSendsPlannedBounds(t *testing.T) { + start := time.Now() + var calls atomic.Int32 + var until string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + until = r.URL.Query().Get("u") + fmt.Fprint(w, `{"data":[]}`) + })) + t.Cleanup(server.Close) + _, stderr, err := runCLI(t, []string{"HOME=" + t.TempDir(), "GLASSNODE_BASE_URL=" + server.URL}, + "metric", "get", "/market/price_usd_close/bulk", + "--api-key", "k", "--asset", "BTC", "--since", "20d", "--until", "2999-01-01", + ) + if err != nil { + t.Fatalf("run CLI: %v\nstderr: %s", err, stderr) + } + u, convErr := strconv.ParseInt(until, 10, 64) + if calls.Load() != 1 || convErr != nil || u < start.Unix() || u > time.Now().Unix() { + t.Errorf("calls=%d u=%q, want one request with u capped at now", calls.Load(), until) + } +} From f277248555e41345c2b999479429016f4a4f2ffb Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Tue, 13 Oct 2026 10:25:46 +0200 Subject: [PATCH 28/37] GN-159: note on stderr when --split is given for a base metric --- cmd/metric_get.go | 5 +++++ cmd/metric_get_test.go | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/cmd/metric_get.go b/cmd/metric_get.go index 25058d2..1b3d954 100644 --- a/cmd/metric_get.go +++ b/cmd/metric_get.go @@ -123,6 +123,11 @@ var metricGetCmd = &cobra.Command{ return output.Print(output.Options{Format: format, Data: resp, TimestampFormat: tsFmt}) } + if split, _ := cmd.Flags().GetBool("split"); split { + // Base metrics have no per-request range limit, so there is nothing + // to split; say so rather than fail a script that shares its flags. + fmt.Fprintln(cmd.ErrOrStderr(), "--split applies to bulk paths only and was ignored") + } if len(assets) > 0 { params["a"] = assets[0] } diff --git a/cmd/metric_get_test.go b/cmd/metric_get_test.go index 237db95..98bbd1c 100644 --- a/cmd/metric_get_test.go +++ b/cmd/metric_get_test.go @@ -226,3 +226,15 @@ func TestMetricGet_Bulk_WithoutSplitSendsPlannedBounds(t *testing.T) { t.Errorf("calls=%d u=%q, want one request with u capped at now", calls.Load(), until) } } + +func TestMetricGet_SplitOnBaseMetricIsIgnoredWithANote(t *testing.T) { + stdout, stderr, err := runCLI(t, []string{"HOME=" + t.TempDir()}, + "metric", "get", "/market/price_usd_close", "--api-key", "k", "--since", "400d", "--split", "--dry-run", + ) + if err != nil { + t.Fatalf("run CLI: %v\nstderr: %s", err, stderr) + } + if !strings.Contains(stderr, "--split applies to bulk paths only") || strings.Contains(stdout, "split") { + t.Errorf("stderr=%q stdout=%q, want the note on stderr and a plain URL on stdout", stderr, stdout) + } +} From 62fe59a22a88bcb997bdd55dca5bddd66fb1342e Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Tue, 13 Oct 2026 10:30:49 +0200 Subject: [PATCH 29/37] GN-159: document that --dry-run previews the request built from flags --- README.md | 2 +- cmd/dryrun.go | 9 ++++++--- cmd/root.go | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0bd8a7a..a72a974 100644 --- a/README.md +++ b/README.md @@ -297,7 +297,7 @@ OAuth session fields (`oauth-access-token`, `oauth-refresh-token`, `oauth-expire |------|-------|-------------| | `--api-key` | | Glassnode API key | | `--output` | `-o` | Output format: `json` (default), `csv`, `table` | -| `--dry-run` | | Print the request URL without executing; the API key is sent in the `X-Api-Key` header and is not part of the URL | +| `--dry-run` | | Print the endpoint and parameters of the request without sending it. The API key is sent in the `X-Api-Key` header and is not part of the URL. The URL is built from the flags, not by the SDK: metric requests are sent with `f=json` appended, and parameter validation (empty values, `--since` after `--until`) happens only when the request is sent | | `--timestamp-format` | | Timestamp format in output | ## Output Formats diff --git a/cmd/dryrun.go b/cmd/dryrun.go index bcffdd5..a1917d5 100644 --- a/cmd/dryrun.go +++ b/cmd/dryrun.go @@ -7,9 +7,12 @@ import ( "github.com/spf13/cobra" ) -// printDryRun prints the request URL that --dry-run would send. The credential -// is sent in a header and is not part of the URL, which the note on stderr -// says, so the URL on stdout stays usable in scripts. +// printDryRun prints the endpoint and parameters --dry-run would send. The URL +// is assembled from the flags rather than by the SDK, so it is a preview, not +// the request byte for byte: the SDK appends f=json to metric requests and +// validates parameters only when sending. The credential is sent in a header +// and is not part of the URL, which the note on stderr says, so the URL on +// stdout stays usable in scripts. func printDryRun(cmd *cobra.Command, client *api.Client, url string) { fmt.Fprintln(cmd.OutOrStdout(), url) fmt.Fprintf(cmd.ErrOrStderr(), "# credentials are sent in the %s header, not in the URL\n", client.AuthHeader()) diff --git a/cmd/root.go b/cmd/root.go index f931b66..f35ef5a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -20,7 +20,7 @@ func init() { rootCmd.Version = "dev" rootCmd.PersistentFlags().String("api-key", "", "API key override") rootCmd.PersistentFlags().StringP("output", "o", "json", "output format: json, csv, table") - rootCmd.PersistentFlags().Bool("dry-run", false, "print the request URL without executing") + rootCmd.PersistentFlags().Bool("dry-run", false, "print the endpoint and parameters of the request without sending it (the SDK adds f=json to metric requests and validates parameters only when sending)") rootCmd.PersistentFlags().String("timestamp-format", "humanized", "timestamp format: unix, humanized, or Go layout (e.g. 2006-01-02 15:04:05); applies to table/csv output") rootCmd.SetVersionTemplate("gn version {{.Version}}\n") From f32bd00af2cd592417f3702cbd58ac630b63c197 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Tue, 13 Oct 2026 10:37:42 +0200 Subject: [PATCH 30/37] GN-159: hand metric selectors to the SDK as Extra, drop the http.Client copy MetricParams.Values reads s, u, i, c, a and e from Extra and validates them itself, so the CLI no longer re-types them through time.Time and back. WithHTTPClient copies the client's configuration, so the adapter's own copy was redundant. --- internal/api/client.go | 3 +-- internal/api/client_test.go | 31 ++++++++++++++------- internal/api/metrics.go | 54 +++---------------------------------- 3 files changed, 26 insertions(+), 62 deletions(-) diff --git a/internal/api/client.go b/internal/api/client.go index 0a0967a..dc3795b 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -41,7 +41,6 @@ func NewClient(apiKey, bearerToken string) *Client { func (c *Client) sdk() (*glassnode.Client, error) { c.init.Do(func() { - httpClient := *c.httpClient options := []glassnode.Option{ glassnode.WithBaseURL(c.baseURL), glassnode.WithUserAgent("glassnode-cli-" + version.Version), @@ -51,7 +50,7 @@ func (c *Client) sdk() (*glassnode.Client, error) { options = append(options, glassnode.WithTokenSource(&sessionTokens{token: c.bearerToken})) key = "" } - options = append(options, glassnode.WithHTTPClient(&httpClient)) + options = append(options, glassnode.WithHTTPClient(c.httpClient)) c.sdkClient, c.initErr = glassnode.NewClient(key, options...) }) return c.sdkClient, c.initErr diff --git a/internal/api/client_test.go b/internal/api/client_test.go index 2857b57..92a6741 100644 --- a/internal/api/client_test.go +++ b/internal/api/client_test.go @@ -7,6 +7,7 @@ import ( "errors" "net/http" "net/http/httptest" + "net/url" "os" "strings" "sync" @@ -798,19 +799,31 @@ func TestGetMetricBulk_KeepsAllSelectors(t *testing.T) { } } -func TestMetricParamsMapsTypedFields(t *testing.T) { - p, err := metricParams(map[string]string{"s": "100", "u": "200", "i": "24h", "c": "USD", "a": "BTC", "network": "eth"}, map[string][]string{"e": {"binance", "coinbase"}}) +// Selectors reach the request as given; the SDK validates them, so a +// malformed s is rejected before anything is sent. +func TestGetMetric_SelectorsReachTheRequest(t *testing.T) { + var query url.Values + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + query = r.URL.Query() + _, _ = w.Write([]byte(`{"data":[]}`)) + })) + defer server.Close() + client := NewClient("key", "") + client.baseURL = server.URL + client.httpClient = server.Client() + + _, err := client.GetMetricBulk(context.Background(), "/market/price", map[string]string{"s": "100", "u": "200", "i": "24h", "c": "USD", "network": "eth"}, map[string][]string{"a": {"BTC", "ETH"}, "e": {"binance", "coinbase"}}) if err != nil { t.Fatal(err) } - if p.Since.Unix() != 100 || p.Until.Unix() != 200 || p.Interval != "24h" || p.Currency != "USD" || p.Asset != "BTC" || len(p.Exchanges) != 2 { - t.Errorf("typed fields: %+v", p) - } - if p.Extra.Get("network") != "eth" || p.Extra.Has("s") || p.Extra.Has("a") || p.Extra.Has("e") { - t.Errorf("extra: %v", p.Extra) + if query.Get("s") != "100" || query.Get("u") != "200" || query.Get("i") != "24h" || query.Get("c") != "USD" || query.Get("network") != "eth" || len(query["a"]) != 2 || len(query["e"]) != 2 { + t.Errorf("query: %v", query) } - if _, err := metricParams(map[string]string{"s": "yesterday"}, nil); err == nil { - t.Error("accepted non-numeric since") + calls := 0 + server.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls++ }) + var inputErr *glassnode.InputError + if _, err := client.GetMetric(context.Background(), "/market/price", map[string]string{"s": "yesterday"}); !errors.As(err, &inputErr) || calls != 0 { + t.Errorf("err = %v, calls = %d; want an InputError and no request", err, calls) } } diff --git a/internal/api/metrics.go b/internal/api/metrics.go index 641be3a..cba3837 100644 --- a/internal/api/metrics.go +++ b/internal/api/metrics.go @@ -3,9 +3,7 @@ package api import ( "context" "fmt" - "strconv" "strings" - "time" glassnode "github.com/glassnode/glassnode-api-go-client" ) @@ -30,56 +28,13 @@ type BulkResponse struct { Data []BulkDataPoint `json:"data"` } -// metricParams maps the CLI's flat query parameters onto the SDK's typed -// fields, so the SDK validates the time range and selectors; anything it has -// no field for stays in Extra. -func metricParams(params map[string]string, repeated map[string][]string) (*glassnode.MetricParams, error) { - p := &glassnode.MetricParams{Extra: queryValues(params, repeated)} - for key, value := range params { - switch key { - case "s", "u": - seconds, err := strconv.ParseInt(value, 10, 64) - if err != nil { - return nil, fmt.Errorf("parameter %s: expected Unix seconds, got %q", key, value) - } - if key == "s" { - p.Since = time.Unix(seconds, 0) - } else { - p.Until = time.Unix(seconds, 0) - } - case "i": - p.Interval = value - case "c": - p.Currency = value - case "a": - p.Asset = value - case "e": - p.Exchanges = []string{value} - default: - continue - } - p.Extra.Del(key) - } - if assets := repeated["a"]; len(assets) > 0 { - p.Assets = assets - p.Extra.Del("a") - } - if exchanges := repeated["e"]; len(exchanges) > 0 { - p.Exchanges = exchanges - p.Extra.Del("e") - } - return p, nil -} - func (c *Client) GetMetric(ctx context.Context, path string, params map[string]string) ([]DataPoint, error) { client, err := c.sdk() if err != nil { return nil, err } - p, err := metricParams(params, nil) - if err != nil { - return nil, err - } + // The SDK reads s, u, i, c, a and e from Extra and validates them. + p := &glassnode.MetricParams{Extra: queryValues(params, nil)} var points []DataPoint if err := client.GetMetric(ctx, path, p, &points); err != nil { return nil, fmt.Errorf("getting metric: %w", err) @@ -97,10 +52,7 @@ func (c *Client) GetMetricBulk(ctx context.Context, path string, params map[stri if err != nil { return nil, err } - p, err := metricParams(params, repeated) - if err != nil { - return nil, err - } + p := &glassnode.MetricParams{Extra: queryValues(params, repeated)} var response BulkResponse if err := client.GetMetric(ctx, strings.TrimSuffix(NormalizePath(path), "/")+"/bulk", p, &response); err != nil { return nil, fmt.Errorf("getting bulk metric: %w", err) From c5b1197579e20e4dbcd3e39273b8cea0f568e12e Mon Sep 17 00:00:00 2001 From: planadecu Date: Tue, 13 Oct 2026 10:55:02 +0200 Subject: [PATCH 31/37] GN-159: plan bulk windows on the interval bounds the API measures The API rounds s and u down to the start of their interval and checks the range between the rounded bounds. PlanBulk checked the raw range, so --since 93d -i 1w passed the plan and was rejected with HTTP 400 (98 days after rounding), and --split paid one rejected request per window before halving. Round --since down to its interval start and put inner window bounds on interval starts, so every window is the range the API checks. --- README.md | 2 +- internal/api/bulk_split.go | 48 +++++++++++++++++++++----- internal/api/bulk_split_test.go | 60 +++++++++++++++++++++++++++++---- 3 files changed, 94 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index a72a974..d053313 100644 --- a/README.md +++ b/README.md @@ -239,7 +239,7 @@ For **bulk metrics**, append `/bulk` to the path (e.g. `market/marketcap_usd/bul Point-in-time metrics (`*_pit`) include `computed_at`, the time the value was computed, as a field in JSON and as a trailing column in CSV and table output. -The API requires `--since` for bulk metrics and accepts at most 10 days per request at `10m`/`1h`, 31 days at `24h` and 93 days at `1w`/`1month`. A longer range is rejected before any request is sent unless you pass `--split`: the CLI then fetches it window by window, prints how many requests that takes on stderr, and merges the points. Every request is billed per asset, so narrow `-a` for long ranges. The merged result is assembled in memory before it is printed. +The API requires `--since` for bulk metrics and accepts at most 10 days per request at `10m`/`1h`, 31 days at `24h` and 93 days at `1w`/`1month`, counted from the start of the interval containing `--since` (weeks start on Monday, months on the 1st, UTC). A longer range is rejected before any request is sent unless you pass `--split`: the CLI then fetches it window by window, prints how many requests that takes on stderr, and merges the points. Every request is billed per asset, so narrow `-a` for long ranges. The merged result is assembled in memory before it is printed. ### `gn user credits` diff --git a/internal/api/bulk_split.go b/internal/api/bulk_split.go index 06cb4e4..4c795cd 100644 --- a/internal/api/bulk_split.go +++ b/internal/api/bulk_split.go @@ -41,6 +41,26 @@ func intervalDuration(interval string) time.Duration { } } +// intervalStart returns the start of the interval containing t, in UTC: weeks +// start on Monday, months on the 1st. The API rounds s and u down this way and +// measures the range between the rounded bounds. +func intervalStart(t time.Time, interval string) time.Time { + t = t.UTC() + switch interval { + case "10m": + return t.Truncate(10 * time.Minute) + case "1h": + return t.Truncate(time.Hour) + case "1w": + midnight := t.Truncate(24 * time.Hour) + return midnight.AddDate(0, 0, -(int(midnight.Weekday())+6)%7) + case "1month": + return time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, time.UTC) + default: + return t.Truncate(24 * time.Hour) + } +} + // BulkWindow returns the longest time range the API accepts in one bulk // request at the given resolution. func BulkWindow(interval string) time.Duration { @@ -83,7 +103,9 @@ func PlanBulk(params map[string]string, now time.Time) (BulkPlan, error) { if err != nil { return plan, fmt.Errorf("parameter s: expected Unix seconds, got %q", raw) } - plan.Since = time.Unix(since, 0).UTC() + // The API counts the range from the start of the interval containing s, + // so the plan does too: at 1w, --since 93d reaches back up to 99 days. + plan.Since = intervalStart(time.Unix(since, 0), plan.Interval) // boundedByNow: the range ends at now, either because --until was not // given or because it lay in the future; the error then talks about now, // not about an --until the user never saw. @@ -108,18 +130,26 @@ func PlanBulk(params map[string]string, now time.Time) (BulkPlan, error) { return plan, nil } -// Windows splits the range into consecutive windows of at most plan.Window. -// Each window's Until is the next window's Since: the API includes the -// interval containing s and only intervals that end by u, so consecutive -// windows neither overlap nor leave gaps. +// Windows splits the range into consecutive windows the API accepts: from the +// start of the interval containing s to the start of the one containing u is +// at most plan.Window. Inner bounds are interval starts, and each window's +// Until is the next window's Since: the API includes the interval containing s +// and only intervals that end by u, so consecutive windows neither overlap nor +// leave gaps. func (plan BulkPlan) Windows() []Window { var windows []Window - for since := plan.Since; since.Before(plan.Until); since = since.Add(plan.Window) { - until := since.Add(plan.Window) - if until.After(plan.Until) { - until = plan.Until + for since := plan.Since; since.Before(plan.Until); { + limit := since.Add(plan.Window) + until := plan.Until + if intervalStart(until, plan.Interval).After(limit) { + until = intervalStart(limit, plan.Interval) + if !until.After(since) { + // A window shorter than one interval: no bound to round to. + until = limit + } } windows = append(windows, Window{since, until}) + since = until } return windows } diff --git a/internal/api/bulk_split_test.go b/internal/api/bulk_split_test.go index e9ab84c..d50a272 100644 --- a/internal/api/bulk_split_test.go +++ b/internal/api/bulk_split_test.go @@ -42,7 +42,7 @@ func TestPlanBulk(t *testing.T) { } }) t.Run("a range exactly one window long is one request", func(t *testing.T) { - for interval, window := range map[string]time.Duration{"24h": 31 * day, "1h": 10 * day, "1month": 93 * day} { + for interval, window := range map[string]time.Duration{"24h": 31 * day, "1h": 10 * day, "1w": 93 * day, "1month": 93 * day} { now := planNow.Add(123456789 * time.Nanosecond) // a clock read after --since was parsed plan, err := PlanBulk(map[string]string{"s": unix(planNow.Add(-window)), "i": interval}, now) if err != nil || len(plan.Windows()) != 1 || !plan.Until.Equal(planNow) { @@ -50,6 +50,20 @@ func TestPlanBulk(t *testing.T) { } } }) + // The API rounds s down to the start of its interval: at 1w, --since 93d + // on a Wednesday counts from the Monday before and needs two requests. + t.Run("since is rounded down to the start of its interval", func(t *testing.T) { + now := time.Date(2026, 10, 13, 9, 0, 0, 0, time.UTC) // a Tuesday + plan, err := PlanBulk(map[string]string{"s": unix(now.Add(-93 * day)), "i": "1w"}, now) + if err != nil { + t.Fatal(err) + } + monday := time.Date(2026, 7, 6, 0, 0, 0, 0, time.UTC) + windows := plan.Windows() + if !plan.Since.Equal(monday) || len(windows) != 2 || !windows[0].Until.Equal(monday.Add(91*day)) || !windows[1].Until.Equal(now) { + t.Fatalf("since=%v windows=%v, want 2026-07-06 and a 91-day first window", plan.Since, windows) + } + }) t.Run("until in the future is capped at now", func(t *testing.T) { plan, err := PlanBulk(map[string]string{"s": unix(planSince), "u": unix(planNow.Add(400 * day))}, planNow) if err != nil || !plan.Until.Equal(planNow) { @@ -288,17 +302,51 @@ func TestFormatBound(t *testing.T) { } } -// At monthly resolution a 93-day window may be halved once (46 days, still -// over one interval) but not twice (23 days would return no points). +// At monthly resolution a 93-day window may be halved once (46.5 days, which +// rounds down to one month) but not twice (15.5 days would return no points). func TestGetMetricBulkSplitDoesNotHalveBelowAnInterval(t *testing.T) { plan := BulkPlan{Since: planSince, Until: planSince.Add(93 * day), Interval: "1month", Window: 93 * day} - server, calls, windows := bulkServer(t, 40*day, 0) // even 46 days is rejected + server, calls, windows := bulkServer(t, 20*day, 0) // even one month is rejected _, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan) if err == nil || !strings.Contains(err.Error(), "time range") { t.Fatalf("err = %v, want the API's too-large error once halving is exhausted", err) } - if calls.Load() != 2 || (*windows)[1].Until.Sub((*windows)[1].Since) != 46*day+12*time.Hour { - t.Errorf("calls=%d windows=%v, want 93d then 46.5d and no 23d window", calls.Load(), *windows) + if calls.Load() != 2 || !(*windows)[1].Until.Equal(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC)) { + t.Errorf("calls=%d windows=%v, want 93d then January and no shorter window", calls.Load(), *windows) + } +} + +func TestIntervalStart(t *testing.T) { + at := time.Date(2026, 10, 15, 13, 47, 5, 0, time.UTC) // a Thursday + for interval, want := range map[string]time.Time{ + "10m": time.Date(2026, 10, 15, 13, 40, 0, 0, time.UTC), + "1h": time.Date(2026, 10, 15, 13, 0, 0, 0, time.UTC), + "24h": time.Date(2026, 10, 15, 0, 0, 0, 0, time.UTC), + "1w": time.Date(2026, 10, 12, 0, 0, 0, 0, time.UTC), + "1month": time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC), + } { + if got := intervalStart(at, interval); !got.Equal(want) { + t.Errorf("%s: %v, want %v", interval, got, want) + } + } + if monday := time.Date(2026, 10, 12, 0, 0, 0, 0, time.UTC); !intervalStart(monday, "1w").Equal(monday) { + t.Errorf("a Monday is its own week start") + } +} + +// Inner window bounds fall on interval starts, so the API rounds neither of +// them and each window is exactly the range the plan checked. +func TestBulkPlanWindowsAlignToIntervals(t *testing.T) { + monday := time.Date(2026, 1, 5, 0, 0, 0, 0, time.UTC) + plan := BulkPlan{Since: monday, Until: monday.Add(300 * day), Interval: "1w", Window: 93 * day} + windows := plan.Windows() + for i, w := range windows { + if i < len(windows)-1 && (w.Until.Weekday() != time.Monday || w.Until.Sub(w.Since) > 93*day) { + t.Errorf("window %d = %v, want a Monday bound at most 93 days on", i, w) + } + if i > 0 && !w.Since.Equal(windows[i-1].Until) { + t.Errorf("gap or overlap between %v and %v", windows[i-1], w) + } } } From e17a8b19deb5b518fe6ef419ec7eaf0ed4eb0cd0 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Tue, 13 Oct 2026 11:02:44 +0200 Subject: [PATCH 32/37] GN-159: validate the dry-run base URL as the SDK does --- internal/api/client_test.go | 15 +++++++++++++++ internal/api/metadata.go | 7 ++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/internal/api/client_test.go b/internal/api/client_test.go index 92a6741..d2b51dc 100644 --- a/internal/api/client_test.go +++ b/internal/api/client_test.go @@ -382,6 +382,21 @@ func TestBuildURL(t *testing.T) { } } +func TestBuildURL_BaseURLAsTheSDKSeesIt(t *testing.T) { + client := NewClient("test-key", "") + client.baseURL = "https://api.example.com/" + got, err := client.BuildURL("/v1/path", nil, nil) + if err != nil || got != "https://api.example.com/v1/path" { + t.Errorf("trailing slash: got %q, %v", got, err) + } + bad := NewClient("test-key", "") + bad.baseURL = "api.example.com" + var inputErr *glassnode.InputError + if _, err := bad.BuildURL("/v1/path", nil, nil); !errors.As(err, &inputErr) { + t.Errorf("a base URL the SDK rejects must fail the dry run too, got %v", err) + } +} + func TestListAssets(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") diff --git a/internal/api/metadata.go b/internal/api/metadata.go index 11bd94f..a2ff8a4 100644 --- a/internal/api/metadata.go +++ b/internal/api/metadata.go @@ -93,8 +93,13 @@ func (c *Client) DescribeMetric(ctx context.Context, path, asset string) (*Metri // BuildURL constructs the request URL without executing the request. The // credential is not part of it: it travels in the header named by AuthHeader. +// The base URL is validated as the SDK validates it, so a dry run fails where +// the request would, and trailing slashes are dropped as the SDK drops them. func (c *Client) BuildURL(path string, params map[string]string, repeatedParams map[string][]string) (string, error) { - u, err := url.Parse(c.baseURL + path) + if _, err := c.sdk(); err != nil { + return "", err + } + u, err := url.Parse(strings.TrimRight(c.baseURL, "/") + path) if err != nil { return "", err } From 7a52107df19448d702f2e8c3d0b62397525c9f76 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Tue, 13 Oct 2026 11:02:45 +0200 Subject: [PATCH 33/37] GN-159: document bulk --since, range caps, --split and dry-run in SKILL.md --- skills/glassnode-cli/SKILL.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/skills/glassnode-cli/SKILL.md b/skills/glassnode-cli/SKILL.md index 9c98d00..9c36418 100644 --- a/skills/glassnode-cli/SKILL.md +++ b/skills/glassnode-cli/SKILL.md @@ -60,10 +60,17 @@ gn metric get distribution/balance_exchanges --asset BTC --exchange binance --cu ``` ### Bulk fetch (multiple assets) -Append `/bulk` to the metric path. Repeat `-a` (or `--asset`) for each asset, or use `-a '*'` for all: +Append `/bulk` to the metric path. Repeat `-a` (or `--asset`) for each asset, or use `-a '*'` for all. +`--since` is required: the API caps one bulk request at 10 days for `10m`/`1h`, 31 days for `24h` +(the default) and 93 days for `1w`/`1month`, counted from the start of the interval containing +`--since` (weeks start on Monday, months on the 1st, UTC). A longer range is rejected before any request is sent +unless `--split` is passed; the CLI then fetches consecutive windows, prints the number of requests +on stderr, and aborts on any error naming the window and the `--since` to retry from. Each window +is a billed request. `--split` on a non-bulk path is ignored with a note on stderr. ```bash -gn metric get market/marketcap_usd/bulk -a BTC -a ETH -a SOL -s 1d +gn metric get market/marketcap_usd/bulk -a BTC -a ETH -a SOL --since 7d gn metric get market/marketcap_usd/bulk -a '*' --interval 24h --since 30d +gn metric get market/marketcap_usd/bulk -a BTC -a ETH --since 2024-01-01 --until 2024-04-01 --split ``` ### API credit usage (`gn user credits`) @@ -107,15 +114,15 @@ gn config get all |------|-------|-------------| | `--api-key` | | Glassnode API key | | `--output` | `-o` | Output format: `json` (default), `csv`, `table` | -| `--dry-run` | | Print the request URL without executing | +| `--dry-run` | | Print the endpoint and parameters without sending the request; the API key goes in the `X-Api-Key` header, not the URL. A preview built from the flags: the SDK appends `f=json` to metric requests and validates parameters only when sending | | `--timestamp-format` | | Timestamp format in output | ## Best Practices - Always call `gn metric describe ` before `gn metric get` to discover valid parameters, assets, exchanges, and intervals for a metric. - Use `--output json` (default) for structured output suitable for piping to `jq`. -- Use `--dry-run` to preview the API request URL without executing: `gn metric get market/price_usd_close --asset BTC --since 30d --dry-run`. -- For bulk metrics (path ending in `/bulk`), use `-a` with specific assets or `-a '*'` for all. +- Use `--dry-run` to preview the request without sending it: `gn metric get market/price_usd_close --asset BTC --since 30d --dry-run`. The URL has no API key; it is sent in the `X-Api-Key` header. +- For bulk metrics (path ending in `/bulk`), use `-a` with specific assets or `-a '*'` for all. `--since` is required; pass `--split` to fetch a range longer than the per-request cap in several billed requests. - Relative time values are supported for `--since` and `--until`: e.g. `30d`, `7d`, `1h`. ## Output Formats From 213d42c51bae7078b33e0d5c350cdaf0a340ffcf Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Tue, 13 Oct 2026 11:12:07 +0200 Subject: [PATCH 34/37] GN-159: report split progress per window, match only "too large" rejections Each split window goes through the SDK's retries and may wait on a rate limit, so a long fetch fell silent after the "splitting into N requests" line; a stderr line per window names its number and bounds. The too-large check matched any 400 mentioning "time range" and attached the --split hint to unrelated range errors; it now requires "too large". --- internal/api/bulk_split.go | 14 +++++++++----- internal/api/bulk_split_test.go | 8 ++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/internal/api/bulk_split.go b/internal/api/bulk_split.go index 4c795cd..96b4a7b 100644 --- a/internal/api/bulk_split.go +++ b/internal/api/bulk_split.go @@ -195,6 +195,9 @@ func (c *Client) GetMetricBulkSplit(ctx context.Context, path string, params map for since.Before(plan.Until) { windows := BulkPlan{Since: since, Until: plan.Until, Interval: plan.Interval, Window: plan.Window}.Windows() w := windows[0] + // One line per window: each goes through the SDK's retries and may + // wait on a rate limit, so a long split fetch should not fall silent. + fmt.Fprintf(stderr, "window %d/%d %s..%s\n", fetched+1, fetched+len(windows), formatBound(w.Since), formatBound(w.Until)) resp, err := c.GetMetricBulk(ctx, path, WindowParams(params, w), repeated) if err != nil { // Halve the range the API actually rejected: near the end of the @@ -228,13 +231,14 @@ func (c *Client) GetMetricBulkSplit(ctx context.Context, path string, params map return merged, nil } -// IsRangeTooLarge reports the API's rejection of a window as too long. Other -// 400s (an invalid asset, an unsupported parameter combination) are not -// treated as a reason to halve: that would spend requests and replace the -// API's message with one about window size. +// IsRangeTooLarge reports the API's rejection of a window as too long ("too +// large time range (...) requested"). Other 400s, including other complaints +// about the range, are not treated as a reason to halve or to suggest --split: +// that would spend requests and replace the API's message with one about +// window size. func IsRangeTooLarge(err error) bool { var apiErr *glassnode.APIError - return errors.As(err, &apiErr) && apiErr.StatusCode == 400 && strings.Contains(strings.ToLower(apiErr.Detail), "time range") + return errors.As(err, &apiErr) && apiErr.StatusCode == 400 && strings.Contains(strings.ToLower(apiErr.Detail), "too large") } // formatBound shows a window bound as a date when it is at midnight UTC and diff --git a/internal/api/bulk_split_test.go b/internal/api/bulk_split_test.go index d50a272..53e148a 100644 --- a/internal/api/bulk_split_test.go +++ b/internal/api/bulk_split_test.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "os" "strconv" "strings" "sync/atomic" @@ -171,10 +172,16 @@ func newTestClient(server *httptest.Server) *Client { func TestGetMetricBulkSplit(t *testing.T) { plan := BulkPlan{Since: planSince, Until: planSince.Add(70 * day), Interval: "24h", Window: 31 * day} server, calls, _ := bulkServer(t, 31*day, 0) + var progress strings.Builder + stderr = &progress + t.Cleanup(func() { stderr = os.Stderr }) resp, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan) if err != nil { t.Fatal(err) } + if got := progress.String(); !strings.Contains(got, "window 1/3 2024-01-01..2024-02-01") || !strings.Contains(got, "window 3/3 ") { + t.Errorf("progress:\n%s", got) + } if calls.Load() != 3 || len(resp.Data) != 70 { t.Fatalf("calls=%d points=%d, want 3 and 70", calls.Load(), len(resp.Data)) } @@ -267,6 +274,7 @@ func TestIsRangeTooLarge(t *testing.T) { {400, "too large time range (155664h0m0s) requested (maximum allowed is 744h0m0s)", true}, {400, "Requested Time Range is too large", true}, {400, "parameter a=NOPE is invalid", false}, + {400, "invalid time range: s must be before u", false}, {403, "time range not in your plan", false}, } { if got := IsRangeTooLarge(&glassnode.APIError{StatusCode: tt.status, Detail: tt.detail}); got != tt.want { From a861c2d71a7e868fd1442a9824eb365e443c3e23 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Tue, 13 Oct 2026 11:29:57 +0200 Subject: [PATCH 35/37] GN-159: refuse unknown bulk intervals and make abort, progress and dry-run output honest The abort error told the user to retry with --since at the failed window although nothing had been printed, and said so even for a too-large rejection that would recur; it now says no data was written and names a remedy that fits. An interval the CLI does not know was planned as 24h and, with --split, billed before the API refused it; PlanBulk refuses it first. Per-window progress goes to the writer the command passes, the dry run prints the header note once and no split summary, and the note names the Authorization header rather than "Authorization: Bearer". --- cmd/dryrun.go | 8 +++++--- cmd/metric_get.go | 15 ++++++++------ cmd/metric_get_test.go | 18 +++++++++++++++++ internal/api/bulk_split.go | 30 ++++++++++++++++++++++++---- internal/api/bulk_split_test.go | 35 +++++++++++++++++++-------------- internal/api/client.go | 11 ++++++----- 6 files changed, 84 insertions(+), 33 deletions(-) diff --git a/cmd/dryrun.go b/cmd/dryrun.go index a1917d5..71433aa 100644 --- a/cmd/dryrun.go +++ b/cmd/dryrun.go @@ -13,7 +13,9 @@ import ( // validates parameters only when sending. The credential is sent in a header // and is not part of the URL, which the note on stderr says, so the URL on // stdout stays usable in scripts. -func printDryRun(cmd *cobra.Command, client *api.Client, url string) { - fmt.Fprintln(cmd.OutOrStdout(), url) - fmt.Fprintf(cmd.ErrOrStderr(), "# credentials are sent in the %s header, not in the URL\n", client.AuthHeader()) +func printDryRun(cmd *cobra.Command, client *api.Client, urls ...string) { + for _, url := range urls { + fmt.Fprintln(cmd.OutOrStdout(), url) + } + fmt.Fprintf(cmd.ErrOrStderr(), "# credentials are sent in the %s, not in the URL\n", client.AuthHeader()) } diff --git a/cmd/metric_get.go b/cmd/metric_get.go index 1b3d954..14cb007 100644 --- a/cmd/metric_get.go +++ b/cmd/metric_get.go @@ -85,21 +85,24 @@ var metricGetCmd = &cobra.Command{ if len(windows) > 1 && !split { return plan.RangeError() } - if len(windows) > 1 { - fmt.Fprintln(cmd.ErrOrStderr(), plan.Describe()) - } - dryRun, _ := cmd.Flags().GetBool("dry-run") if dryRun { + // One URL per window and the header note once; nothing is + // fetched, so the split summary does not apply. + urls := make([]string, 0, len(windows)) for _, w := range windows { u, err := client.BuildURL("/v1/metrics"+path+"/bulk", api.WindowParams(params, w), repeatedParams) if err != nil { return err } - printDryRun(cmd, client, u) + urls = append(urls, u) } + printDryRun(cmd, client, urls...) return nil } + if len(windows) > 1 { + fmt.Fprintln(cmd.ErrOrStderr(), plan.Describe()) + } // --split is the only consent to more than one request: without it // the single request goes out as planned, with the bounds PlanBulk @@ -107,7 +110,7 @@ var metricGetCmd = &cobra.Command{ // reported rather than answered with halved windows. var resp *api.BulkResponse if split { - resp, err = client.GetMetricBulkSplit(cmd.Context(), path, params, repeatedParams, plan) + resp, err = client.GetMetricBulkSplit(cmd.Context(), path, params, repeatedParams, plan, cmd.ErrOrStderr()) } else { resp, err = client.GetMetricBulk(cmd.Context(), path, api.WindowParams(params, windows[0]), repeatedParams) if api.IsRangeTooLarge(err) { diff --git a/cmd/metric_get_test.go b/cmd/metric_get_test.go index 98bbd1c..ecf58cb 100644 --- a/cmd/metric_get_test.go +++ b/cmd/metric_get_test.go @@ -238,3 +238,21 @@ func TestMetricGet_SplitOnBaseMetricIsIgnoredWithANote(t *testing.T) { t.Errorf("stderr=%q stdout=%q, want the note on stderr and a plain URL on stdout", stderr, stdout) } } + +// A bulk dry run prints one URL per window, the header note once, and no +// split summary, since nothing is fetched. +func TestMetricGet_BulkDryRun_NoteOnce(t *testing.T) { + stdout, stderr, err := runCLI(t, []string{"HOME=" + t.TempDir()}, + "metric", "get", "/market/price_usd_close/bulk", "--api-key", "k", "--asset", "BTC", + "--since", "2024-01-01", "--until", "2024-03-01", "--split", "--dry-run", + ) + if err != nil { + t.Fatalf("run CLI: %v\nstderr: %s", err, stderr) + } + if strings.Count(stdout, "/v1/metrics/market/price_usd_close/bulk?") != 2 { + t.Errorf("stdout should hold two window URLs:\n%s", stdout) + } + if strings.Count(stderr, "# credentials") != 1 || strings.Contains(stderr, "splitting") { + t.Errorf("stderr should carry the note once and no split summary:\n%s", stderr) + } +} diff --git a/internal/api/bulk_split.go b/internal/api/bulk_split.go index 96b4a7b..59603fa 100644 --- a/internal/api/bulk_split.go +++ b/internal/api/bulk_split.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "strconv" "strings" "time" @@ -61,6 +62,16 @@ func intervalStart(t time.Time, interval string) time.Time { } } +// knownInterval reports whether the API offers the resolution; the window +// helpers fall back to 24h, so an unknown one must be refused before planning. +func knownInterval(interval string) bool { + switch interval { + case "10m", "1h", "24h", "1w", "1month": + return true + } + return false +} + // BulkWindow returns the longest time range the API accepts in one bulk // request at the given resolution. func BulkWindow(interval string) time.Duration { @@ -93,6 +104,9 @@ func PlanBulk(params map[string]string, now time.Time) (BulkPlan, error) { if plan.Interval == "" { plan.Interval = defaultResolution } + if !knownInterval(plan.Interval) { + return plan, fmt.Errorf("unknown --interval %q; bulk metrics accept 10m, 1h, 24h, 1w or 1month", plan.Interval) + } plan.Window = BulkWindow(plan.Interval) raw, ok := params["s"] if !ok { @@ -184,7 +198,10 @@ func WindowParams(params map[string]string, w Window) map[string]string { // points. If the API rejects a window as too large, the window is halved for // the rest of the range, up to maxWindowShrinks times. Any other failure // aborts the whole fetch: a partial result would look complete to a script. -func (c *Client) GetMetricBulkSplit(ctx context.Context, path string, params map[string]string, repeated map[string][]string, plan BulkPlan) (*BulkResponse, error) { +func (c *Client) GetMetricBulkSplit(ctx context.Context, path string, params map[string]string, repeated map[string][]string, plan BulkPlan, progress io.Writer) (*BulkResponse, error) { + if progress == nil { + progress = io.Discard + } merged := &BulkResponse{} // Adjacent windows share no timestamp (the API rounds s down and includes // only intervals ending by u), so index is a guard: should a timestamp @@ -197,7 +214,7 @@ func (c *Client) GetMetricBulkSplit(ctx context.Context, path string, params map w := windows[0] // One line per window: each goes through the SDK's retries and may // wait on a rate limit, so a long split fetch should not fall silent. - fmt.Fprintf(stderr, "window %d/%d %s..%s\n", fetched+1, fetched+len(windows), formatBound(w.Since), formatBound(w.Until)) + fmt.Fprintf(progress, "window %d/%d %s..%s\n", fetched+1, fetched+len(windows), formatBound(w.Since), formatBound(w.Until)) resp, err := c.GetMetricBulk(ctx, path, WindowParams(params, w), repeated) if err != nil { // Halve the range the API actually rejected: near the end of the @@ -210,9 +227,14 @@ func (c *Client) GetMetricBulkSplit(ctx context.Context, path string, params map plan.Window = halved continue } + // Nothing has been printed, so the whole command is what to retry. // The total is an estimate: a halved window changes how many remain. - return nil, fmt.Errorf("window %d of about %d (%s..%s): %w; the fetch is aborted, retry it with --since %s", - fetched+1, fetched+len(windows), formatBound(w.Since), formatBound(w.Until), err, formatBound(w.Since)) + remedy := "retry the command" + if IsRangeTooLarge(err) { + remedy = "shorten the range or use a coarser --interval" + } + return nil, fmt.Errorf("window %d of about %d (%s..%s): %w; the fetch is aborted and no data was written (%d windows had succeeded), %s", + fetched+1, fetched+len(windows), formatBound(w.Since), formatBound(w.Until), err, fetched, remedy) } fetched++ for _, point := range resp.Data { diff --git a/internal/api/bulk_split_test.go b/internal/api/bulk_split_test.go index 53e148a..1740832 100644 --- a/internal/api/bulk_split_test.go +++ b/internal/api/bulk_split_test.go @@ -5,7 +5,6 @@ import ( "fmt" "net/http" "net/http/httptest" - "os" "strconv" "strings" "sync/atomic" @@ -30,6 +29,14 @@ func TestPlanBulk(t *testing.T) { t.Fatalf("err = %v", err) } }) + t.Run("an unknown interval is refused before planning", func(t *testing.T) { + for _, interval := range []string{"1d", "24H", "weekly"} { + _, err := PlanBulk(map[string]string{"s": unix(planSince), "i": interval}, planNow) + if err == nil || !strings.Contains(err.Error(), "unknown --interval") || !strings.Contains(err.Error(), "1month") { + t.Errorf("%q: err = %v", interval, err) + } + } + }) t.Run("defaults and window by resolution", func(t *testing.T) { for interval, want := range map[string]time.Duration{"": 31 * day, "24h": 31 * day, "10m": 10 * day, "1h": 10 * day, "1w": 93 * day, "1month": 93 * day} { params := map[string]string{"s": unix(planSince)} @@ -95,7 +102,7 @@ func TestPlanBulk(t *testing.T) { func TestGetMetricBulkSplitHalvesTheRejectedRange(t *testing.T) { plan := BulkPlan{Since: planSince, Until: planSince.Add(12 * day), Interval: "24h", Window: 31 * day} server, calls, windows := bulkServer(t, 10*day, 0) - resp, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan) + resp, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan, nil) if err != nil { t.Fatal(err) } @@ -173,9 +180,7 @@ func TestGetMetricBulkSplit(t *testing.T) { plan := BulkPlan{Since: planSince, Until: planSince.Add(70 * day), Interval: "24h", Window: 31 * day} server, calls, _ := bulkServer(t, 31*day, 0) var progress strings.Builder - stderr = &progress - t.Cleanup(func() { stderr = os.Stderr }) - resp, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan) + resp, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan, &progress) if err != nil { t.Fatal(err) } @@ -196,7 +201,7 @@ func TestGetMetricBulkSplitShrinksWindow(t *testing.T) { plan := BulkPlan{Since: planSince, Until: planSince.Add(31 * day), Interval: "24h", Window: 31 * day} // The server allows only 10 days: 31 -> 15.5 -> 7.75 days, two shrinks. server, calls, windows := bulkServer(t, 10*day, 0) - resp, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan) + resp, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan, nil) if err != nil { t.Fatal(err) } @@ -210,7 +215,7 @@ func TestGetMetricBulkSplitShrinksWindow(t *testing.T) { // A limit below what three halvings reach is an error. tiny, _, _ := bulkServer(t, 2*day, 0) - if _, err := newTestClient(tiny).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan); err == nil || !strings.Contains(err.Error(), "too large time range") { + if _, err := newTestClient(tiny).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan, nil); err == nil || !strings.Contains(err.Error(), "too large time range") { t.Fatalf("err = %v", err) } } @@ -218,11 +223,11 @@ func TestGetMetricBulkSplitShrinksWindow(t *testing.T) { func TestGetMetricBulkSplitAbortsOnFailure(t *testing.T) { plan := BulkPlan{Since: planSince, Until: planSince.Add(70 * day), Interval: "24h", Window: 31 * day} server, calls, _ := bulkServer(t, 31*day, 2) - resp, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan) + resp, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan, nil) if err == nil || resp != nil { t.Fatalf("expected an error and no data, got %v %v", resp, err) } - for _, want := range []string{"window 2 of about 3", "2024-02-01..2024-03-03", "metric not in your plan", "--since 2024-02-01"} { + for _, want := range []string{"window 2 of about 3", "2024-02-01..2024-03-03", "metric not in your plan", "no data was written (1 windows had succeeded), retry the command"} { if !strings.Contains(err.Error(), want) { t.Errorf("error %q lacks %q", err, want) } @@ -256,7 +261,7 @@ func TestGetMetricBulkSplitKeepsLaterPointOnOverlap(t *testing.T) { fmt.Fprintf(w, `{"data":[{"t":%d,"bulk":[{"a":"BTC","v":2},{"a":"ETH","v":3}]}]}`, boundary) })) t.Cleanup(server.Close) - resp, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan) + resp, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan, nil) if err != nil { t.Fatal(err) } @@ -290,11 +295,11 @@ func TestGetMetricBulkSplitErrorAfterHalving(t *testing.T) { since := planSince.Add(6 * time.Hour) plan := BulkPlan{Since: since, Until: since.Add(20 * day), Interval: "1h", Window: 10 * day} server, _, _ := bulkServer(t, 5*day, 4) // call 1 too large; calls 2-3 succeed; call 4 fails - _, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan) + _, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan, nil) if err == nil { t.Fatal("expected an error") } - for _, want := range []string{"window 3 of about 4", "2024-01-11T06:00:00Z..2024-01-16T06:00:00Z", "--since 2024-01-11T06:00:00Z"} { + for _, want := range []string{"window 3 of about 4", "2024-01-11T06:00:00Z..2024-01-16T06:00:00Z", "retry the command"} { if !strings.Contains(err.Error(), want) { t.Errorf("error %q lacks %q", err, want) } @@ -315,9 +320,9 @@ func TestFormatBound(t *testing.T) { func TestGetMetricBulkSplitDoesNotHalveBelowAnInterval(t *testing.T) { plan := BulkPlan{Since: planSince, Until: planSince.Add(93 * day), Interval: "1month", Window: 93 * day} server, calls, windows := bulkServer(t, 20*day, 0) // even one month is rejected - _, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan) - if err == nil || !strings.Contains(err.Error(), "time range") { - t.Fatalf("err = %v, want the API's too-large error once halving is exhausted", err) + _, err := newTestClient(server).GetMetricBulkSplit(context.Background(), "/market/price", map[string]string{"s": unix(plan.Since)}, nil, plan, nil) + if err == nil || !strings.Contains(err.Error(), "time range") || !strings.Contains(err.Error(), "coarser --interval") || strings.Contains(err.Error(), "retry") { + t.Fatalf("err = %v, want the API's too-large error and a remedy other than retrying", err) } if calls.Load() != 2 || !(*windows)[1].Until.Equal(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC)) { t.Errorf("calls=%d windows=%v, want 93d then January and no shorter window", calls.Load(), *windows) diff --git a/internal/api/client.go b/internal/api/client.go index dc3795b..c616b15 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -56,14 +56,15 @@ func (c *Client) sdk() (*glassnode.Client, error) { return c.sdkClient, c.initErr } -// AuthHeader names the header that carries the credential. The SDK sends the -// API key in X-Api-Key rather than in the URL, so it stays out of proxy and -// access logs; dry-run output therefore shows the URL without it. +// AuthHeader describes the header that carries the credential, for the +// dry-run note. The SDK sends the API key in X-Api-Key rather than in the +// URL, so it stays out of proxy and access logs; dry-run output therefore +// shows the URL without it. func (c *Client) AuthHeader() string { if c.bearerToken != "" { - return "Authorization: Bearer" + return "Authorization header (Bearer scheme)" } - return "X-Api-Key" + return "X-Api-Key header" } // sessionTokens hands the CLI's OAuth session token to the SDK and refreshes From 942ac9155b6d617a574c3cff9a12ff60e5d8ceb3 Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Tue, 13 Oct 2026 11:53:52 +0200 Subject: [PATCH 36/37] GN-159: lay out bulk windows once, relayout only after a halving GetMetricBulkSplit rebuilt the whole window list on every iteration to read its head; the list is built once and again only when a halving changes the remaining windows. RangeError and Describe take the count the command already holds instead of laying the windows out again. --- cmd/metric_get.go | 4 ++-- internal/api/bulk_split.go | 22 ++++++++++++---------- internal/api/bulk_split_test.go | 6 +++--- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/cmd/metric_get.go b/cmd/metric_get.go index 14cb007..4c0dea2 100644 --- a/cmd/metric_get.go +++ b/cmd/metric_get.go @@ -83,7 +83,7 @@ var metricGetCmd = &cobra.Command{ split, _ := cmd.Flags().GetBool("split") windows := plan.Windows() if len(windows) > 1 && !split { - return plan.RangeError() + return plan.RangeError(len(windows)) } dryRun, _ := cmd.Flags().GetBool("dry-run") if dryRun { @@ -101,7 +101,7 @@ var metricGetCmd = &cobra.Command{ return nil } if len(windows) > 1 { - fmt.Fprintln(cmd.ErrOrStderr(), plan.Describe()) + fmt.Fprintln(cmd.ErrOrStderr(), plan.Describe(len(windows))) } // --split is the only consent to more than one request: without it diff --git a/internal/api/bulk_split.go b/internal/api/bulk_split.go index 59603fa..7c3d6b9 100644 --- a/internal/api/bulk_split.go +++ b/internal/api/bulk_split.go @@ -171,16 +171,16 @@ func (plan BulkPlan) Windows() []Window { // WindowDays is the window's length as shown to the user, such as "31 days". func (plan BulkPlan) WindowDays() string { return days(plan.Window) } -// RangeError explains why the range needs --split. -func (plan BulkPlan) RangeError() error { +// RangeError explains why the range needs --split; requests is len(Windows()). +func (plan BulkPlan) RangeError(requests int) error { return fmt.Errorf("range %s..%s exceeds the API limit of %s per bulk request at %s; shorten it or add --split to fetch it in %d requests", - formatBound(plan.Since), formatBound(plan.Until), plan.WindowDays(), plan.Interval, len(plan.Windows())) + formatBound(plan.Since), formatBound(plan.Until), plan.WindowDays(), plan.Interval, requests) } -// Describe summarises the split for a warning to the user. -func (plan BulkPlan) Describe() string { +// Describe summarises the split for a warning to the user; requests is len(Windows()). +func (plan BulkPlan) Describe(requests int) string { return fmt.Sprintf("splitting %s..%s into %d requests of up to %s (%s); the result is assembled in memory", - formatBound(plan.Since), formatBound(plan.Until), len(plan.Windows()), plan.WindowDays(), plan.Interval) + formatBound(plan.Since), formatBound(plan.Until), requests, plan.WindowDays(), plan.Interval) } // WindowParams returns params with s and u set to the window's bounds. @@ -208,9 +208,10 @@ func (c *Client) GetMetricBulkSplit(ctx context.Context, path string, params map // repeat, the later response, which is at least as fresh, replaces it. index := map[int64]int{} shrinks, fetched := 0, 0 - since := plan.Since - for since.Before(plan.Until) { - windows := BulkPlan{Since: since, Until: plan.Until, Interval: plan.Interval, Window: plan.Window}.Windows() + // The windows are laid out once and again only after a halving, which + // changes every window that remains. + windows := plan.Windows() + for len(windows) > 0 { w := windows[0] // One line per window: each goes through the SDK's retries and may // wait on a rate limit, so a long split fetch should not fall silent. @@ -225,6 +226,7 @@ func (c *Client) GetMetricBulkSplit(ctx context.Context, path string, params map if halved := w.Until.Sub(w.Since) / 2; IsRangeTooLarge(err) && shrinks < maxWindowShrinks && halved >= intervalDuration(plan.Interval) { shrinks++ plan.Window = halved + windows = BulkPlan{Since: w.Since, Until: plan.Until, Interval: plan.Interval, Window: plan.Window}.Windows() continue } // Nothing has been printed, so the whole command is what to retry. @@ -245,7 +247,7 @@ func (c *Client) GetMetricBulkSplit(ctx context.Context, path string, params map index[point.T] = len(merged.Data) merged.Data = append(merged.Data, point) } - since = w.Until + windows = windows[1:] } if merged.Data == nil { merged.Data = []BulkDataPoint{} diff --git a/internal/api/bulk_split_test.go b/internal/api/bulk_split_test.go index 1740832..e14240a 100644 --- a/internal/api/bulk_split_test.go +++ b/internal/api/bulk_split_test.go @@ -128,8 +128,8 @@ func TestBulkPlanWindows(t *testing.T) { if !windows[0].Since.Equal(planSince) || !windows[2].Until.Equal(plan.Until) || windows[2].Until.Sub(windows[2].Since) != 8*day { t.Errorf("bounds: %v", windows) } - if !strings.Contains(plan.RangeError().Error(), "3 requests") || !strings.Contains(plan.Describe(), "3 requests of up to 31 days (24h)") { - t.Errorf("messages: %v / %s", plan.RangeError(), plan.Describe()) + if !strings.Contains(plan.RangeError(len(plan.Windows())).Error(), "3 requests") || !strings.Contains(plan.Describe(len(plan.Windows())), "3 requests of up to 31 days (24h)") { + t.Errorf("messages: %v / %s", plan.RangeError(len(plan.Windows())), plan.Describe(len(plan.Windows()))) } if one := (BulkPlan{Since: planSince, Until: planSince.Add(day), Window: 31 * day}).Windows(); len(one) != 1 { t.Errorf("one-day range should be one window, got %d", len(one)) @@ -366,7 +366,7 @@ func TestBulkPlanWindowsAlignToIntervals(t *testing.T) { func TestRangeMessagesKeepIntradayBounds(t *testing.T) { since := planSince.Add(6 * time.Hour) plan := BulkPlan{Since: since, Until: since.Add(19 * day), Interval: "1h", Window: 10 * day} - for _, msg := range []string{plan.RangeError().Error(), plan.Describe()} { + for _, msg := range []string{plan.RangeError(len(plan.Windows())).Error(), plan.Describe(len(plan.Windows()))} { if !strings.Contains(msg, "2024-01-01T06:00:00Z..2024-01-20T06:00:00Z") { t.Errorf("message drops the time of day: %s", msg) } From df2147d71c9d7663ced801803921aa5dcc32576a Mon Sep 17 00:00:00 2001 From: Aleh Biruk Date: Tue, 13 Oct 2026 12:08:38 +0200 Subject: [PATCH 37/37] GN-159: drop docs/go-sdk-extraction.md; the PR description and README cover it --- docs/go-sdk-extraction.md | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 docs/go-sdk-extraction.md diff --git a/docs/go-sdk-extraction.md b/docs/go-sdk-extraction.md deleted file mode 100644 index 2c54854..0000000 --- a/docs/go-sdk-extraction.md +++ /dev/null @@ -1,26 +0,0 @@ -# Go SDK extraction - -The reusable API implementation lives in the -[`glassnode-api-go-client`](https://github.com/glassnode/glassnode-api-go-client) -module; its design notes are in that repository's `docs/design.md`. - -`internal/api` is a CLI adapter. It retains environment/config resolution, -OAuth login and refresh, dry-run URL rendering, asset pruning, credit -presentation and the historical output shapes. Endpoint HTTP handling, -retries and decoding are provided by the SDK. - -- The API key is sent in the `X-Api-Key` header, as the SDK does by default, so - it stays out of URLs and access logs; `--dry-run` prints the URL without it - and notes the header on stderr. -- OAuth sessions reach the SDK through its `TokenRefresher` interface: the CLI - supplies the current token and refreshes it once when the API answers 401. -- SDK clients are initialized lazily once per CLI adapter and reused. -- GET calls retry transport failures, 429 and 5xx responses with jitter and - honour the API's rate-limit reset. Context cancellation interrupts calls and - backoff. Errors expose SDK types and redact credentials. Redirects are not - followed. -- Bulk metrics require `--since`; ranges over the API's per-request limit are - fetched in windows with `--split`. - -For local SDK development, use a temporary `go.work` outside both repositories -with both checkouts; never commit a filesystem `replace` into the CLI module.