diff --git a/CHANGELOG.md b/CHANGELOG.md index efe52da27..acbd830f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **`-align-queries-with-step` puts metric range queries on Loki's evaluation + grid.** Loki's step-align middleware truncates a range query's `start` and + `end` down to multiples of `step`, so on 3.7.1 with `step=137s` every returned + timestamp satisfies `ts % 137 == 0` and the first point sits before the + requested start. The proxy started its grid at the request's own `start`, + which shifted the whole series by `start mod step` — invisible whenever the + step happened to divide the start, which is why it only shows on odd steps. + Off by default, matching Loki's binary default; the Loki Helm chart turns the + option on, so set the flag to `true` to match a chart-deployed Loki. Log + queries are never moved: they have no evaluation grid, and shifting their + bounds would change which lines they return. Requests that omit `step`, and + `/loki/api/v1/patterns`, are left unaligned: Loki aligns both to a default + step of `range/250` that this proxy does not use downstream, so aligning to + it here would move the bounds off the grid the response is actually built on. + A request without `start` or `end` is aligned on the bounds Loki derives for + it (`end=now`, `start=end-since`). Binary expressions joining two vector + operands keep aligning both bounds whatever the flag says, as before: with + the flag off the join would otherwise have no common timestamps whenever one + operand is served from tumbling VictoriaLogs buckets. The e2e stack runs + every proxy variant with the flag on, matching its Loki. ### Fixed +- **`max_query_length` is checked on the range the client asked for**, before + step alignment, as Loki's limits middleware runs before its step-align + middleware. Alignment can grow the range by up to `step-1ns`, so a query + exactly at the limit was rejected where Loki accepts it. +- **The 11,000-point resolution check reads the step as the evaluators do.** It + parsed the step with a duration-only parser, so the raw nanosecond integer + Grafana sometimes sends (a proxy extension; Loki reads a bare integer as + seconds) was exempt from the limit while being evaluated, and now aligned, + on that step. - **A release is validated, tagged and published from the commit its version was computed from.** Since releases queue instead of cancelling each other, the first release job re-reads `main` before computing the next version, but diff --git a/charts/loki-vl-proxy/values.yaml b/charts/loki-vl-proxy/values.yaml index 9ef2f6bf0..e181f80d0 100644 --- a/charts/loki-vl-proxy/values.yaml +++ b/charts/loki-vl-proxy/values.yaml @@ -109,6 +109,7 @@ extraArgs: # query-range-partial-responses: "false" # return partial response on retryable backend failures # query-range-background-warm: "true" # continue warming failed windows in background # query-range-background-warm-max-windows: "24" # cap background warm fanout + # align-queries-with-step: "false" # truncate metric range bounds down to the step grid, as Loki does under query_range.align_queries_with_step (off by default like Loki's binary; the Loki Helm chart turns it on) # default-max-query-length: "0" # max query time range for all tenants (0 = unlimited; per-tenant limits override) # max-stats-query-series: "500" # max series a metric query may return (count_over_time, rate, bytes_rate); 0 = built-in default of 500, Loki's own max_query_series default; above it the query fails with Loki's HTTP 400 "maximum number of series (N) reached for a single query" (Logs Drilldown gets a partial result with a warning) # stats-query-range-concurrency: "4" # max concurrent stats_query_range calls to VL; higher = more VL CPU pressure; 0 = built-in default of 4 diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 5b0de0e2b..9a6c4227c 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -204,6 +204,7 @@ type proxyRuntimeConfig struct { coldBackendManifestRefresh time.Duration coldBackendTimeout time.Duration defaultMaxQueryLength time.Duration + alignQueriesWithStep bool maxStatsQuerySeries int statsQueryRangeConcurrency int backendMaxConcurrentHeavyQueries int @@ -627,6 +628,7 @@ func run( patternsPersistInterval := fs.Duration("patterns-persist-interval", 30*time.Second, "How often to persist in-memory patterns snapshots to disk") patternsStartupStale := fs.Duration("patterns-startup-stale-threshold", 60*time.Second, "Treat on-disk patterns snapshot older than this as stale and warm from peers before serving") patternsPeerWarmTimeout := fs.Duration("patterns-startup-peer-warm-timeout", 5*time.Second, "Maximum time to wait for startup patterns snapshot warm from peers") + alignQueriesWithStep := fs.Bool("align-queries-with-step", false, "Truncate a metric range query's start and end down to multiples of the step before evaluation, the way Loki's step-align middleware does under query_range.align_queries_with_step. Applies to /loki/api/v1/query_range with an explicit step. Off by default, matching Loki's binary default; the Loki Helm chart turns it on, so set this to true to match a chart-deployed Loki.") allowGlobalTenant := fs.Bool("tenant.allow-global", false, `Allow X-Scope-OrgID "*" to bypass AccountID/ProjectID scoping and use the backend default tenant`) // Cold storage backend (Victoria Lakehouse) @@ -916,6 +918,7 @@ func run( coldBackendManifestRefresh: *coldBackendManifestRefresh, coldBackendTimeout: *coldBackendTimeout, defaultMaxQueryLength: *defaultMaxQueryLength, + alignQueriesWithStep: *alignQueriesWithStep, maxStatsQuerySeries: *maxStatsQuerySeries, statsQueryRangeConcurrency: *statsQueryRangeConcurrency, backendMaxConcurrentHeavyQueries: *backendMaxConcurrentHeavyQueries, @@ -2094,6 +2097,7 @@ func buildProxyConfig(cfg proxyRuntimeConfig) (proxy.Config, error) { Timeout: cfg.coldBackendTimeout, }, DefaultMaxQueryLength: cfg.defaultMaxQueryLength, + AlignQueriesWithStep: cfg.alignQueriesWithStep, MaxStatsQuerySeries: cfg.maxStatsQuerySeries, StatsQueryRangeConcurrency: cfg.statsQueryRangeConcurrency, BackendMaxConcurrentHeavyQueries: cfg.backendMaxConcurrentHeavyQueries, diff --git a/conformance/registry/generated/wiring.json b/conformance/registry/generated/wiring.json index 5c7b47e83..d4774e804 100644 --- a/conformance/registry/generated/wiring.json +++ b/conformance/registry/generated/wiring.json @@ -85,7 +85,8 @@ "internal/proxy/tumbling_axis_parity_test.go", "internal/translator/regexp_escaping_test.go", "test/e2e-compat/heavy_query_protection_test.go", - "test/e2e-compat/range_metric_offset_series_limit_test.go" + "test/e2e-compat/range_metric_offset_series_limit_test.go", + "test/e2e-compat/step_grid_parity_test.go" ], "operator-configurable-limits": [ "cmd/proxy/main_test.go", @@ -244,9 +245,12 @@ "semantics/offset-sample-timestamps", "series-limits-and-partial-results", "window-bounds-and-step-alignment" + ], + "test/e2e-compat/step_grid_parity_test.go": [ + "loki_api_v1_query_range" ] }, - "tests_with_markers": 17, + "tests_with_markers": 18, "unknown_ids": {}, "unwired_items": [ "api_prom_label", diff --git a/docs/compatibility-loki.md b/docs/compatibility-loki.md index c2a0724d7..5c9c8f33a 100644 --- a/docs/compatibility-loki.md +++ b/docs/compatibility-loki.md @@ -204,6 +204,13 @@ Some important compatibility behavior is still tested, but it is not part of the Those cases live in the proxy contract suite because Loki itself is not the source of truth for them. +Documented deviations from Loki's step alignment (`-align-queries-with-step`): + +- A metric range request without `step` is not aligned. Loki derives `max(floor(range/250), 1s)` at decode time and aligns to it; the proxy's range evaluator falls back to `1m`, so aligning to Loki's derived step would move the bounds off the grid the response is built on. +- `/loki/api/v1/patterns` is not aligned. Loki runs it through the same middleware, but its default step is `range/250` against the proxy's `range/120`. +- A binary expression joining two vector operands is aligned to the step grid whether or not the flag is set. Operands served from tumbling VictoriaLogs buckets sit on the epoch-aligned grid, so without alignment a join with an unaligned `start` has no common timestamps; against a default (unaligned) Loki the join's grid is shifted by less than one step. +- A bare integer `step` too large to be seconds is read as nanoseconds, a proxy extension for the value Grafana sometimes sends; Loki reads every bare integer as seconds. The alignment, the 11,000-point check and the evaluators all read it the same way. + ## Parity Rule Valid Loki behavior is not an allowed exclusion category. diff --git a/docs/configuration.md b/docs/configuration.md index 0d88d4ba1..937578b4c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -458,6 +458,7 @@ These flags control Loki-compatible `query_range` split/merge execution with per | Flag | Env | Default | Description | |---|---|---|---| +| `-align-queries-with-step` | — | `false` | Truncate a metric range query's `start` and `end` down to multiples of `step` before evaluation, the way Loki's step-align middleware does under `query_range.align_queries_with_step`. Off matches Loki's binary default; the Loki Helm chart turns the option on, so set this to `true` to match a chart-deployed Loki. Log queries are never moved | | `-drilldown-scan-timeout` | — | `5s` | Per-request timeout for the `detected_fields` / `detected_field/{name}/values` log-scan path. `0` disables the cap | | `-max-stats-query-series` | — | `0` (uses `500`) | Maximum series a metric query may return (`count_over_time`, `rate`, `bytes_rate`, ...), the proxy's equivalent of Loki's `max_query_series`; `0` uses the built-in default of `500`, which is also Loki's default. Above it the query is rejected with HTTP `400` and Loki's message `maximum number of series (N) reached for a single query`, on range and instant queries alike; the log line for the rejected request names this flag. Grafana Logs Drilldown requests receive a partial result (the busiest series) with Loki's `... returning partial results` warning instead, as Loki does for that client (see [Fixed Execution Limits](#fixed-execution-limits)) | | `-stats-query-range-concurrency` | — | `0` (uses `4`) | Maximum concurrent `stats_query_range` calls to VictoriaLogs. `0` uses the built-in default of `4` | diff --git a/docs/real-window-compatibility-gaps.md b/docs/real-window-compatibility-gaps.md index 82de83a98..e523cecd7 100644 --- a/docs/real-window-compatibility-gaps.md +++ b/docs/real-window-compatibility-gaps.md @@ -30,9 +30,10 @@ The exhaustive checks compare status, result type and non-emptiness. Strict quan - Tumbling range aggregations (range equal to step) served from VictoriaLogs `stats_query_range` buckets are relabelled to Loki's evaluation timestamps. VictoriaLogs labels a bucket by its start and covers `[T, T+step)`; Loki's sample at `T` covers `(T-range, T]`. The proxy fetches from `start-range` and moves each bucket forward by one range, instead of trimming the bucket Loki's first sample needs. It covers `rate` and `bytes_rate` in any aggregation, and ungrouped `sum(count_over_time(...))` and `sum(bytes_over_time(...))`. - Sliding range aggregations (`count_over_time`, `rate`, `bytes_over_time`, `bytes_rate` with a range different from the step) served from `stats_query_range` buckets sum buckets of `gcd(step, range)` anchored to the request start and shifted by one nanosecond through the `offset` argument, so each bucket covers `(T, T+bucket]` and every window `(t-range, t]` is an exact union of buckets, including lines on window edges and unaligned starts. A step whose window holds no line is absent, as in Loki, for every client including Drilldown. Buckets below 1 ms, stats responses over the byte limit, and unaligned grids on a backend older than VictoriaLogs v1.45 or of undetected version use the raw-sample evaluator, which applies the same `(t-range, t]` boundaries to these functions for range and instant queries. - Bare parser range aggregations without an outer aggregation (`count_over_time({...} | logfmt [90s])`, also `regexp`, `pattern` and `json` with parameters) use the same anchored `gcd(step, range)` buckets: `stats_query_range` grouped by stream for `count_over_time`, `rate`, `bytes_over_time` and `bytes_rate` with a range greater than or equal to the step (byte sums add a `count()` presence column, so a window holding only empty lines is a zero sample), VictoriaLogs `/hits` with the same `offset` argument for sliding `count_over_time` and `rate` when stream label fields are declared, and bucket values for `sum_over_time`, `max_over_time` and `min_over_time` over `unwrap`. Tumbling queries on this path are no longer merged into one unlabelled series. Unaligned grids on a backend without `offset` support use the raw evaluator. Bare `| json` without parameters keeps the ordered JSON evaluator. -- Binary expressions joining two vector operands evaluate both on the step-aligned grid, as Loki's query frontend does when `align_queries_with_step` is enabled. The compatibility stack enables it; a default Loki evaluates at `start+k*step`, so results with an unaligned start differ by less than one step. Operands with different ranges take different execution paths (sliding evaluation versus tumbling buckets), and an unaligned start previously left no common timestamps to join. +- Binary expressions joining two vector operands evaluate both on the step-aligned grid, as Loki's query frontend does when `align_queries_with_step` is enabled, whether or not `-align-queries-with-step` is set on the proxy. The compatibility stack enables both; a default Loki evaluates at `start+k*step`, so a binary join with an unaligned start differs from it by less than one step. Operands with different ranges take different execution paths (sliding evaluation versus tumbling buckets), and an unaligned start previously left no common timestamps to join. - `unwrap duration(...)` and `unwrap bytes(...)` over bare parsers convert unit strings such as `15ms` and `1024B` instead of dropping every sample. - Range operands grouped `by (level)` keep Loki's `level` key, so `on(level)` matching works for range queries as it already did for instant queries. +- With `-align-queries-with-step`, every metric range query (not only binary joins) evaluates on Loki's `k*step` grid: both bounds truncate down to a multiple of the step, as Loki's step-align middleware does. Measured on the stack at `step=137` with an unaligned start, Loki and the proxy return the same 22 timestamps where an unaligned proxy returns 21 starting 26s late. - Scalar results carry second timestamps and all sample values use Loki's fixed-point rendering (`1234000`, not `1.234e+06`). These corrections reject previously accepted invalid queries and change incorrect numeric results. They do not imply unchanged behavior for all clients. IP validation is eager: Loki can bypass invalid pipeline construction for historical empty ranges, while the proxy rejects the invalid expression. diff --git a/docs/reference/configuration-reference.md b/docs/reference/configuration-reference.md index 3e70a348c..6c8b0702e 100644 --- a/docs/reference/configuration-reference.md +++ b/docs/reference/configuration-reference.md @@ -2,7 +2,7 @@ # Configuration reference -Every command-line flag of the proxy (208), grouped by category, with the Helm value that sets it. Narrative guidance lives in [configuration.md](../configuration.md); the bounds on work are collected in [limits-registry.md](limits-registry.md). +Every command-line flag of the proxy (209), grouped by category, with the Helm value that sets it. Narrative guidance lives in [configuration.md](../configuration.md); the bounds on work are collected in [limits-registry.md](limits-registry.md). Helm passes any flag through `extraArgs.`; the chart sets a few of them from dedicated values, marked chart-managed. @@ -71,6 +71,7 @@ Helm passes any flag through `extraArgs.`; the chart sets a few of them fr | Flag | Type | Default | Helm value | Description | |---|---|---|---|---| +| `-align-queries-with-step` | bool | `false` | `extraArgs.align-queries-with-step` | Truncate a metric range query's start and end down to multiples of the step before evaluation, the way Loki's step-align middleware does under query_range.align_queries_with_step. Applies to /loki/api/v1/query_range with an explicit step. Off by default, matching Loki's binary default; the Loki Helm chart turns it on, so set this to true to match a chart-deployed Loki. | | `-backend-allow-unsupported-version` | bool | `false` | `extraArgs.backend-allow-unsupported-version` | Allow startup with backend versions lower than -backend-min-version (at your own risk). Ignored when --backend-version-strict=true. | | `-backend-min-version` | string | `"v1.30.0"` | `extraArgs.backend-min-version` | Minimum VictoriaLogs version considered fully supported at startup | | `-backend-version-strict` | bool | `false` | `extraArgs.backend-version-strict` | When true, /health failure, non-2xx response, or missing/sub-min backend semver causes startup to fail. Default false (warn only). Overrides --backend-allow-unsupported-version when both are set. | diff --git a/docs/translation-reference.md b/docs/translation-reference.md index e04dae062..4e88099e1 100644 --- a/docs/translation-reference.md +++ b/docs/translation-reference.md @@ -136,7 +136,7 @@ Supported operators: `+`, `-`, `*`, `/`, `%`, `^`, `==`, `!=`, `>`, `<`, `>=`, ` Binary expression notes: - Each operand is executed through the normal query handlers, so trailing range windows, parser errors and extraction aliases are preserved. Operator precedence, parentheses and comparison filtering versus `bool` follow Loki. -- Vector-vector operands are evaluated on the step-aligned grid, as Loki's query frontend does with `align_queries_with_step` enabled. Against a default Loki, results for an unaligned `start` can differ by less than one step. +- Vector-vector operands are evaluated on the step-aligned grid, as Loki's query frontend does with `align_queries_with_step` enabled, whatever `-align-queries-with-step` says. Against a default Loki, results for an unaligned `start` can differ by less than one step. With `-align-queries-with-step` every metric range query is aligned the same way. - Implicit many-to-one matches (without `group_left`/`group_right`) are rejected with HTTP 500 and Loki's `multiple matches for labels` error; cardinality is checked at each timestamp. - Evaluation is bounded: 64 nesting levels, 1,024 child evaluations, 256 MiB of captured child responses, two million decoded arrays, one million output samples and 64 MiB of label work, with each encoded result capped at 64 MiB. A valid expression exceeding these limits fails explicitly. diff --git a/internal/proxy/binary_logql.go b/internal/proxy/binary_logql.go index 3f84401d0..2e24fcd91 100644 --- a/internal/proxy/binary_logql.go +++ b/internal/proxy/binary_logql.go @@ -290,18 +290,23 @@ func (p *Proxy) proxyBinaryLogQL(w http.ResponseWriter, r *http.Request, expr *l // the operands are evaluated. Operands with different range windows take // different execution paths: sliding windows are evaluated at start+k*step, // tumbling windows come from VictoriaLogs buckets on the epoch-aligned grid. -// Sharing one aligned axis lets the join match samples, and mirrors Loki's -// query frontend with align_queries_with_step. Requests without a parseable -// start, end or step are returned unchanged. +// Sharing one aligned axis lets the join match samples at all. +// +// This runs regardless of -align-queries-with-step, and that is a recorded +// deviation rather than an oversight: with the flag off the join would +// otherwise have no common timestamps whenever one operand is tumbling and the +// start is unaligned, and an empty join is worse than one shifted by less than +// a step against a default Loki. With the flag on the request arrives already +// aligned and this is a no-op. Requests without a parseable start, end or step +// are returned unchanged. func alignBinaryRangeRequest(r *http.Request) *http.Request { start, startOK := parseLokiTimeToUnixNano(r.FormValue("start")) end, endOK := parseLokiTimeToUnixNano(r.FormValue("end")) - step, stepOK := parsePositiveStepDuration(r.FormValue("step")) + stepNs, stepOK := parseStepToNanos(r.FormValue("step")) if !startOK || !endOK || !stepOK || end < start { return r } - stepNs := int64(step) - alignedStart, alignedEnd := start-start%stepNs, end-end%stepNs + alignedStart, alignedEnd := alignDownToStep(start, stepNs), alignDownToStep(end, stepNs) if alignedStart == start && alignedEnd == end { return r } diff --git a/internal/proxy/config.go b/internal/proxy/config.go index f5e70ed7b..2f3e04097 100644 --- a/internal/proxy/config.go +++ b/internal/proxy/config.go @@ -49,6 +49,7 @@ type HandlerConfig struct { tenantDefaultLimits map[string]any // snapshot at construction — not hot-reloadable via HandlerConfig tenantLimits map[string]map[string]any // snapshot at construction — not hot-reloadable via HandlerConfig defaultMaxQueryLength time.Duration + alignQueriesWithStep bool queryRangeWindowing bool queryRangeSplitInterval time.Duration queryRangeMaxParallel int diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 72b6a602a..e35c35892 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -281,6 +281,10 @@ type Config struct { PatternsStartupStale time.Duration // PatternsPeerWarmTimeout bounds startup peer warm attempts when disk is stale/missing. PatternsPeerWarmTimeout time.Duration + // AlignQueriesWithStep truncates a metric range query's start and end to + // multiples of the step before evaluation, as Loki's step-align middleware + // does under query_range.align_queries_with_step. + AlignQueriesWithStep bool // Peer cache (fleet distribution) PeerCache *cache.PeerCache // optional peer cache for distributed fleet @@ -528,6 +532,7 @@ type Proxy struct { tenantLimits map[string]map[string]any defaultMaxQueryLength time.Duration // 0 = unlimited translationCache *cache.Cache + alignQueriesWithStep bool // Loki's query_range.align_queries_with_step queryRangeWindowing bool queryRangeSplitInterval time.Duration queryRangeMaxParallel int @@ -1153,6 +1158,7 @@ func New(cfg Config) (*Proxy, error) { defaultMaxQueryLength: cfg.DefaultMaxQueryLength, translationCache: cache.New(5*time.Minute, 5000), streamFieldNamesCache: cache.New(30*time.Second, 500), + alignQueriesWithStep: cfg.AlignQueriesWithStep, queryRangeWindowing: cfg.QueryRangeWindowingEnabled && cfg.QueryRangeSplitInterval > 0, queryRangeSplitInterval: cfg.QueryRangeSplitInterval, queryRangeMaxParallel: queryRangeMaxParallel, @@ -1292,6 +1298,7 @@ func New(cfg Config) (*Proxy, error) { tenantDefaultLimits: p.tenantDefaultLimits, tenantLimits: p.tenantLimits, defaultMaxQueryLength: p.defaultMaxQueryLength, + alignQueriesWithStep: p.alignQueriesWithStep, queryRangeWindowing: p.queryRangeWindowing, queryRangeSplitInterval: p.queryRangeSplitInterval, queryRangeMaxParallel: p.queryRangeMaxParallel, @@ -2012,41 +2019,19 @@ const ( // range parameters for exceeding lokiMaxPointsPerSeries. It applies Loki's // defaults (end=now, start=min(end, now)-since, since=1h). A missing step uses // Loki's range/250 default, which never exceeds the limit; unparseable or -// inverted parameters are left to the handlers that already report them. +// inverted parameters are left to the handlers that already report them. The +// step is read with the same parser as the range evaluators and the step +// alignment, so all three agree on what one step is. func exceedsLokiRangeResolution(r *http.Request, now time.Time) bool { - step, ok := parsePositiveStepDuration(r.FormValue("step")) + stepNs, ok := parseStepToNanos(r.FormValue("step")) if !ok { return false } - end := now - if raw := strings.TrimSpace(r.FormValue("end")); raw != "" { - ns, ok := parseLokiTimeToUnixNano(raw) - if !ok { - return false - } - end = time.Unix(0, ns) - } - var start time.Time - if raw := strings.TrimSpace(r.FormValue("start")); raw != "" { - ns, ok := parseLokiTimeToUnixNano(raw) - if !ok { - return false - } - start = time.Unix(0, ns) - } else { - since := time.Hour - if raw := strings.TrimSpace(r.FormValue("since")); raw != "" { - if since, ok = parsePositiveStepDuration(raw); !ok { - return false - } - } - endOrNow := end - if end.After(now) { - endOrNow = now - } - start = endOrNow.Add(-since) + start, end, ok := lokiRangeBounds(r, now) + if !ok { + return false } - return end.After(start) && end.Sub(start)/step > lokiMaxPointsPerSeries + return end.After(start) && end.Sub(start)/time.Duration(stepNs) > lokiMaxPointsPerSeries } // Loki: GET /loki/api/v1/query_range?query={...}&start=...&end=...&limit=...&step=... @@ -2095,6 +2080,28 @@ func (p *Proxy) handleQueryRange(w http.ResponseWriter, r *http.Request) { } } + // max_query_length is checked on the range the client asked for, as Loki's + // limits middleware runs before its step alignment. Alignment below can + // grow the range by up to step-1ns, and an offset shifts both bounds + // equally, so neither changes the length being checked. + if startNs, okS := parseLokiTimeToUnixNano(r.FormValue("start")); okS { + if endNs, okE := parseLokiTimeToUnixNano(r.FormValue("end")); okE { + if errMsg := p.checkQueryRangeLength(r.Context(), startNs, endNs); errMsg != "" { + p.writeError(w, http.StatusBadRequest, errMsg) + p.metrics.RecordRequest("query_range", http.StatusBadRequest, time.Since(start)) + return + } + } + } + + // Loki evaluates a metric range query on the grid of `k * step`, not on the + // client's `start`: its step-align middleware truncates BOTH bounds down to + // a multiple of the step. Align here, at the entry, so every downstream path + // builds its request and its grid from Loki's bounds. + if p.alignQueriesWithStep { + alignRangeRequestToStepGrid(r, logqlQuery, start) + } + categorizedLabels := requestWantsCategorizedLabels(r) emitStructuredMetadata := p.shouldEmitStructuredMetadata(r) tupleMode := tupleModeForRequest(categorizedLabels, emitStructuredMetadata) @@ -2187,17 +2194,6 @@ func (p *Proxy) handleQueryRange(w http.ResponseWriter, r *http.Request) { defer rewriter.finish() } - // Enforce max query length AFTER offset is applied (start/end reflect shifted range). - if startNs, okS := parseLokiTimeToUnixNano(r.FormValue("start")); okS { - if endNs, okE := parseLokiTimeToUnixNano(r.FormValue("end")); okE { - if errMsg := p.checkQueryRangeLength(r.Context(), startNs, endNs); errMsg != "" { - p.writeError(w, http.StatusBadRequest, errMsg) - p.metrics.RecordRequest("query_range", http.StatusBadRequest, time.Since(start)) - return - } - } - } - if p.handleOrderedJSONMetric(w, r, start, logqlQuery, true) { return } diff --git a/internal/proxy/step_grid.go b/internal/proxy/step_grid.go new file mode 100644 index 000000000..f45b042d5 --- /dev/null +++ b/internal/proxy/step_grid.go @@ -0,0 +1,143 @@ +package proxy + +import ( + "net/http" + "net/url" + "strconv" + "strings" + "time" + + logqlpkg "github.com/ReliablyObserve/Loki-VL-proxy/internal/logql" +) + +// lokiRangeBounds returns the start and end of a range request as Loki decodes +// them (pkg/loghttp.ParseRangeQuery): `end` defaults to now, `start` to +// min(end, now) minus `since` (1h). It fails on a bound that does not parse and +// on a `since` that does not. +func lokiRangeBounds(r *http.Request, now time.Time) (start, end time.Time, ok bool) { + end = now + if raw := strings.TrimSpace(r.FormValue("end")); raw != "" { + ns, ok := parseLokiTimeToUnixNano(raw) + if !ok { + return time.Time{}, time.Time{}, false + } + end = time.Unix(0, ns) + } + if raw := strings.TrimSpace(r.FormValue("start")); raw != "" { + ns, ok := parseLokiTimeToUnixNano(raw) + if !ok { + return time.Time{}, time.Time{}, false + } + return time.Unix(0, ns), end, true + } + since := time.Hour + if raw := strings.TrimSpace(r.FormValue("since")); raw != "" { + var ok bool + if since, ok = parsePositiveStepDuration(raw); !ok { + return time.Time{}, time.Time{}, false + } + } + endOrNow := end + if end.After(now) { + endOrNow = now + } + return endOrNow.Add(-since), end, true +} + +// alignDownToStep truncates a unix-nanosecond timestamp down to the nearest +// multiple of step, including a timestamp before the epoch. +func alignDownToStep(ns, stepNs int64) int64 { + rem := ns % stepNs + if rem < 0 { + rem += stepNs + } + return ns - rem +} + +// alignRangeRequestToStepGrid rewrites a metric range request's start and end +// onto Loki's evaluation grid: multiples of `step`, both bounds truncated DOWN. +// +// This is what Loki's queryrange step-align middleware does when +// `query_range.align_queries_with_step` is set, and it is visible from the +// outside. On 3.7.1 with step=137s and a start that is not a multiple of 137, +// every returned timestamp satisfies `ts % 137 == 0` and the first point lands +// 17 seconds BEFORE the requested start. Starting the grid at `start` instead +// shifts the whole series by `start mod step` — 17s at step=137 and 19s at +// step=97 for start=1700000000 — while a step that happens to divide the start +// looks perfectly fine, which is why the mismatch only shows up on odd steps. +// +// A request without `start` or `end` is aligned on the bounds Loki derives for +// it (end=now, start=end-since), which are written into the request so every +// downstream path evaluates the same grid. +// +// LOG queries are left alone: they have no evaluation grid, and moving their +// bounds would change which lines they return. The query has passed +// validateQuery by the time this runs, so it parses. +// +// Two places Loki aligns and this does not, both because aligning them here +// would align to a step nothing downstream uses. A request with no `step` is +// left untouched: Loki computes max(floor(range/250), 1s) at decode time and +// aligns to that, while this proxy's range evaluator falls back to 1m, so the +// two would disagree either way. `/loki/api/v1/patterns` is left untouched for +// the same reason: Loki runs it through the same middleware (patternConfig +// inherits align_queries_with_step in roundtrip.go), but its default step is +// range/250 against this proxy's range/120. Both are listed in +// docs/compatibility-loki.md. +func alignRangeRequestToStepGrid(r *http.Request, logqlQuery string, now time.Time) { + if r == nil { + return + } + // The same parser as the 11,000-point check and the range evaluators, so + // the grid, the limit and the evaluation agree on what one step is. That + // includes the raw nanosecond integer Grafana sometimes sends, a proxy + // extension Loki does not have (Loki reads a bare integer as seconds). + stepNs, ok := parseStepToNanos(r.FormValue("step")) + if !ok || stepNs <= 0 { + return + } + if !isMetricRangeExpr(logqlQuery) { + return + } + start, end, ok := lokiRangeBounds(r, now) + if !ok { + return + } + + setForm := func(key string, value int64) { + encoded := strconv.FormatInt(value, 10) + if r.Form == nil { + r.Form = url.Values{} + } + r.Form.Set(key, encoded) + if r.PostForm != nil && r.PostForm.Get(key) != "" { + r.PostForm.Set(key, encoded) + } + if r.URL != nil { + q := r.URL.Query() + q.Set(key, encoded) + r.URL.RawQuery = q.Encode() + } + } + + if aligned := alignDownToStep(start.UnixNano(), stepNs); aligned != start.UnixNano() || r.FormValue("start") == "" { + setForm("start", aligned) + } + if aligned := alignDownToStep(end.UnixNano(), stepNs); aligned != end.UnixNano() || r.FormValue("end") == "" { + setForm("end", aligned) + } +} + +// isMetricRangeExpr reports whether a LogQL query produces a matrix — the only +// shape Loki evaluates on a step grid. +func isMetricRangeExpr(logqlQuery string) bool { + parsed, err := logqlpkg.Parse(logqlQuery) + if err != nil { + return false + } + switch parsed.(type) { + case *logqlpkg.RangeAggregation, *logqlpkg.VectorAggregation, *logqlpkg.BinOpExpr, + *logqlpkg.OpaqueMetricExpr, *logqlpkg.LiteralExpr: + return true + } + return false +} diff --git a/internal/proxy/step_grid_test.go b/internal/proxy/step_grid_test.go new file mode 100644 index 000000000..c58b58755 --- /dev/null +++ b/internal/proxy/step_grid_test.go @@ -0,0 +1,235 @@ +package proxy + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "testing" + "time" + + "github.com/ReliablyObserve/Loki-VL-proxy/internal/cache" +) + +func TestAlignRangeRequestToStepGrid(t *testing.T) { + const step = 137 * time.Second + + tests := []struct { + name string + query string + start int64 + end int64 + wantStart int64 + wantEnd int64 + }{ + { + name: "both bounds truncate down to a multiple of the step", + query: `sum(rate({app="x"}[5m]))`, + // 1700000000 = 12408759*137 + 17 and 1700003600 = 12408785*137 + 55, + // so both bounds move DOWN to the grid point below them. + start: 1700000000, end: 1700003600, + wantStart: 1700000000 - 17, wantEnd: 1700003600 - 55, + }, + { + name: "already aligned bounds are untouched", + query: `sum(rate({app="x"}[5m]))`, + start: 12408759 * 137, end: 12408785 * 137, + wantStart: 12408759 * 137, wantEnd: 12408785 * 137, + }, + { + name: "a log query has no evaluation grid", + query: `{app="x"} |~ "boom"`, + start: 1700000000, end: 1700003600, + wantStart: 1700000000, wantEnd: 1700003600, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + form := url.Values{} + form.Set("start", strconv.FormatInt(tc.start*int64(time.Second), 10)) + form.Set("end", strconv.FormatInt(tc.end*int64(time.Second), 10)) + form.Set("step", strconv.Itoa(int(step.Seconds()))) + r := httptest.NewRequest(http.MethodGet, "/loki/api/v1/query_range?"+form.Encode(), nil) + + alignRangeRequestToStepGrid(r, tc.query, time.Now()) + + gotStart, _ := parseLokiTimeToUnixNano(r.FormValue("start")) + gotEnd, _ := parseLokiTimeToUnixNano(r.FormValue("end")) + if gotStart != tc.wantStart*int64(time.Second) { + t.Errorf("start = %d, want %d", gotStart/int64(time.Second), tc.wantStart) + } + if gotEnd != tc.wantEnd*int64(time.Second) { + t.Errorf("end = %d, want %d", gotEnd/int64(time.Second), tc.wantEnd) + } + // The rewritten bound has to reach the URL too: downstream paths + // rebuild the backend request from r.URL. + if q := r.URL.Query().Get("start"); q != r.FormValue("start") { + t.Errorf("URL start = %q, form start = %q", q, r.FormValue("start")) + } + }) + } +} + +func TestAlignRangeRequestToStepGrid_NoStepIsANoOp(t *testing.T) { + r := httptest.NewRequest(http.MethodGet, "/loki/api/v1/query_range?start=1700000000000000000&end=1700003600000000000", nil) + alignRangeRequestToStepGrid(r, `sum(rate({app="x"}[5m]))`, time.Now()) + if got := r.FormValue("start"); got != "1700000000000000000" { + t.Errorf("start = %q, want it untouched without a step", got) + } +} + +// Loki aligns the bounds it derives for a request without start or end +// (end=now, start=end-since) the same way, so the proxy writes the aligned +// derived bounds into the request. +func TestAlignRangeRequestToStepGrid_DerivesLokiDefaultBounds(t *testing.T) { + now := time.Unix(1700000000, 0) // 12408759*137 + 17 + for _, tc := range []struct { + name, rawQuery string + wantStart int64 + }{ + {"no bounds: end=now, start=now-1h", "step=137", 1699996284}, // 1699996400 = 12408732*137 + 116 + {"since only", "step=137&since=10m", 1699999298}, // 1699999400 = 12408754*137 + 102 + } { + t.Run(tc.name, func(t *testing.T) { + r := httptest.NewRequest(http.MethodGet, "/loki/api/v1/query_range?"+tc.rawQuery, nil) + alignRangeRequestToStepGrid(r, `sum(rate({app="x"}[5m]))`, now) + gotStart, _ := parseLokiTimeToUnixNano(r.FormValue("start")) + gotEnd, _ := parseLokiTimeToUnixNano(r.FormValue("end")) + if gotStart != tc.wantStart*int64(time.Second) { + t.Errorf("start = %d, want %d", gotStart/int64(time.Second), tc.wantStart) + } + if gotEnd != (1700000000-17)*int64(time.Second) { + t.Errorf("end = %d, want %d", gotEnd/int64(time.Second), 1700000000-17) + } + if q := r.URL.Query().Get("end"); q != r.FormValue("end") { + t.Errorf("URL end = %q, form end = %q", q, r.FormValue("end")) + } + }) + } +} + +// The wiring half: with align-queries-with-step on, the bounds the backend sees +// are Loki's grid; with it off they are the client's own, which is Loki's +// binary default. +func TestQueryRangeAlignsBoundsBeforeQueryingTheBackend(t *testing.T) { + const ( + step = 137 + start = int64(1700000000) // 12408759*137 + 17 + lookback = 300 // the [5m] range vector of the query below + ) + + for _, tc := range []struct { + name string + align bool + wantStart int64 + }{ + {"aligned", true, start - 17 - lookback}, + {"not aligned", false, start - lookback}, + } { + t.Run(tc.name, func(t *testing.T) { + var seen url.Values + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = r.ParseForm() + seen = r.Form + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"status":"success","data":{"resultType":"matrix","result":[]}}`) + })) + defer backend.Close() + + p, err := New(Config{ + BackendURL: backend.URL, + Cache: cache.New(60*time.Second, 100), + LogLevel: "error", + AlignQueriesWithStep: tc.align, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = p.Shutdown(t.Context()) }) + + form := url.Values{} + form.Set("query", `sum(count_over_time({app="x"}[5m]))`) + form.Set("start", strconv.FormatInt(start*int64(time.Second), 10)) + form.Set("end", strconv.FormatInt((start+3600)*int64(time.Second), 10)) + form.Set("step", strconv.Itoa(step)) + + rec := httptest.NewRecorder() + p.handleQueryRange(rec, httptest.NewRequest(http.MethodGet, "/loki/api/v1/query_range?"+form.Encode(), nil)) + if rec.Code != http.StatusOK { + t.Fatalf("HTTP %d: %s", rec.Code, rec.Body.String()) + } + + got := seen.Get("start") + if got == "" { + t.Fatal("backend was never asked for a range") + } + ts, ok := backendStartUnix(got) + if !ok { + t.Fatalf("backend start %q is neither RFC3339 nor a Loki timestamp", got) + } + // The backend read starts one range-vector window before the first + // evaluation point, so the two cases differ by exactly the 17s the + // alignment removes. + if ts != tc.wantStart { + t.Errorf("backend start = %s (%d), want %d", got, ts, tc.wantStart) + } + }) + } +} + +func backendStartUnix(raw string) (int64, bool) { + if ts, err := time.Parse(time.RFC3339Nano, raw); err == nil { + return ts.Unix(), true + } + if ns, ok := parseLokiTimeToUnixNano(raw); ok { + return ns / int64(time.Second), true + } + return 0, false +} + +// The alignment reads the step with the same parser as the range evaluators +// and the 11,000-point check, including the raw nanosecond integer Grafana +// sometimes sends. That spelling is a proxy extension (Loki reads a bare +// integer as seconds); what matters is that every consumer of the step in one +// request agrees on it, so a panel is never aligned on one grid and evaluated +// on another. +func TestAlignRangeRequestAcceptsNanosecondStep(t *testing.T) { + for _, step := range []string{"137s", "137", "137000000000"} { + r := httptest.NewRequest(http.MethodGet, + "/loki/api/v1/query_range?query="+url.QueryEscape(`sum(rate({app="a"}[5m]))`)+ + "&start=1700000000&end=1700003600&step="+step, nil) + alignRangeRequestToStepGrid(r, `sum(rate({app="a"}[5m]))`, time.Now()) + if got := r.FormValue("start"); got != "1699999983000000000" { + t.Errorf("step=%s: start = %s, want 1699999983000000000", step, got) + } + if got := r.FormValue("end"); got != "1700003545000000000" { + t.Errorf("step=%s: end = %s, want 1700003545000000000", step, got) + } + } +} + +// The 11,000-point check reads the step exactly as the alignment and the +// evaluators do, so a nanosecond-integer step is neither aligned on one grid +// and limited on another, nor exempt from the limit. (A bare integer small +// enough to be seconds is seconds; the nanosecond reading applies to the +// integers Grafana sends, which overflow as seconds.) +func TestResolutionLimitReadsStepLikeTheEvaluator(t *testing.T) { + now := time.Unix(1700000000, 0) + const rangeDur = 130 * 24 * time.Hour // 11,232 steps of 1000s + for _, tc := range []struct { + step string + want bool + }{ + {"1000s", true}, + {"1000000000000", true}, // the same step as a nanosecond integer + {"2000s", false}, + {"2000000000000", false}, + } { + r := httptest.NewRequest(http.MethodGet, "/loki/api/v1/query_range?start="+strconv.FormatInt(now.Add(-rangeDur).UnixNano(), 10)+"&end="+strconv.FormatInt(now.UnixNano(), 10)+"&step="+tc.step, nil) + if got := exceedsLokiRangeResolution(r, now); got != tc.want { + t.Errorf("step=%s: exceeds = %v, want %v", tc.step, got, tc.want) + } + } +} diff --git a/test/e2e-compat/docker-compose.yml b/test/e2e-compat/docker-compose.yml index dc0dafb38..0e02f6cd8 100644 --- a/test/e2e-compat/docker-compose.yml +++ b/test/e2e-compat/docker-compose.yml @@ -272,6 +272,7 @@ services: # sides at the same cardinality, so parity tests compare answers # rather than two different limits. - "-max-stats-query-series=1000000" + - "-align-queries-with-step=true" - "-ruler-backend=http://vmalert:8880" - "-alerts-backend=http://vmalert:8880" - "-log-level=info" @@ -345,6 +346,7 @@ services: # sides at the same cardinality, so parity tests compare answers # rather than two different limits. - "-max-stats-query-series=1000000" + - "-align-queries-with-step=true" - "-ruler-backend=http://vmalert:8880" - "-alerts-backend=http://vmalert:8880" - "-log-level=info" @@ -410,6 +412,7 @@ services: # sides at the same cardinality, so parity tests compare answers # rather than two different limits. - "-max-stats-query-series=1000000" + - "-align-queries-with-step=true" - "-ruler-backend=http://vmalert:8880" - "-alerts-backend=http://vmalert:8880" - "-log-level=info" @@ -476,6 +479,7 @@ services: # exercised by the e2e suite. Every other proxy here matches this # stack's Loki (max_query_series: 1000000) so parity tests compare # answers rather than two different limits. + - "-align-queries-with-step=true" - "-ruler-backend=http://vmalert:8880" - "-alerts-backend=http://vmalert:8880" - "-log-level=info" @@ -520,6 +524,7 @@ services: command: - "-listen=:3100" - "-backend=http://vmauth:8427" + - "-align-queries-with-step=true" - "-ruler-backend=http://vmalert:8880" - "-alerts-backend=http://vmalert:8880" - "-log-level=info" @@ -569,6 +574,7 @@ services: # sides at the same cardinality, so parity tests compare answers # rather than two different limits. - "-max-stats-query-series=1000000" + - "-align-queries-with-step=true" - "-ruler-backend=http://vmalert:8880" - "-alerts-backend=http://vmalert:8880" - "-log-level=info" @@ -617,6 +623,7 @@ services: # sides at the same cardinality, so parity tests compare answers # rather than two different limits. - "-max-stats-query-series=1000000" + - "-align-queries-with-step=true" - "-ruler-backend=http://vmalert:8880" - "-alerts-backend=http://vmalert:8880" - "-log-level=info" @@ -660,6 +667,7 @@ services: # sides at the same cardinality, so parity tests compare answers # rather than two different limits. - "-max-stats-query-series=1000000" + - "-align-queries-with-step=true" - "-ruler-backend=http://vmalert:8880" - "-alerts-backend=http://vmalert:8880" - "-log-level=info" @@ -703,6 +711,7 @@ services: # sides at the same cardinality, so parity tests compare answers # rather than two different limits. - "-max-stats-query-series=1000000" + - "-align-queries-with-step=true" - "-ruler-backend=http://vmalert:8880" - "-alerts-backend=http://vmalert:8880" - "-log-level=info" @@ -746,6 +755,7 @@ services: # sides at the same cardinality, so parity tests compare answers # rather than two different limits. - "-max-stats-query-series=1000000" + - "-align-queries-with-step=true" - "-ruler-backend=http://vmalert:8880" - "-alerts-backend=http://vmalert:8880" - "-log-level=debug" @@ -788,6 +798,7 @@ services: # sides at the same cardinality, so parity tests compare answers # rather than two different limits. - "-max-stats-query-series=1000000" + - "-align-queries-with-step=true" - "-ruler-backend=http://vmalert:8880" - "-alerts-backend=http://vmalert:8880" - "-log-level=debug" diff --git a/test/e2e-compat/step_grid_parity_test.go b/test/e2e-compat/step_grid_parity_test.go new file mode 100644 index 000000000..c0159ffc6 --- /dev/null +++ b/test/e2e-compat/step_grid_parity_test.go @@ -0,0 +1,87 @@ +//go:build e2e + +package e2e_compat + +import ( + "fmt" + "net/http" + "net/url" + "sort" + "strings" + "testing" + "time" +) + +// The stack's Loki runs with query_range.align_queries_with_step: true and +// every proxy variant in docker-compose.yml with -align-queries-with-step, so +// a metric range query with an odd step and an unaligned start has to land +// its samples on the same k*step grid on both sides. On an unaligned proxy +// the whole series is shifted by start mod step and the first point differs. +// +// conformance: loki_api_v1_query_range +func TestCompat_StepAlignedGridMatchesLoki(t *testing.T) { + ensureDataIngested(t) + + const step = 137 * time.Second + end := time.Now().Truncate(time.Second) + start := end.Add(-30 * time.Minute) + if start.Unix()%int64(step.Seconds()) == 0 { + // An aligned start proves nothing; move it off the grid. + start = start.Add(time.Second) + } + params := url.Values{} + params.Set("query", `sum(count_over_time({app="api-gateway"}[5m]))`) + params.Set("start", fmt.Sprintf("%d", start.UnixNano())) + params.Set("end", fmt.Sprintf("%d", end.UnixNano())) + params.Set("step", fmt.Sprintf("%ds", int(step.Seconds()))) + + loki := stepGridTimestamps(t, lokiURL, params) + proxy := stepGridTimestamps(t, proxyURL, params) + if len(loki) == 0 { + t.Fatalf("Loki returned no samples for %s", params.Get("query")) + } + for _, ts := range loki { + if ts%int64(step.Seconds()) != 0 { + t.Fatalf("Loki timestamp %d is not on the %s grid; is align_queries_with_step on in loki-local-config.yaml?", ts, step) + } + } + if strings.Join(int64Strings(proxy), ",") != strings.Join(int64Strings(loki), ",") { + t.Fatalf("evaluation grids differ (start %d, step %s)\n loki: %v\n proxy: %v", start.Unix(), step, loki, proxy) + } +} + +// stepGridTimestamps returns the sorted, de-duplicated second timestamps of a +// matrix response. +func stepGridTimestamps(t *testing.T, baseURL string, params url.Values) []int64 { + t.Helper() + status, body, resp := doJSONGET(t, baseURL+"/loki/api/v1/query_range?"+params.Encode(), nil) + if status != http.StatusOK || !checkStatus(resp) { + t.Fatalf("%s: HTTP %d %s", baseURL, status, body) + } + seen := map[int64]struct{}{} + for _, item := range extractArray(extractMap(resp, "data"), "result") { + series, _ := item.(map[string]interface{}) + for _, point := range extractArray(series, "values") { + pair, _ := point.([]interface{}) + if len(pair) != 2 { + continue + } + ts, _ := pair[0].(float64) + seen[int64(ts)] = struct{}{} + } + } + out := make([]int64, 0, len(seen)) + for ts := range seen { + out = append(out, ts) + } + sort.Slice(out, func(i, j int) bool { return out[i] < out[j] }) + return out +} + +func int64Strings(values []int64) []string { + out := make([]string, len(values)) + for i, v := range values { + out[i] = fmt.Sprint(v) + } + return out +} diff --git a/test/e2e-ui/tests/explore-regression.spec.ts b/test/e2e-ui/tests/explore-regression.spec.ts index 32c9177f8..0a8e450b5 100644 --- a/test/e2e-ui/tests/explore-regression.spec.ts +++ b/test/e2e-ui/tests/explore-regression.spec.ts @@ -401,13 +401,20 @@ test.describe("@regression Proxy series cap", () => { // ingesters) and the proxy both list them immediately. const query = `rate({app="api-gateway"}[5m])`; await ensureStackWarm(page); - const end = windowEnd(); + // The proxy runs with -align-queries-with-step, like this stack's Loki, so + // a query_range is evaluated on the window truncated down to the step + // grid. Count indexed streams over that same window: with an unaligned end + // the index also lists streams created in the last partial step, which the + // aligned query no longer covers. + const step = 60; + const rawEnd = windowEnd(); + const end = rawEnd - (rawEnd % step); // 15 minutes is enough to reach the cap on a stack older than a few // minutes and keeps both the proxy's zero-filled matrix and Loki's series // listing bounded on a long-running stack. const windowSec = 15 * 60; const [proxy, indexed] = await Promise.all([ - queryRange(page, proxyUID, query, { step: "60", windowSec, endSec: end }), + queryRange(page, proxyUID, query, { step: String(step), windowSec, endSec: end }), lokiIndexedStreamCount(page, lokiUID, `{app="api-gateway"}`, end - windowSec, end), ]); expect(proxy.statusCode, "raw rate: status code").toBe(200);