feat(proxy): add -align-queries-with-step for Loki's evaluation grid

CHANGELOG.md

@@ -7,8 +7,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- **`-align-queries-with-step` puts metric range queries on Loki's evaluation
+  grid.** Loki's step-align middleware truncates a range query's `start` and
+  `end` down to multiples of `step`, so on 3.7.1 with `step=137s` every returned
+  timestamp satisfies `ts % 137 == 0` and the first point sits before the
+  requested start. The proxy started its grid at the request's own `start`,
+  which shifted the whole series by `start mod step` — invisible whenever the
+  step happened to divide the start, which is why it only shows on odd steps.
+  Off by default, matching Loki's binary default; the Loki Helm chart turns the
+  option on, so set the flag to `true` to match a chart-deployed Loki. Log
+  queries are never moved: they have no evaluation grid, and shifting their
+  bounds would change which lines they return. Requests that omit `step`, and
+  `/loki/api/v1/patterns`, are left unaligned: Loki aligns both to a default
+  step of `range/250` that this proxy does not use downstream, so aligning to
+  it here would move the bounds off the grid the response is actually built on.
+  A request without `start` or `end` is aligned on the bounds Loki derives for
+  it (`end=now`, `start=end-since`). Binary expressions joining two vector
+  operands keep aligning both bounds whatever the flag says, as before: with
+  the flag off the join would otherwise have no common timestamps whenever one
+  operand is served from tumbling VictoriaLogs buckets. The e2e stack runs
+  every proxy variant with the flag on, matching its Loki.
 ### Fixed
 
+- **`max_query_length` is checked on the range the client asked for**, before
+  step alignment, as Loki's limits middleware runs before its step-align
+  middleware. Alignment can grow the range by up to `step-1ns`, so a query
+  exactly at the limit was rejected where Loki accepts it.
+- **The 11,000-point resolution check reads the step as the evaluators do.** It
+  parsed the step with a duration-only parser, so the raw nanosecond integer
+  Grafana sometimes sends (a proxy extension; Loki reads a bare integer as
+  seconds) was exempt from the limit while being evaluated, and now aligned,
+  on that step.
 - **A release is validated, tagged and published from the commit its version
   was computed from.** Since releases queue instead of cancelling each other,
   the first release job re-reads `main` before computing the next version, but

charts/loki-vl-proxy/values.yaml

@@ -109,6 +109,7 @@ extraArgs:
   # query-range-partial-responses: "false"             # return partial response on retryable backend failures
   # query-range-background-warm: "true"                # continue warming failed windows in background
   # query-range-background-warm-max-windows: "24"      # cap background warm fanout
+  # align-queries-with-step: "false"                    # truncate metric range bounds down to the step grid, as Loki does under query_range.align_queries_with_step (off by default like Loki's binary; the Loki Helm chart turns it on)
   # default-max-query-length: "0"                      # max query time range for all tenants (0 = unlimited; per-tenant limits override)
   # max-stats-query-series: "500"                       # max series a metric query may return (count_over_time, rate, bytes_rate); 0 = built-in default of 500, Loki's own max_query_series default; above it the query fails with Loki's HTTP 400 "maximum number of series (N) reached for a single query" (Logs Drilldown gets a partial result with a warning)
   # stats-query-range-concurrency: "4"                  # max concurrent stats_query_range calls to VL; higher = more VL CPU pressure; 0 = built-in default of 4

cmd/proxy/main.go

@@ -204,6 +204,7 @@ type proxyRuntimeConfig struct {
 	coldBackendManifestRefresh          time.Duration
 	coldBackendTimeout                  time.Duration
 	defaultMaxQueryLength               time.Duration
+	alignQueriesWithStep                bool
 	maxStatsQuerySeries                 int
 	statsQueryRangeConcurrency          int
 	backendMaxConcurrentHeavyQueries    int
@@ -627,6 +628,7 @@ func run(
 	patternsPersistInterval := fs.Duration("patterns-persist-interval", 30*time.Second, "How often to persist in-memory patterns snapshots to disk")
 	patternsStartupStale := fs.Duration("patterns-startup-stale-threshold", 60*time.Second, "Treat on-disk patterns snapshot older than this as stale and warm from peers before serving")
 	patternsPeerWarmTimeout := fs.Duration("patterns-startup-peer-warm-timeout", 5*time.Second, "Maximum time to wait for startup patterns snapshot warm from peers")
+	alignQueriesWithStep := fs.Bool("align-queries-with-step", false, "Truncate a metric range query's start and end down to multiples of the step before evaluation, the way Loki's step-align middleware does under query_range.align_queries_with_step. Applies to /loki/api/v1/query_range with an explicit step. Off by default, matching Loki's binary default; the Loki Helm chart turns it on, so set this to true to match a chart-deployed Loki.")
 	allowGlobalTenant := fs.Bool("tenant.allow-global", false, `Allow X-Scope-OrgID "*" to bypass AccountID/ProjectID scoping and use the backend default tenant`)
 
 	// Cold storage backend (Victoria Lakehouse)
@@ -916,6 +918,7 @@ func run(
 			coldBackendManifestRefresh:          *coldBackendManifestRefresh,
 			coldBackendTimeout:                  *coldBackendTimeout,
 			defaultMaxQueryLength:               *defaultMaxQueryLength,
+			alignQueriesWithStep:                *alignQueriesWithStep,
 			maxStatsQuerySeries:                 *maxStatsQuerySeries,
 			statsQueryRangeConcurrency:          *statsQueryRangeConcurrency,
 			backendMaxConcurrentHeavyQueries:    *backendMaxConcurrentHeavyQueries,
@@ -2094,6 +2097,7 @@ func buildProxyConfig(cfg proxyRuntimeConfig) (proxy.Config, error) {
 			Timeout:         cfg.coldBackendTimeout,
 		},
 		DefaultMaxQueryLength:            cfg.defaultMaxQueryLength,
+		AlignQueriesWithStep:             cfg.alignQueriesWithStep,
 		MaxStatsQuerySeries:              cfg.maxStatsQuerySeries,
 		StatsQueryRangeConcurrency:       cfg.statsQueryRangeConcurrency,
 		BackendMaxConcurrentHeavyQueries: cfg.backendMaxConcurrentHeavyQueries,

conformance/registry/generated/wiring.json

@@ -85,7 +85,8 @@
       "internal/proxy/tumbling_axis_parity_test.go",
       "internal/translator/regexp_escaping_test.go",
       "test/e2e-compat/heavy_query_protection_test.go",
-      "test/e2e-compat/range_metric_offset_series_limit_test.go"
+      "test/e2e-compat/range_metric_offset_series_limit_test.go",
+      "test/e2e-compat/step_grid_parity_test.go"
     ],
     "operator-configurable-limits": [
       "cmd/proxy/main_test.go",
@@ -244,9 +245,12 @@
       "semantics/offset-sample-timestamps",
       "series-limits-and-partial-results",
       "window-bounds-and-step-alignment"
+    ],
+    "test/e2e-compat/step_grid_parity_test.go": [
+      "loki_api_v1_query_range"
     ]
   },
-  "tests_with_markers": 17,
+  "tests_with_markers": 18,
   "unknown_ids": {},
   "unwired_items": [
     "api_prom_label",

docs/compatibility-loki.md

@@ -204,6 +204,13 @@ Some important compatibility behavior is still tested, but it is not part of the
 
 Those cases live in the proxy contract suite because Loki itself is not the source of truth for them.
 
+Documented deviations from Loki's step alignment (`-align-queries-with-step`):
+
+- A metric range request without `step` is not aligned. Loki derives `max(floor(range/250), 1s)` at decode time and aligns to it; the proxy's range evaluator falls back to `1m`, so aligning to Loki's derived step would move the bounds off the grid the response is built on.
+- `/loki/api/v1/patterns` is not aligned. Loki runs it through the same middleware, but its default step is `range/250` against the proxy's `range/120`.
+- A binary expression joining two vector operands is aligned to the step grid whether or not the flag is set. Operands served from tumbling VictoriaLogs buckets sit on the epoch-aligned grid, so without alignment a join with an unaligned `start` has no common timestamps; against a default (unaligned) Loki the join's grid is shifted by less than one step.
+- A bare integer `step` too large to be seconds is read as nanoseconds, a proxy extension for the value Grafana sometimes sends; Loki reads every bare integer as seconds. The alignment, the 11,000-point check and the evaluators all read it the same way.
+
 ## Parity Rule
 
 Valid Loki behavior is not an allowed exclusion category.

docs/configuration.md

@@ -458,6 +458,7 @@ These flags control Loki-compatible `query_range` split/merge execution with per
 
 | Flag | Env | Default | Description |
 |---|---|---|---|
+| `-align-queries-with-step` | — | `false` | Truncate a metric range query's `start` and `end` down to multiples of `step` before evaluation, the way Loki's step-align middleware does under `query_range.align_queries_with_step`. Off matches Loki's binary default; the Loki Helm chart turns the option on, so set this to `true` to match a chart-deployed Loki. Log queries are never moved |
 | `-drilldown-scan-timeout` | — | `5s` | Per-request timeout for the `detected_fields` / `detected_field/{name}/values` log-scan path. `0` disables the cap |
 | `-max-stats-query-series` | — | `0` (uses `500`) | Maximum series a metric query may return (`count_over_time`, `rate`, `bytes_rate`, ...), the proxy's equivalent of Loki's `max_query_series`; `0` uses the built-in default of `500`, which is also Loki's default. Above it the query is rejected with HTTP `400` and Loki's message `maximum number of series (N) reached for a single query`, on range and instant queries alike; the log line for the rejected request names this flag. Grafana Logs Drilldown requests receive a partial result (the busiest series) with Loki's `... returning partial results` warning instead, as Loki does for that client (see [Fixed Execution Limits](#fixed-execution-limits)) |
 | `-stats-query-range-concurrency` | — | `0` (uses `4`) | Maximum concurrent `stats_query_range` calls to VictoriaLogs. `0` uses the built-in default of `4` |

docs/real-window-compatibility-gaps.md

@@ -30,9 +30,10 @@ The exhaustive checks compare status, result type and non-emptiness. Strict quan
 - Tumbling range aggregations (range equal to step) served from VictoriaLogs `stats_query_range` buckets are relabelled to Loki's evaluation timestamps. VictoriaLogs labels a bucket by its start and covers `[T, T+step)`; Loki's sample at `T` covers `(T-range, T]`. The proxy fetches from `start-range` and moves each bucket forward by one range, instead of trimming the bucket Loki's first sample needs. It covers `rate` and `bytes_rate` in any aggregation, and ungrouped `sum(count_over_time(...))` and `sum(bytes_over_time(...))`.
 - Sliding range aggregations (`count_over_time`, `rate`, `bytes_over_time`, `bytes_rate` with a range different from the step) served from `stats_query_range` buckets sum buckets of `gcd(step, range)` anchored to the request start and shifted by one nanosecond through the `offset` argument, so each bucket covers `(T, T+bucket]` and every window `(t-range, t]` is an exact union of buckets, including lines on window edges and unaligned starts. A step whose window holds no line is absent, as in Loki, for every client including Drilldown. Buckets below 1 ms, stats responses over the byte limit, and unaligned grids on a backend older than VictoriaLogs v1.45 or of undetected version use the raw-sample evaluator, which applies the same `(t-range, t]` boundaries to these functions for range and instant queries.
 - Bare parser range aggregations without an outer aggregation (`count_over_time({...} | logfmt [90s])`, also `regexp`, `pattern` and `json` with parameters) use the same anchored `gcd(step, range)` buckets: `stats_query_range` grouped by stream for `count_over_time`, `rate`, `bytes_over_time` and `bytes_rate` with a range greater than or equal to the step (byte sums add a `count()` presence column, so a window holding only empty lines is a zero sample), VictoriaLogs `/hits` with the same `offset` argument for sliding `count_over_time` and `rate` when stream label fields are declared, and bucket values for `sum_over_time`, `max_over_time` and `min_over_time` over `unwrap`. Tumbling queries on this path are no longer merged into one unlabelled series. Unaligned grids on a backend without `offset` support use the raw evaluator. Bare `| json` without parameters keeps the ordered JSON evaluator.
-- Binary expressions joining two vector operands evaluate both on the step-aligned grid, as Loki's query frontend does when `align_queries_with_step` is enabled. The compatibility stack enables it; a default Loki evaluates at `start+k*step`, so results with an unaligned start differ by less than one step. Operands with different ranges take different execution paths (sliding evaluation versus tumbling buckets), and an unaligned start previously left no common timestamps to join.
+- Binary expressions joining two vector operands evaluate both on the step-aligned grid, as Loki's query frontend does when `align_queries_with_step` is enabled, whether or not `-align-queries-with-step` is set on the proxy. The compatibility stack enables both; a default Loki evaluates at `start+k*step`, so a binary join with an unaligned start differs from it by less than one step. Operands with different ranges take different execution paths (sliding evaluation versus tumbling buckets), and an unaligned start previously left no common timestamps to join.
 - `unwrap duration(...)` and `unwrap bytes(...)` over bare parsers convert unit strings such as `15ms` and `1024B` instead of dropping every sample.
 - Range operands grouped `by (level)` keep Loki's `level` key, so `on(level)` matching works for range queries as it already did for instant queries.
+- With `-align-queries-with-step`, every metric range query (not only binary joins) evaluates on Loki's `k*step` grid: both bounds truncate down to a multiple of the step, as Loki's step-align middleware does. Measured on the stack at `step=137` with an unaligned start, Loki and the proxy return the same 22 timestamps where an unaligned proxy returns 21 starting 26s late.
 - Scalar results carry second timestamps and all sample values use Loki's fixed-point rendering (`1234000`, not `1.234e+06`).
 
 These corrections reject previously accepted invalid queries and change incorrect numeric results. They do not imply unchanged behavior for all clients. IP validation is eager: Loki can bypass invalid pipeline construction for historical empty ranges, while the proxy rejects the invalid expression.

docs/reference/configuration-reference.md

@@ -2,7 +2,7 @@
 
 # Configuration reference
 
-Every command-line flag of the proxy (208), grouped by category, with the Helm value that sets it. Narrative guidance lives in [configuration.md](../configuration.md); the bounds on work are collected in [limits-registry.md](limits-registry.md).
+Every command-line flag of the proxy (209), grouped by category, with the Helm value that sets it. Narrative guidance lives in [configuration.md](../configuration.md); the bounds on work are collected in [limits-registry.md](limits-registry.md).
 
 Helm passes any flag through `extraArgs.<flag>`; the chart sets a few of them from dedicated values, marked chart-managed.
 
@@ -71,6 +71,7 @@ Helm passes any flag through `extraArgs.<flag>`; the chart sets a few of them fr
 
 | Flag | Type | Default | Helm value | Description |
 |---|---|---|---|---|
+| `-align-queries-with-step` | bool | `false` | `extraArgs.align-queries-with-step` | Truncate a metric range query's start and end down to multiples of the step before evaluation, the way Loki's step-align middleware does under query_range.align_queries_with_step. Applies to /loki/api/v1/query_range with an explicit step. Off by default, matching Loki's binary default; the Loki Helm chart turns it on, so set this to true to match a chart-deployed Loki. |
 | `-backend-allow-unsupported-version` | bool | `false` | `extraArgs.backend-allow-unsupported-version` | Allow startup with backend versions lower than -backend-min-version (at your own risk). Ignored when --backend-version-strict=true. |
 | `-backend-min-version` | string | `"v1.30.0"` | `extraArgs.backend-min-version` | Minimum VictoriaLogs version considered fully supported at startup |
 | `-backend-version-strict` | bool | `false` | `extraArgs.backend-version-strict` | When true, /health failure, non-2xx response, or missing/sub-min backend semver causes startup to fail. Default false (warn only). Overrides --backend-allow-unsupported-version when both are set. |

docs/translation-reference.md

@@ -136,7 +136,7 @@ Supported operators: `+`, `-`, `*`, `/`, `%`, `^`, `==`, `!=`, `>`, `<`, `>=`, `
 Binary expression notes:
 
 - Each operand is executed through the normal query handlers, so trailing range windows, parser errors and extraction aliases are preserved. Operator precedence, parentheses and comparison filtering versus `bool` follow Loki.
-- Vector-vector operands are evaluated on the step-aligned grid, as Loki's query frontend does with `align_queries_with_step` enabled. Against a default Loki, results for an unaligned `start` can differ by less than one step.
+- Vector-vector operands are evaluated on the step-aligned grid, as Loki's query frontend does with `align_queries_with_step` enabled, whatever `-align-queries-with-step` says. Against a default Loki, results for an unaligned `start` can differ by less than one step. With `-align-queries-with-step` every metric range query is aligned the same way.
 - Implicit many-to-one matches (without `group_left`/`group_right`) are rejected with HTTP 500 and Loki's `multiple matches for labels` error; cardinality is checked at each timestamp.
 - Evaluation is bounded: 64 nesting levels, 1,024 child evaluations, 256 MiB of captured child responses, two million decoded arrays, one million output samples and 64 MiB of label work, with each encoded result capped at 64 MiB. A valid expression exceeding these limits fails explicitly.
 

internal/proxy/binary_logql.go

@@ -290,18 +290,23 @@ func (p *Proxy) proxyBinaryLogQL(w http.ResponseWriter, r *http.Request, expr *l
 // the operands are evaluated. Operands with different range windows take
 // different execution paths: sliding windows are evaluated at start+k*step,
 // tumbling windows come from VictoriaLogs buckets on the epoch-aligned grid.
-// Sharing one aligned axis lets the join match samples, and mirrors Loki's
-// query frontend with align_queries_with_step. Requests without a parseable
-// start, end or step are returned unchanged.
+// Sharing one aligned axis lets the join match samples at all.
+//
+// This runs regardless of -align-queries-with-step, and that is a recorded
+// deviation rather than an oversight: with the flag off the join would
+// otherwise have no common timestamps whenever one operand is tumbling and the
+// start is unaligned, and an empty join is worse than one shifted by less than
+// a step against a default Loki. With the flag on the request arrives already
+// aligned and this is a no-op. Requests without a parseable start, end or step
+// are returned unchanged.
 func alignBinaryRangeRequest(r *http.Request) *http.Request {
 	start, startOK := parseLokiTimeToUnixNano(r.FormValue("start"))
 	end, endOK := parseLokiTimeToUnixNano(r.FormValue("end"))
-	step, stepOK := parsePositiveStepDuration(r.FormValue("step"))
+	stepNs, stepOK := parseStepToNanos(r.FormValue("step"))
 	if !startOK || !endOK || !stepOK || end < start {
 		return r
 	}
-	stepNs := int64(step)
-	alignedStart, alignedEnd := start-start%stepNs, end-end%stepNs
+	alignedStart, alignedEnd := alignDownToStep(start, stepNs), alignDownToStep(end, stepNs)
 	if alignedStart == start && alignedEnd == end {
 		return r
 	}

internal/proxy/config.go

@@ -49,6 +49,7 @@ type HandlerConfig struct {
 	tenantDefaultLimits                   map[string]any            // snapshot at construction — not hot-reloadable via HandlerConfig
 	tenantLimits                          map[string]map[string]any // snapshot at construction — not hot-reloadable via HandlerConfig
 	defaultMaxQueryLength                 time.Duration
+	alignQueriesWithStep                  bool
 	queryRangeWindowing                   bool
 	queryRangeSplitInterval               time.Duration
 	queryRangeMaxParallel                 int