From 30a0e24fc09bde5704efd23577655402e725b73b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Gro=C3=9F?= Date: Sat, 24 Oct 2026 19:51:10 +0200 Subject: [PATCH] =?UTF-8?q?test:=20composition=20sweeps=20over=20outer=20j?= =?UTF-8?q?oins,=20nullable=20operand=20pairs,=20quantifiers,=20repeated?= =?UTF-8?q?=20execution,=20filtered=20aggregates=20and=20declared=20nullab?= =?UTF-8?q?ility=20(#375=E2=80=93#380)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six integration sweeps vary how a translated range, RangeSet or value set call composes inside a query, against a rule (c) reference that never lets a NULL reach the library (Composition/NullRule.cs), plus a reference-free partition check on every call: Where(P) and Where(!P) partition the rows and Select(P) says which side each row is on. - OuterJoinProbeTests (#375): every member over the child of a parent without one, through a navigation, GroupJoin with DefaultIfEmpty and LeftJoin. - NullableOperandPairProbeTests (#376): every binary member over two nullable columns, a CASE and a COALESCE of them. - QuantifierProbeTests (#377): Any/All/Count/Contains over captured lists holding NULL, compound predicates with De Morgan, nested predicates. - RepeatedExecutionProbeTests (#378): one query re-run with its parameter walked through null, finite, empty and infinite; compiled queries. - FilteredAggregateProbeTests (#379): RangeAgg/RangeIntersectAgg over groups a filter empties, all NULL, holding empty, or a LeftJoin key without rows, consumed by every member, HAVING and an ordering; E07 mapped. - DeclaredNullabilityTests (#380): every emitted function's declaration against pg_proc.proisstrict and the server, and every value's null test against its projection. Disagreements are registered as candidates: #381 and #382 (new), #360. Two new seeds prove the re-execution and aggregate sweeps; the existing null seeds now name the sweeps they were shown to catch. Co-Authored-By: Claude Opus 5.5 --- docs/oracles.md | 26 ++ .../Composition/CompositionDatabase.cs | 27 ++ .../Composition/CompositionMembers.cs | 171 ++++++++ .../Composition/CompositionRunner.cs | 137 ++++++ .../Composition/NullRule.cs | 202 +++++++++ .../DeclaredNullabilityTests.cs | 369 ++++++++++++++++ .../FilteredAggregateProbeTests.cs | 332 +++++++++++++++ .../NullableOperandPairProbeTests.cs | 186 +++++++++ .../OuterJoinProbeTests.cs | 256 ++++++++++++ .../QuantifierProbeTests.cs | 394 ++++++++++++++++++ .../RepeatedExecutionProbeTests.cs | 394 ++++++++++++++++++ test/oracles/axes.md | 6 +- test/oracles/candidates.json | 308 ++++++++++++++ test/oracles/claims.json | 16 +- test/oracles/oracles.json | 54 +++ .../seeds/aggregates-fold-a-null-input.patch | 4 +- ...clared-null-only-for-a-null-argument.patch | 4 +- .../seeds/range-agg-declared-never-null.patch | 17 + ...predicate-over-a-null-column-is-null.patch | 4 +- ...ql-processor-reuses-its-first-output.patch | 21 + ...predicate-over-a-null-column-is-null.patch | 4 +- 21 files changed, 2919 insertions(+), 13 deletions(-) create mode 100644 test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/Composition/CompositionDatabase.cs create mode 100644 test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/Composition/CompositionMembers.cs create mode 100644 test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/Composition/CompositionRunner.cs create mode 100644 test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/Composition/NullRule.cs create mode 100644 test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/DeclaredNullabilityTests.cs create mode 100644 test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/FilteredAggregateProbeTests.cs create mode 100644 test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/NullableOperandPairProbeTests.cs create mode 100644 test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/OuterJoinProbeTests.cs create mode 100644 test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/QuantifierProbeTests.cs create mode 100644 test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/RepeatedExecutionProbeTests.cs create mode 100644 test/oracles/seeds/range-agg-declared-never-null.patch create mode 100644 test/oracles/seeds/sql-processor-reuses-its-first-output.patch diff --git a/docs/oracles.md b/docs/oracles.md index f927ecca..71f0e4a2 100644 --- a/docs/oracles.md +++ b/docs/oracles.md @@ -265,6 +265,32 @@ range collection no `Contains` reads also fails loudly, as `42809` for `Any(==)` found #137: `Any` or `Count` with a predicate over an empty collection the plugin sends one parameter per element throws. +### The composition sweeps + +The sweeps above vary shapes, types and operand kinds, mostly in a projection over one nullable column at +most. #349, #350, #355, #360 and #361 came from how a translated call *composes*: where a NULL comes from, +what a filter, a negation or EF Core's null semantics do around it, and what the next execution reuses. Six +classes in the integration project (#362) vary that instead, each over `DateRange`, its `RangeSet` and +`StringSet`, with the members discovered (`Composition/CompositionMembers.cs`): + +| Class | What it composes | +|---|---| +| `OuterJoinProbeTests` (#375) | every member over the child of a parent without one: a navigation, `GroupJoin` with `DefaultIfEmpty`, `LeftJoin`; the range factories over the child's element columns | +| `NullableOperandPairProbeTests` (#376) | every binary member over two nullable columns, a `CASE` and a `COALESCE` of them, on every (null, value) pair; equality and `CompareTo` under their own rules | +| `QuantifierProbeTests` (#377) | `Any`, `All`, `Count`, `Contains` over captured lists holding a NULL; `P && Q`, `P \|\| Q` against De Morgan; a predicate over a value-producing member | +| `RepeatedExecutionProbeTests` (#378) | the same query run again with its parameter walked null → finite → empty → `(,)` → null and back; `EF.CompileQuery` and `EF.CompileAsyncQuery` | +| `FilteredAggregateProbeTests` (#379) | `RangeAgg` and `RangeIntersectAgg` over groups a filter empties, all NULL, holding `empty`, or reached by a `LeftJoin` with no rows, consumed by every member, `HAVING` and an ordering | +| `DeclaredNullabilityTests` (#380) | every function the translators emit, its declaration against `pg_proc.proisstrict` and the server; and every translated value's null test against its projection | + +Their reference is `Composition/NullRule.cs`: the expression rewritten so that a NULL never reaches the +library, the explicit guard a consumer writes over loaded rows. A predicate with a null on either side is +`false` and its negation `true` (rule (c)); a value-producing member over a null, or over a missing element, +has no value; equality and `CompareTo` keep their documented rules; the aggregates skip a null. So what the +library does *today* for a null in memory (#350's 11.0.0 half) is never what a row is judged against. Beside +the reference, `CompositionRunner` checks every call without one: `Where(P)` and `Where(!P)` partition the +rows, and `Select(P)` is `true` exactly on `Where(P)`'s; for a value, the same with `== null`. A +declaration that disagrees with the server shows there even where memory has no answer. + ## How to use it **Run the oracles.** They are ordinary tests; `dotnet test` runs them all. The registered classes are diff --git a/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/Composition/CompositionDatabase.cs b/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/Composition/CompositionDatabase.cs new file mode 100644 index 00000000..8e771f9a --- /dev/null +++ b/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/Composition/CompositionDatabase.cs @@ -0,0 +1,27 @@ +using Microsoft.EntityFrameworkCore; +using Npgsql; + +namespace CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests.Composition; + +/// +/// A database of its own per composition sweep, so its nullable columns and joined tables never change +/// the shared schema and its rows never meet another test's. +/// +internal static class CompositionDatabase +{ + public static async Task Create(string prefix) + { + var name = $"{prefix}_{Guid.NewGuid():N}"; + await using (var connection = new NpgsqlConnection(ContainerLifecycle.ServerConnectionString)) + { + await connection.OpenAsync(); + await using var command = new NpgsqlCommand($"CREATE DATABASE {name}", connection); + await command.ExecuteNonQueryAsync(); + } + + return new NpgsqlConnectionStringBuilder(ContainerLifecycle.ServerConnectionString) { Database = name }.ConnectionString; + } + + public static void Configure(DbContextOptionsBuilder optionsBuilder, string connectionString) + => optionsBuilder.UseNpgsql(connectionString, npgsql => npgsql.UseValueRanges()); +} diff --git a/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/Composition/CompositionMembers.cs b/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/Composition/CompositionMembers.cs new file mode 100644 index 00000000..91c9f725 --- /dev/null +++ b/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/Composition/CompositionMembers.cs @@ -0,0 +1,171 @@ +using System.Linq.Expressions; +using System.Reflection; + +namespace CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests.Composition; + +/// An operand a call can be given: its readable name, the expression, and whether it can be NULL. +internal sealed record Operand(string Name, Expression Value, bool Nullable); + +/// One call of a library member with its operands spliced in, written as C#. +internal sealed record Call(string Name, Expression Body) +{ + public bool Predicate => Body.Type == typeof(bool); +} + +/// +/// The members of the library a query can call, discovered rather than listed, closed over one family: +/// , its and . The +/// composition sweeps vary how a call is composed (the join, the operands' nullability, the quantifier, +/// the execution, the aggregate), not the type; the type axis is the other sweeps'. +/// +/// +/// Every public method of , of (with its +/// operators) and of , plus the value set's Count and IsEmpty +/// properties. A member EF Core cannot translate is found by the sweep (it throws "could not be +/// translated") and counted, not judged. +/// +internal static class CompositionMembers +{ + public static readonly Type RangeType = typeof(DateRange); + public static readonly Type ElementType = typeof(DateOnly); + public static readonly Type SetType = typeof(RangeSet); + public static readonly Type ValueSetType = typeof(StringSet); + + private static readonly string[] NotQueryable = + [nameof(Equals), nameof(GetHashCode), nameof(ToString), "GetEnumerator", "CompareTo", "Parse", "TryParse", "Create", "From", "op_Equality", "op_Inequality"]; + + /// The members, closed over the family; a static one takes its receiver first. + public static IReadOnlyList Methods { get; } = Discover(); + + private static List Discover() + { + var methods = new List(); + + foreach (var (declaring, one, two) in new[] + { + (typeof(RangeExtensions), new[] { ElementType }, new[] { RangeType, ElementType }), + (typeof(ValueSetExtensions), new[] { typeof(string) }, new[] { ValueSetType, typeof(string) }) + }) + { + methods.AddRange(declaring.GetMethods(BindingFlags.Public | BindingFlags.Static) + .Select(method => Close(method, one, two)) + .OfType() + .Where(method => method.GetParameters().Length is 1 or 2)); + } + + methods.AddRange(SetType.GetMethods(BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static | BindingFlags.DeclaredOnly) + .Where(method => !method.IsSpecialName || method.Name.StartsWith("op_", StringComparison.Ordinal)) + .Where(method => method.GetParameters().Length <= (method.IsStatic ? 2 : 1))); + + return [.. methods.Where(method => !NotQueryable.Contains(method.Name)) + .Where(method => method.ReturnType != typeof(void)) + .OrderBy(method => method.DeclaringType!.Name, StringComparer.Ordinal) + .ThenBy(method => method.Name, StringComparer.Ordinal) + .ThenBy(method => string.Join(",", method.GetParameters().Select(p => p.ParameterType.Name)), StringComparer.Ordinal)]; + } + + private static MethodInfo? Close(MethodInfo method, Type[] one, Type[] two) + { + try + { + return method.GetGenericArguments().Length switch + { + 0 => method, + 1 => method.MakeGenericMethod(one), + 2 => method.MakeGenericMethod(two), + _ => null + }; + } + catch (ArgumentException) + { + return null; // a constraint the family does not meet + } + } + + /// + /// The parameters of in call order, the receiver first: the instance for + /// an instance method, the first argument otherwise. + /// + public static IReadOnlyList Slots(MethodInfo method) => + [.. method.IsStatic ? [] : new[] { method.DeclaringType! }, .. method.GetParameters().Select(p => p.ParameterType)]; + + /// + /// Every call of every member with the operands offers per slot, kept + /// when accepts its operands (typically: one of them can be NULL), plus the + /// value set's Count and IsEmpty on each operand the first slot of a value set takes. + /// + public static List Calls(Func> operands, Func, bool> keep) + { + var calls = new List(); + + foreach (var method in Methods) + { + var slots = Slots(method); + foreach (var combination in Product(slots.Select((type, index) => operands(type, index).ToList()).ToList())) + { + if (!keep(combination)) continue; + calls.Add(new Call(Name(method, combination), Build(method, combination.Select(o => o.Value).ToList()))); + } + } + + foreach (var receiver in operands(ValueSetType, 0).Where(o => keep([o]))) + { + calls.Add(new Call($"{receiver.Name}.Count", Expression.Property(receiver.Value, nameof(StringSet.Count)))); + calls.Add(new Call($"{receiver.Name}.IsEmpty", Expression.Property(receiver.Value, nameof(StringSet.IsEmpty)))); + } + + return calls; + } + + private static Expression Build(MethodInfo method, IReadOnlyList values) + { + if (method.IsSpecialName && method.Name.StartsWith("op_", StringComparison.Ordinal)) + { + var type = method.Name switch + { + "op_BitwiseOr" => ExpressionType.Or, + "op_BitwiseAnd" => ExpressionType.And, + "op_Subtraction" => ExpressionType.Subtract, + _ => throw new NotSupportedException(method.Name) + }; + return Expression.MakeBinary(type, values[0], values[1], false, method); + } + + return method.IsStatic + ? Expression.Call(method, values) + : Expression.Call(values[0], method, values.Skip(1)); + } + + private static string Name(MethodInfo method, IReadOnlyList operands) => + method.Name switch + { + "op_BitwiseOr" => $"{operands[0].Name} | {operands[1].Name}", + "op_BitwiseAnd" => $"{operands[0].Name} & {operands[1].Name}", + "op_Subtraction" => $"{operands[0].Name} - {operands[1].Name}", + _ => $"{operands[0].Name}.{method.Name}({string.Join(", ", operands.Skip(1).Select(o => o.Name))})" + }; + + private static IEnumerable> Product(IReadOnlyList> slots) + { + IEnumerable> result = [[]]; + foreach (var slot in slots) + result = result.SelectMany(prefix => slot.Select(operand => (IReadOnlyList)[.. prefix, operand])).ToList(); + return result; + } + + /// A captured variable, which EF Core sends as a parameter. + public static MemberExpression Captured(T value) => + Expression.Field(Expression.Constant(new System.Runtime.CompilerServices.StrongBox(value)), nameof(System.Runtime.CompilerServices.StrongBox<>.Value)); + + /// + /// Which family a parameter of type takes, if any: a range, a multirange, + /// a value set, or an element of either. + /// + public static string? Kind(Type parameter) => + parameter == ElementType ? "element" + : parameter == typeof(string) ? "text" + : parameter.IsAssignableFrom(RangeType) ? "range" + : parameter == SetType ? "set" + : parameter.IsAssignableFrom(ValueSetType) ? "values" + : null; +} diff --git a/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/Composition/CompositionRunner.cs b/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/Composition/CompositionRunner.cs new file mode 100644 index 00000000..2cfa6d5a --- /dev/null +++ b/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/Composition/CompositionRunner.cs @@ -0,0 +1,137 @@ +using System.Data.Common; +using System.Linq.Expressions; +using Microsoft.EntityFrameworkCore; + +namespace CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests.Composition; + +/// A row's id and one projected value, as the server returns them. +public sealed class Projected +{ + public int Id { get; set; } + + public object? Value { get; set; } +} + +/// What became of one call: translated and compared with the reference on at least one row, translated +/// with no row the reference could answer (only the reference-free checks ran), or not translated. +internal enum Outcome +{ + Compared, + Unanswered, + Untranslated +} + +/// +/// Runs one call in the query contexts a consumer puts it in, on the server and against +/// over the rows as loaded, and writes one line per row that disagrees. +/// +internal static class CompositionRunner +{ + /// + /// A predicate in Where(P), Where(!P) and Select(P); a value tested against + /// both ways and projected. Each form is compared with the reference, and the + /// three with each other, which needs no reference: the two filters partition the rows, and the + /// projection says which side each row is on. Each line starts with the call's name, then + /// , the query form and the row. A call EF Core cannot translate is counted + /// rather than judged. Without only the reference-free checks run. + /// + public static async Task Judge( + string context, Call call, ParameterExpression row, IQueryable source, IReadOnlyList loaded, + Expression> id, Func rowName, List failures, bool reference = true) + { + var idOf = id.Compile(); + + // A row whose reference throws has no answer to compare (a library member refusing a non-null + // operand), and is left out of every form. + var expected = new Dictionary(); + if (reference) + { + var answer = NullRule.Reference(call.Body, row); + foreach (var loadedRow in loaded) + { + try { expected[idOf(loadedRow)] = answer(loadedRow); } + catch (Exception e) when (e is ArgumentException or InvalidOperationException or OverflowException) { } + } + } + + var forms = call.Predicate + ? new (string Form, Expression Filter, Func Selected)[] + { + ("Where", call.Body, value => value is true), + ("Where(!)", Expression.Not(call.Body), value => value is false) + } + : [ + ("Where(== null)", IsNull(call.Body), value => value is null), + ("Where(!= null)", Expression.Not(IsNull(call.Body)), value => value is not null) + ]; + + try + { + var filtered = new List>(); + foreach (var (form, filter, selected) in forms) + { + var server = (await source.Where(Expression.Lambda>(filter, row)).Select(id).ToListAsync()).ToHashSet(); + filtered.Add(server); + foreach (var (rowId, value) in expected.OrderBy(pair => pair.Key)) + { + if (server.Contains(rowId) != selected(value)) + failures.Add($"{call.Name} [{context}] {form}, row {rowName(rowId)}: server {Selects(server.Contains(rowId))} it, rule (c) {Selects(selected(value))} it"); + } + } + + var projection = Expression.MemberInit( + Expression.New(typeof(Projected)), + Expression.Bind(typeof(Projected).GetProperty(nameof(Projected.Id))!, Splice(id, row)), + Expression.Bind(typeof(Projected).GetProperty(nameof(Projected.Value))!, + Expression.Convert(Expression.Convert(call.Body, NullRule.Lift(call.Body.Type)), typeof(object)))); + var projected = await source.Select(Expression.Lambda>(projection, row)).ToListAsync(); + foreach (var item in projected.OrderBy(p => p.Id)) + { + if (expected.TryGetValue(item.Id, out var value) && !Equals(item.Value, value)) + failures.Add($"{call.Name} [{context}] Select, row {rowName(item.Id)}: server {Show(item.Value)}, rule (c) {Show(value)}"); + + var (first, second) = (filtered[0].Contains(item.Id), filtered[1].Contains(item.Id)); + if (first == second || first != forms[0].Selected(item.Value)) + failures.Add($"{call.Name} [{context}] partition, row {rowName(item.Id)}: {forms[0].Form} {Selects(first)} it, " + + $"{forms[1].Form} {Selects(second)} it, Select {Show(item.Value)}"); + } + + return expected.Count > 0 || !reference ? Outcome.Compared : Outcome.Unanswered; + } + catch (InvalidOperationException e) when (e.Message.Contains("could not be translated", StringComparison.Ordinal)) + { + return Outcome.Untranslated; + } + catch (Exception e) when (e is InvalidOperationException or DbException or InvalidCastException) + { + failures.Add($"{call.Name} [{context}]: server threw {e.GetType().Name}: {FirstLine(e)}"); + return Outcome.Compared; + } + } + + /// == null, lifting a value type so the test can be written. + internal static Expression IsNull(Expression value) + { + var type = NullRule.Lift(value.Type); + return Expression.Equal(type == value.Type ? value : Expression.Convert(value, type), Expression.Constant(null, type)); + } + + internal static Expression Splice(Expression> lambda, ParameterExpression row) => + new Rebind(lambda.Parameters[0], row).Visit(lambda.Body); + + internal static string Show(object? value) => value switch + { + null => "NULL", + bool b => b ? "true" : "false", + _ => value.ToString() ?? "NULL" + }; + + internal static string FirstLine(Exception e) => e.Message.Split('\n')[0]; + + private static string Selects(bool selected) => selected ? "selects" : "drops"; + + private sealed class Rebind(ParameterExpression from, Expression to) : ExpressionVisitor + { + protected override Expression VisitParameter(ParameterExpression node) => node == from ? to : node; + } +} diff --git a/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/Composition/NullRule.cs b/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/Composition/NullRule.cs new file mode 100644 index 00000000..a4401789 --- /dev/null +++ b/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/Composition/NullRule.cs @@ -0,0 +1,202 @@ +using System.Linq.Expressions; +using System.Reflection; + +namespace CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests.Composition; + +/// +/// The answer a query should give for an expression over values that can be NULL, computed in memory: +/// the expression rewritten so that a NULL never reaches the library and is answered by rule (c) +/// instead (#350, #355, docs/efcore.md#query-ranges-with-linq). +/// +/// +/// +/// The rewrite is the null propagation EF Core applies to its own members, made explicit as the +/// guard a consumer writes over loaded rows: +/// +/// +/// A member read through a value that is (the child of an outer join, +/// a nullable column) is , lifted to for a value type. +/// A predicate of the library with a null on either side is , so +/// its negation is : rule (c). That holds for a value set predicate too, which +/// throws in memory for a null set by design. +/// Any other library member with a null operand, or with an element that is null, has no value: +/// . That is the 11.0.0 half of #350 for Intersect, Gap, +/// Merge, Union and Except, and #360's reading for a value set's Union. +/// The library is never called with a null, so what it does today for one is not judged here. +/// A comparison, a coalesce and a conditional over a lifted value follow C#'s lifted operators: +/// null > 0 is , null == null is . +/// Equality keeps its own rule (EF Core's, which is C#'s). CompareTo keeps +/// 's, under which a null sorts first (#220, #253, #256, #261, +/// docs/ranges.md#sort-ranges). The aggregates skip a null as SQL's do (#361). +/// +/// +internal sealed class NullRule : ExpressionVisitor +{ + private static readonly Assembly[] Library = [typeof(RangeExtensions).Assembly, typeof(LocalDateRange).Assembly]; + + /// The reference for , a lambda over , boxed. + public static Func Reference(Expression body, ParameterExpression row) => + Expression.Lambda>(Expression.Convert(new NullRule().Visit(body), typeof(object)), row).Compile(); + + /// The reference for , a lambda over , taking and returning a boxed value. + public static Func Reference(Expression body, ParameterExpression parameter) + { + var value = Expression.Parameter(typeof(object), "value"); + return Expression.Lambda>( + Expression.Block([parameter], + Expression.Assign(parameter, Expression.Convert(value, parameter.Type)), + Expression.Convert(new NullRule().Visit(body)!, typeof(object))), + value).Compile(); + } + + private static bool IsLibrary(MemberInfo member) => + member.DeclaringType is { } type && Library.Contains(type.Assembly) + && type != typeof(RangeAggregateExtensions) && !IsAggregates(type); + + // The satellite's aggregate overloads skip a null as the core's do (#361). + private static bool IsAggregates(Type type) => type.Name.EndsWith("AggregateExtensions", StringComparison.Ordinal); + + private static bool IsEquality(MethodInfo method) => method.Name is "op_Equality" or "op_Inequality" or nameof(Equals); + + /// The type a value of has once it can be missing. + internal static Type Lift(Type type) => + type.IsValueType && Nullable.GetUnderlyingType(type) is null ? typeof(Nullable<>).MakeGenericType(type) : type; + + private static Expression NoValue(Type type, bool libraryPredicate) => + libraryPredicate ? Expression.Constant(false) : Expression.Constant(null, Lift(type)); + + protected override Expression VisitMember(MemberExpression node) + { + // A static, a captured variable, or the row itself: present. + if (node.Expression is null or ConstantExpression or ParameterExpression) return node; + + var target = Visit(node.Expression)!; + var original = node.Expression.Type; + var lifted = target.Type != original; + + // .Value of a nullable column is the column in SQL, so a missing value stays missing rather than throwing. + if (node.Member.Name == nameof(Nullable<>.Value) && Nullable.GetUnderlyingType(original) is not null) return target; + + // A member of Nullable itself (HasValue, Value), or of a value that cannot be missing. + if (!lifted && original.IsValueType) return node.Update(target); + + var temp = Expression.Variable(target.Type, "value"); + var access = Expression.MakeMemberAccess(lifted ? Expression.Property(temp, nameof(Nullable<>.Value)) : temp, node.Member); + var isNull = lifted + ? (Expression)Expression.Not(Expression.Property(temp, nameof(Nullable<>.HasValue))) + : Expression.ReferenceEqual(temp, Expression.Constant(null, target.Type)); + + var predicate = IsLibrary(node.Member) && node.Type == typeof(bool); + var type = predicate ? typeof(bool) : Lift(node.Type); + return Expression.Block(type, [temp], + Expression.Assign(temp, target), + Expression.Condition(isNull, NoValue(node.Type, predicate), Expression.Convert(access, type))); + } + + protected override Expression VisitMethodCall(MethodCallExpression node) + { + var instance = node.Object is null ? null : Visit(node.Object); + var arguments = node.Arguments.Select(argument => Visit(argument)!).ToList(); + + if (node.Method.Name == nameof(IComparable<>.CompareTo) && IsLibrary(node.Method) && instance is not null) + return Expression.Call( + Expression.Property(null, typeof(Comparer<>).MakeGenericType(node.Object!.Type), nameof(Comparer<>.Default)), + nameof(Comparer<>.Compare), null, instance, arguments[0]); + + if (!IsLibrary(node.Method) || IsEquality(node.Method)) + { + if (instance?.Type != node.Object?.Type || arguments.Where((a, i) => a.Type != node.Arguments[i].Type).Any()) + throw new NotSupportedException($"A null reaches '{node.Method}', which NullRule does not answer for."); + return node.Update(instance, arguments); + } + + List<(Expression Value, Type Declared)> operands = [.. arguments.Select((a, i) => (a, node.Arguments[i].Type))]; + if (instance is not null) operands.Insert(0, (instance, node.Object!.Type)); + + return Guarded(node.Method, node.Type, operands, + values => node.Object is null + ? Expression.Call(node.Method, values) + : Expression.Call(values[0], node.Method, values.Skip(1))); + } + + protected override Expression VisitBinary(BinaryExpression node) + { + var left = Visit(node.Left)!; + var right = Visit(node.Right)!; + + // A library operator other than equality (RangeSet's |, & and -) is a value-producing member. + if (node.Method is { } method && IsLibrary(method) && !IsEquality(method)) + return Guarded(method, node.Type, [(left, node.Left.Type), (right, node.Right.Type)], + values => Expression.MakeBinary(node.NodeType, values[0], values[1], node.IsLiftedToNull, method)); + + if (left.Type == node.Left.Type && right.Type == node.Right.Type) return node.Update(left, node.Conversion, right); + + if (node.NodeType == ExpressionType.Coalesce) return Expression.Coalesce(left, right); + + // C#'s lifted operators: both sides nullable, a comparison answers bool. + var common = Lift(Nullable.GetUnderlyingType(left.Type) ?? Nullable.GetUnderlyingType(right.Type) ?? left.Type); + return Expression.MakeBinary(node.NodeType, Expression.Convert(left, common), Expression.Convert(right, common)); + } + + protected override Expression VisitUnary(UnaryExpression node) + { + var operand = Visit(node.Operand)!; + if (operand.Type == node.Operand.Type) return node.Update(operand); + + return node.NodeType switch + { + ExpressionType.Convert when node.Type == operand.Type => operand, + ExpressionType.Convert when node.Type.IsValueType && Nullable.GetUnderlyingType(node.Type) is null + => Expression.Convert(operand, Lift(node.Type)), + ExpressionType.Convert => Expression.Convert(operand, node.Type), + _ => throw new NotSupportedException($"A null reaches '{node}', which NullRule does not answer for.") + }; + } + + protected override Expression VisitConditional(ConditionalExpression node) + { + var test = Visit(node.Test)!; + var ifTrue = Visit(node.IfTrue)!; + var ifFalse = Visit(node.IfFalse)!; + if (ifTrue.Type == node.Type && ifFalse.Type == node.Type) return node.Update(test, ifTrue, ifFalse); + + var type = Lift(node.Type); + return Expression.Condition(test, Expression.Convert(ifTrue, type), Expression.Convert(ifFalse, type)); + } + + /// + /// A library member called only when every operand is present: otherwise for + /// a predicate and for anything else. + /// + private static Expression Guarded(MethodInfo method, Type returnType, IReadOnlyList<(Expression Value, Type Declared)> operands, + Func, Expression> call) + { + var temps = operands.Select((o, i) => Expression.Variable(o.Value.Type, $"operand{i}")).ToList(); + var checks = new List(); + var values = new List(); + + foreach (var ((value, declared), temp) in operands.Zip(temps)) + { + if (value.Type != declared) + { + checks.Add(Expression.Not(Expression.Property(temp, nameof(Nullable<>.HasValue)))); + values.Add(Expression.Property(temp, nameof(Nullable<>.Value))); + } + else + { + if (!declared.IsValueType) checks.Add(Expression.ReferenceEqual(temp, Expression.Constant(null, declared))); + values.Add(temp); + } + } + + var predicate = returnType == typeof(bool); + var type = predicate ? typeof(bool) : Lift(returnType); + var invoked = Expression.Convert(call(values), type); + Expression body = checks.Count == 0 + ? invoked + : Expression.Condition(checks.Aggregate(Expression.OrElse), NoValue(returnType, predicate), invoked); + + var steps = operands.Zip(temps).Select(p => (Expression)Expression.Assign(p.Second, p.First.Value)).Append(body); + return Expression.Block(type, temps, steps); + } +} diff --git a/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/DeclaredNullabilityTests.cs b/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/DeclaredNullabilityTests.cs new file mode 100644 index 00000000..5dc9d4c1 --- /dev/null +++ b/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/DeclaredNullabilityTests.cs @@ -0,0 +1,369 @@ +using System.Collections.Concurrent; +using System.Linq.Expressions; +using CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests.Composition; +using CodoMetis.ValueRanges.Oracles; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Query; +using Microsoft.EntityFrameworkCore.Query.SqlExpressions; +using Microsoft.Extensions.DependencyInjection; +using Npgsql; +using static CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests.Composition.CompositionMembers; + +namespace CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests; + +/// +/// The nullability each translated function declares to EF Core, against what PostgreSQL computes: +/// per function, and for the complete translated expression (#380, the static half of #362). +/// +/// +/// +/// A tells EF Core when its result can be NULL. With an argument +/// declared as propagating nullability, EF Core takes the result to be NULL exactly when a propagating +/// argument is NULL: it rewrites f(x) IS NULL to x IS NULL, folds it to FALSE for +/// a required column, and drops the COALESCE of a ??. #349 was lower declared that +/// way, though lower('empty') is NULL; #360 is array_cat, though +/// array_cat(NULL, '{a}') is {a}. +/// +/// +/// The first test records every function the translators emit for every discovered call (through a +/// recording decorator over the plugin's own SQL processor, so the plugin's chain stays as it is) and +/// checks each declaration against pg_proc.proisstrict and against the server, over typed NULLs +/// and samples that include empty, (,) and {}. The second checks the complete +/// expression a call becomes, CASE wrappers and arithmetic included: EF Core's own null test, +/// which follows the declarations, selects exactly the rows whose projected value is NULL. +/// +/// +[TestClass] +public sealed class DeclaredNullabilityTests +{ + private static DateOnly March(int day) => new(2026, 3, day); + + private static DateRange Days(int from, int to) => DateRange.CreateFinite(March(from), March(to), true, true); + + private static RangeSet Blocks(params DateRange[] ranges) => RangeSet.From(ranges); + + private static readonly ParameterExpression Row = Expression.Parameter(typeof(DeclaredRow), "r"); + + private static readonly Dictionary Names = new() + { + [1] = "null", [2] = "[1,10]", [3] = "empty", [4] = "(,)", [5] = "[20,)", [6] = "(,10]" + }; + + /// Non-NULL samples per store type, chosen to reach the NULLs a function returns for non-NULL input. + private static readonly Dictionary Samples = new() + { + ["daterange"] = ["'[2026-03-01,2026-03-10]'::daterange", "'empty'::daterange", "'(,)'::daterange", "'[2026-03-20,)'::daterange", "'(,2026-03-10]'::daterange"], + ["datemultirange"] = ["'{}'::datemultirange", "'{[2026-03-01,2026-03-10]}'::datemultirange", "'{(,)}'::datemultirange", "'{[2026-03-20,2026-03-25],[2026-03-27,)}'::datemultirange"], + ["date"] = ["DATE '2026-03-07'"], + ["text[]"] = ["'{}'::text[]", "'{a}'::text[]", "'{a,b}'::text[]"], + ["text"] = ["'a'::text"], + ["integer"] = ["1"] + }; + + private static async Task Database() + { + var connectionString = await CompositionDatabase.Create("declared_nullability"); + await using var seed = new DeclaredContext(connectionString, null); + await seed.Database.EnsureCreatedAsync(); + (DateRange? Range, RangeSet? Set, StringSet? Tags, DateOnly? Day, string? Label)[] values = + [ + (null, null, null, null, null), + (Days(1, 10), Blocks(Days(1, 10)), StringSet.From(["a"]), March(7), "a"), + (DateRange.Empty, Blocks(), StringSet.Empty, March(3), "b"), + (DateRange.Infinite, Blocks(DateRange.Infinite), StringSet.From(["a", "b"]), null, "a"), + (DateRange.CreateUnboundedEnd(March(20), true), Blocks(Days(20, 25), DateRange.CreateUnboundedEnd(March(27), true)), StringSet.From(["b"]), March(20), null), + (DateRange.CreateUnboundedStart(March(10), true), Blocks(DateRange.CreateUnboundedStart(March(10), true)), StringSet.From(["a"]), March(31), "c") + ]; + seed.Rows.AddRange(values.Select((v, index) => new DeclaredRow + { + Id = index + 1, Period = v.Range, Required = v.Range ?? DateRange.Empty, Blocks = v.Set, RequiredBlocks = v.Set ?? Blocks(), + Tags = v.Tags, RequiredTags = v.Tags ?? StringSet.Empty, Day = v.Day, Label = v.Label + })); + await seed.SaveChangesAsync(); + return connectionString; + } + + private static Expression Column(string name) => Expression.Property(Row, name); + + /// + /// The receiver is a nullable or a required column; the operand a captured value, a captured + /// , or for an element a nullable column read through .Value. + /// + private static Operand[] Slot(Type type, int position) => Kind(type) switch + { + "range" => position == 0 + ? [new("r.Period", Column(nameof(DeclaredRow.Period)), true), new("r.Required", Column(nameof(DeclaredRow.Required)), false)] + : [new("@[3,8]", Captured(Days(3, 8)), false), new("@null", Captured(null!), true)], + "set" => position == 0 + ? [new("r.Blocks", Column(nameof(DeclaredRow.Blocks)), true), new("r.RequiredBlocks", Column(nameof(DeclaredRow.RequiredBlocks)), false)] + : [new("@{[3,8]}", Captured(Blocks(Days(3, 8))), false), new("@null", Captured>(null!), true)], + "values" => position == 0 + ? [new("r.Tags", Column(nameof(DeclaredRow.Tags)), true), new("r.RequiredTags", Column(nameof(DeclaredRow.RequiredTags)), false)] + : [new("@{a}", Captured(StringSet.From(["a"])), false), new("@null", Captured(null!), true)], + "element" => [new("r.Day.Value", Expression.Property(Column(nameof(DeclaredRow.Day)), nameof(Nullable<>.Value)), true), new("@2026-03-07", Captured(March(7)), false)], + "text" => [new("r.Label", Column(nameof(DeclaredRow.Label)), true), new("@a", Captured("a"), false)], + _ => [] + }; + + /// Every discovered call, the range factories over the nullable element column, and the two aggregates. + private static List Discovered() + { + var calls = Calls(Slot, _ => true); + var day = Expression.Property(Column(nameof(DeclaredRow.Day)), nameof(Nullable<>.Value)); + calls.Add(new Call("DateRange.CreateFinite(@2026-03-01, r.Day.Value)", + Expression.Call(typeof(DateRange), nameof(DateRange.CreateFinite), null, Captured(March(1)), day, Expression.Constant(true), Expression.Constant(true)))); + calls.Add(new Call("DateRange.CreateUnboundedStart(r.Day.Value)", + Expression.Call(typeof(DateRange), nameof(DateRange.CreateUnboundedStart), null, day, Expression.Constant(true)))); + calls.Add(new Call("DateRange.CreateUnboundedEnd(r.Day.Value)", + Expression.Call(typeof(DateRange), nameof(DateRange.CreateUnboundedEnd), null, day, Expression.Constant(true)))); + return calls; + } + + /// + /// Every function the translators emit, declared as PostgreSQL evaluates it: an argument declared as + /// propagating nullability belongs to a strict function, and the result is NULL exactly when a + /// propagating argument is NULL. + /// + [TestMethod] + public async Task EveryEmittedFunction_IsDeclaredAsPostgresEvaluatesIt() + { + ContainerLifecycle.RequireServer(); + + var connectionString = await Database(); + var recorder = new ConcurrentQueue(); + await using var context = new DeclaredContext(connectionString, recorder); + + // Translate every call as a projection and the aggregates per group, and record the SQL trees. + // ToQueryString runs the processor without executing, so a member EF Core evaluates on the + // client never runs over a NULL. + foreach (var call in Discovered()) + { + var projection = Expression.Lambda>(Expression.Convert(call.Body, typeof(object)), Row); + try { _ = context.Rows.Select(projection).ToQueryString(); } + catch (InvalidOperationException) { } + } + + _ = context.Rows.GroupBy(r => r.Id % 2).Select(g => new { A = g.Select(r => r.Period!).RangeAgg(), B = g.Select(r => r.Period!).RangeIntersectAgg(), + C = g.Select(r => r.Blocks!).RangeAgg(), D = g.Select(r => r.Blocks!).RangeIntersectAgg() }) + .ToQueryString(); + + var functions = new Dictionary(); + foreach (var tree in recorder) + new FunctionCollector(functions).Visit(tree); + + var failures = new List(); + await using var connection = new NpgsqlConnection(connectionString); + await connection.OpenAsync(); + + foreach (var function in functions.Values.OrderBy(f => f.Signature, StringComparer.Ordinal)) + { + var catalogue = await Strictness(connection, function); + if (catalogue is null) + { + failures.Add($"{function.Signature}: no pg_proc entry takes these argument types"); + continue; + } + + if (function.Propagating.Count == 0 || catalogue.Value.Aggregate) continue; + + if (!catalogue.Value.Strict) + failures.Add($"{function.Signature}: declared NULL for a NULL argument {string.Join(" or ", function.Propagating)}, but pg_proc.proisstrict is false"); + + // Every argument NULL or a sample, constants kept: NULL exactly when a propagating one is NULL. + foreach (var arguments in Product(function.Arguments.Select((a, i) => a.Constant is { } literal ? [literal] : (string[])["NULL", .. Samples[a.StoreType]]).ToList())) + { + var expectNull = function.Propagating.Any(i => arguments[i] == "NULL"); + var sql = $"{function.Name}({string.Join(", ", arguments.Select((a, i) => a == "NULL" ? $"NULL::{function.Arguments[i].StoreType}" : a))})"; + await using var command = new NpgsqlCommand($"SELECT ({sql})::text", connection); + var result = await command.ExecuteScalarAsync(); + if ((result is null or DBNull) != expectNull) + failures.Add($"{function.Signature}: declared {(expectNull ? "NULL" : "not NULL")} for {sql}, which is {(result is string text ? $"'{text}'" : "NULL")}"); + } + } + + // Fourteen today: isempty, lower_inf, upper_inf, lower_inc, lower, upper, range_merge, cardinality, + // array_remove, array_cat, the range and multirange constructors and the two aggregates. + Assert.IsGreaterThanOrEqualTo(12, functions.Select(f => f.Value.Name).Distinct().Count(), + $"Only {string.Join(", ", functions.Values.Select(f => f.Name).Distinct())} were recorded"); + + Candidates.Verdict(nameof(DeclaredNullabilityTests), failures, + "A translated function's declared nullability disagrees with PostgreSQL", + method: nameof(EveryEmittedFunction_IsDeclaredAsPostgresEvaluatesIt)); + } + + /// + /// For every value a discovered call produces, over a nullable and a required column and a captured + /// value or : Where(v == null) and Where(v != null), which EF Core + /// rewrites from the declarations, select exactly the rows Select(v) reads as NULL and not NULL. + /// + [TestMethod] + public async Task EveryTranslatedValue_IsNullInAFilterExactlyWhereItIsNullInAProjection() + { + ContainerLifecycle.RequireServer(); + + var connectionString = await Database(); + await using var context = new DeclaredContext(connectionString, null); + var loaded = await context.Rows.AsNoTracking().OrderBy(r => r.Id).ToListAsync(); + + var failures = new List(); + var judged = 0; + foreach (var call in Discovered().Where(c => !c.Predicate)) + { + if (await CompositionRunner.Judge("declared", call, Row, context.Rows.AsNoTracking(), loaded, r => r.Id, id => Names[id], failures, + reference: false) == Outcome.Compared) + judged++; + } + + + Assert.IsGreaterThanOrEqualTo(80, judged, $"Only {judged} values were judged"); + + Candidates.Verdict(nameof(DeclaredNullabilityTests), failures, + "A translated value is NULL in a filter and not in a projection, or the other way", + method: nameof(EveryTranslatedValue_IsNullInAFilterExactlyWhereItIsNullInAProjection)); + } + + private static async Task<(bool Strict, bool Aggregate)?> Strictness(NpgsqlConnection connection, Declared function) + { + // The overloads of that name and arity, an exact argument type preferred over a polymorphic one. + await using var command = new NpgsqlCommand( + """ + SELECT p.proisstrict, p.prokind = 'a', p.proargtypes::regtype[]::text[] + FROM pg_proc p + WHERE p.proname = $1 AND p.pronargs = $2 AND p.provariadic = 0 + """, connection); + command.Parameters.Add(new NpgsqlParameter { Value = function.Name }); + command.Parameters.Add(new NpgsqlParameter { Value = (short)function.Arguments.Count }); + + var candidates = new List<(bool Strict, bool Aggregate, int Exact)>(); + await using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + var types = reader.GetFieldValue(2); + var exact = 0; + var fits = true; + for (var i = 0; i < types.Length; i++) + { + if (types[i] == function.Arguments[i].StoreType) exact++; + else if (!types[i].StartsWith("any", StringComparison.Ordinal)) fits = false; + } + + if (fits) candidates.Add((reader.GetBoolean(0), reader.GetBoolean(1), exact)); + } + + return candidates.Count == 0 ? null : candidates.OrderByDescending(c => c.Exact).Select(c => ((bool, bool)?)(c.Strict, c.Aggregate)).First(); + } + + private static IEnumerable Product(IReadOnlyList slots) + { + IEnumerable result = [[]]; + foreach (var slot in slots) result = result.SelectMany(prefix => slot.Select(value => (string[])[.. prefix, value])).ToList(); + return result; + } + + /// One argument of a recorded function: its store type, and its literal when it is a constant. + private sealed record Argument(string StoreType, string? Constant); + + /// A function as the translators declare it. + private sealed record Declared(string Name, IReadOnlyList Arguments, IReadOnlyList Propagating, bool Nullable) + { + public string Signature => $"{Name}({string.Join(", ", Arguments.Select(a => a.Constant ?? a.StoreType))})"; + } + + /// Every in a SQL tree, by signature and declaration. + private sealed class FunctionCollector(Dictionary functions) : ExpressionVisitor + { + protected override Expression VisitExtension(Expression node) + { + if (node is SqlFunctionExpression { IsBuiltIn: true, Arguments: { } arguments } function + && function.Name != "COALESCE" + && arguments.All(a => a.TypeMapping is not null)) + { + var declared = new Declared( + function.Name, + [.. arguments.Select(a => new Argument(a.TypeMapping!.StoreType, + a is SqlConstantExpression { Value: { } value } ? a.TypeMapping.GenerateSqlLiteral(value) : null))], + [.. (function.ArgumentsPropagateNullability ?? []).Select((p, i) => (p, i)).Where(x => x.p).Select(x => x.i)], + function.IsNullable); + functions.TryAdd($"{declared.Signature} [{string.Join(",", declared.Propagating)}]", declared); + } + + return base.VisitExtension(node); + } + } + + public sealed class DeclaredRow + { + public int Id { get; set; } + + public DateRange? Period { get; set; } + + public DateRange Required { get; set; } = DateRange.Empty; + + public RangeSet? Blocks { get; set; } + + public RangeSet RequiredBlocks { get; set; } = RangeSet.Empty; + + public StringSet? Tags { get; set; } + + public StringSet RequiredTags { get; set; } = StringSet.Empty; + + public DateOnly? Day { get; set; } + + public string? Label { get; set; } + } + + /// + /// With a recorder, a context whose internal service provider wraps the plugin's SQL processor in one + /// that records each final SQL tree; the plugin's own decorator stays in the chain, so the guard and + /// the empty-collection rewrite run as in any other context. + /// + private sealed class DeclaredContext(string connectionString, ConcurrentQueue? recorder) : DbContext + { + private static readonly ConcurrentDictionary, IServiceProvider> Providers = new(); + + public DbSet Rows => Set(); + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + { + if (recorder is not null) optionsBuilder.UseInternalServiceProvider(Providers.GetOrAdd(recorder, Recording)); + CompositionDatabase.Configure(optionsBuilder, connectionString); + } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + => modelBuilder.Entity().Property(r => r.Id).ValueGeneratedNever(); + + private static IServiceProvider Recording(ConcurrentQueue recorder) + { + var services = new ServiceCollection().AddEntityFrameworkNpgsql().AddEntityFrameworkNpgsqlValueRanges(); + var plugin = services.Last(d => d.ServiceType == typeof(IRelationalParameterBasedSqlProcessorFactory)); + services.Remove(plugin); + services.Add(new ServiceDescriptor( + typeof(IRelationalParameterBasedSqlProcessorFactory), + provider => new RecordingFactory((IRelationalParameterBasedSqlProcessorFactory)plugin.ImplementationFactory!(provider), + provider.GetRequiredService(), recorder), + plugin.Lifetime)); + return services.BuildServiceProvider(); + } + } + + private sealed class RecordingFactory( + IRelationalParameterBasedSqlProcessorFactory inner, RelationalParameterBasedSqlProcessorDependencies dependencies, ConcurrentQueue recorder) + : IRelationalParameterBasedSqlProcessorFactory + { + public RelationalParameterBasedSqlProcessor Create(RelationalParameterBasedSqlProcessorParameters parameters) + => new Recording(inner.Create(parameters), dependencies, parameters, recorder); + + private sealed class Recording( + RelationalParameterBasedSqlProcessor inner, RelationalParameterBasedSqlProcessorDependencies dependencies, + RelationalParameterBasedSqlProcessorParameters parameters, ConcurrentQueue recorder) + : RelationalParameterBasedSqlProcessor(dependencies, parameters) + { + public override Expression Process(Expression queryExpression, ParametersCacheDecorator parametersDecorator) + { + var processed = inner.Process(queryExpression, parametersDecorator); + recorder.Enqueue(processed); + return processed; + } + } + } +} diff --git a/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/FilteredAggregateProbeTests.cs b/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/FilteredAggregateProbeTests.cs new file mode 100644 index 00000000..da33eb11 --- /dev/null +++ b/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/FilteredAggregateProbeTests.cs @@ -0,0 +1,332 @@ +using System.Data.Common; +using System.Linq.Expressions; +using System.Reflection; +using CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests.Composition; +using CodoMetis.ValueRanges.Oracles; +using Microsoft.EntityFrameworkCore; +using static CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests.Composition.CompositionMembers; + +namespace CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests; + +/// +/// RangeAgg and RangeIntersectAgg over groups whose inputs a filter can remove, and the +/// aggregate then consumed: by every discovered member on its result, ??, a null test, a group +/// filter (HAVING) and an ordering, against memory over the rows as loaded (#379). +/// +/// +/// +/// RangeAggregateParityTests compares the aggregates per group, projected. Here a group can +/// exist while its filter (g.Where(…), a FILTER clause) removes every input, hold only +/// NULLs, hold an empty range, or be absent (a key left-joined to no rows); and +/// the result is consumed rather than projected. The groups, by key: 1 two finite ranges; 2 two ranges +/// the filter removes; 3 two NULLs; 4 empty and a finite range; 5 (,) and a range the +/// filter removes; 6 a NULL and a finite range; 7 only empty; 8 no rows. +/// +/// +/// The reference is the in-memory aggregate over the same inputs, which skips a NULL as SQL does +/// (#361), with the documented difference mapped rather than reported: over zero inputs, or only NULLs, +/// SQL answers NULL where the in-memory RangeAgg() answers the empty set (claim E07, +/// docs/efcore.md#read-bounds-merge-ranges-and-aggregate), so the expected server answer is the +/// consumption of NULL there. An ordering by a bound puts the groups without it where PostgreSQL puts +/// NULLs (claim E72). +/// +/// +[TestClass] +public sealed class FilteredAggregateProbeTests +{ + private static DateOnly March(int day) => new(2026, 3, day); + + private static DateRange Days(int from, int to) => DateRange.CreateFinite(March(from), March(to), true, true); + + private static readonly Dictionary Names = new() + { + [1] = "two finite", [2] = "all filtered away", [3] = "only NULLs", [4] = "empty and finite", [5] = "(,) and filtered", + [6] = "NULL and finite", [7] = "only empty", [8] = "no rows" + }; + + [TestMethod] + public async Task EveryConsumption_OfAFilteredAggregate_AnswersAsMemory() + { + ContainerLifecycle.RequireServer(); + + (int Key, DateRange? Period, int Weight)[] inputs = + [ + (1, Days(1, 10), 1), (1, Days(5, 15), 1), + (2, Days(1, 10), 0), (2, Days(20, 25), -1), + (3, null, 1), (3, null, 1), + (4, DateRange.Empty, 1), (4, Days(3, 8), 1), + (5, DateRange.Infinite, 1), (5, Days(5, 8), 0), + (6, null, 1), (6, Days(3, 8), 1), + (7, DateRange.Empty, 1) + ]; + + var connectionString = await CompositionDatabase.Create("filtered_aggregate"); + await using (var seed = new AggregateContext(connectionString)) + { + await seed.Database.EnsureCreatedAsync(); + seed.Keys.AddRange(Names.Keys.Select(key => new AggregateKey { Id = key })); + seed.Rows.AddRange(inputs.Select((input, index) => new AggregateRow + { + Id = index + 1, GroupKey = input.Key, Weight = input.Weight, Period = input.Period, + Blocks = input.Period is null ? null : RangeSet.From(input.Period) + })); + await seed.SaveChangesAsync(); + } + + await using var context = new AggregateContext(connectionString); + var loaded = await context.Rows.AsNoTracking().ToListAsync(); + + var row = Expression.Parameter(typeof(AggregateRow), "r"); + var weighed = Expression.Lambda>(Expression.GreaterThan(Expression.Property(row, nameof(AggregateRow.Weight)), Expression.Constant(0)), row); + + var failures = new List(); + var untranslated = new HashSet(); + var judged = 0; + + foreach (var aggregate in Aggregates()) + foreach (var (filterName, filter) in new (string, Expression>?)[] { ("", null), (".Where(r => r.Weight > 0)", weighed) }) + { + var label = $"g{filterName}.Select(r => r.{aggregate.Column}).{aggregate.Method.Name}()"; + + // The aggregate per key in memory, with E07 mapped: zero non-NULL inputs is NULL. + var kept = filter?.Compile() ?? (_ => true); + var memory = Names.Keys.ToDictionary(key => key, key => + { + var group = loaded.Where(r => r.GroupKey == key && kept(r)).Select(aggregate.Read).ToList(); + return group.All(value => value is null) ? null : aggregate.Memory(group); + }); + + foreach (var consumption in Consumptions(aggregate.ResultType)) + { + var name = $"{label}{consumption.Name}"; + var reference = NullRule.Reference(consumption.Body, consumption.Parameter); + var expected = Names.Keys.ToDictionary(key => key, key => Answer(reference, memory[key])); + + try + { + // The consumption as a group filter, HAVING, both ways: a predicate and its negation, a + // value tested against null. First, so a member EF Core would evaluate on the client in + // the projection is found untranslated here instead. + var predicate = consumption.Body.Type == typeof(bool); + var groups = (await context.Rows.Select(r => r.GroupKey).Distinct().ToListAsync()).Order().ToList(); + foreach (var (form, negate) in predicate ? new[] { ("HAVING", false), ("HAVING NOT", true) } : [("HAVING (== null)", false), ("HAVING (!= null)", true)]) + { + var having = await context.Rows.GroupBy(r => r.GroupKey).Where(Having(aggregate, filter, consumption, negate)).Select(g => g.Key).ToListAsync(); + foreach (var key in groups.Where(key => expected[key].Known)) + { + var keeps = (predicate ? expected[key].Value is true : expected[key].Value is null) != negate; + if (having.Contains(key) != keeps) + failures.Add($"{name} [GroupBy] {form}, group {Names[key]}: server {(having.Contains(key) ? "keeps" : "drops")} it, memory {(keeps ? "keeps" : "drops")} it"); + } + } + + // GroupBy, then the consumption projected: the groups that exist. + var grouped = await context.Rows.GroupBy(r => r.GroupKey) + .Select(Projection>(g => g.Key, aggregate, filter, consumption)) + .ToListAsync(); + foreach (var item in grouped.OrderBy(p => p.Id).Where(p => !Equals(p.Value, expected[p.Id].Value) && expected[p.Id].Known)) + failures.Add($"{name} [GroupBy] Select, group {Names[item.Id]}: server {CompositionRunner.Show(item.Value)}, memory {CompositionRunner.Show(expected[item.Id].Value)}"); + + // Every key left-joined to its rows and grouped, which reaches the key without rows too: + // its group holds the one NULL row the join makes. + var joined = await context.Keys.LeftJoin(context.Rows, k => k.Id, r => r.GroupKey, (k, r) => new KeyedRow { Key = k.Id, Row = r! }) + .GroupBy(x => x.Key, x => x.Row) + .Select(Projection>(g => g.Key, aggregate, filter, consumption)) + .ToListAsync(); + foreach (var item in joined.OrderBy(p => p.Id).Where(p => !Equals(p.Value, expected[p.Id].Value) && expected[p.Id].Known)) + failures.Add($"{name} [LeftJoin GroupBy] Select, key {Names[item.Id]}: server {CompositionRunner.Show(item.Value)}, memory {CompositionRunner.Show(expected[item.Id].Value)}"); + + if (consumption.Name is ".LowerBound()" or ".UpperBound()") + await Order(context, name, aggregate, filter, consumption, expected, grouped.Select(p => p.Id).ToList(), failures); + + judged++; + } + catch (InvalidOperationException e) when (e.Message.Contains("could not be translated", StringComparison.Ordinal)) + { + untranslated.Add(name); + } + catch (Exception e) when (e is InvalidOperationException or DbException) + { + failures.Add($"{name}: server threw {e.GetType().Name}: {CompositionRunner.FirstLine(e)}"); + } + } + } + + + // Some 300 consumptions are judged today; the floor catches a discovery that stopped matching. + Assert.IsGreaterThanOrEqualTo(200, judged, $"Only {judged} consumptions were judged; untranslated: {string.Join("; ", untranslated.Order())}"); + + Candidates.Verdict(nameof(FilteredAggregateProbeTests), failures, + "A filtered aggregate, consumed: PostgreSQL and memory (with E07 mapped) disagree", + method: nameof(EveryConsumption_OfAFilteredAggregate_AnswersAsMemory)); + } + + /// + /// OrderBy and OrderByDescending of the groups by the bound, then the key: memory's + /// order with the groups whose bound is NULL moved where PostgreSQL puts NULLs (E72). + /// + private static async Task Order(AggregateContext context, string name, Aggregate aggregate, Expression>? filter, + Consumption consumption, Dictionary expected, List groups, List failures) + { + var g = Expression.Parameter(typeof(IGrouping), "g"); + var key = Expression.Lambda, DateOnly?>>(consumption.Splice(aggregate.Over(g, filter)), g); + + foreach (var descending in new[] { false, true }) + { + var ordered = context.Rows.GroupBy(r => r.GroupKey); + var server = await (descending ? ordered.OrderByDescending(key) : ordered.OrderBy(key)).ThenBy(x => x.Key).Select(x => x.Key).ToListAsync(); + + var present = groups.Where(id => expected[id].Value is not null).OrderBy(id => (DateOnly?)expected[id].Value).ThenBy(id => id).ToList(); + if (descending) present = [.. groups.Where(id => expected[id].Value is not null).OrderByDescending(id => (DateOnly?)expected[id].Value).ThenBy(id => id)]; + var missing = groups.Where(id => expected[id].Value is null).Order().ToList(); + List documented = descending ? [.. missing, .. present] : [.. present, .. missing]; + + if (!server.SequenceEqual(documented)) + failures.Add($"{name} [GroupBy] {(descending ? "OrderByDescending" : "OrderBy")}: server [{string.Join(",", server)}], documented [{string.Join(",", documented)}]"); + } + } + + private static (bool Known, object? Value) Answer(Func reference, object? aggregate) + { + try { return (true, reference(aggregate)); } + catch (Exception e) when (e is ArgumentException or InvalidOperationException) { return (false, null); } + } + + /// new Projected { Id = key, Value = consume(aggregate) } over a group. + private static Expression> Projection(Expression> key, Aggregate aggregate, + Expression>? filter, Consumption consumption) + { + var g = key.Parameters[0]; + return Expression.Lambda>(Projected(key.Body, consumption.Splice(aggregate.Over(g, filter))), g); + } + + /// The consumption as a group filter: a predicate itself, a value tested against null; negated when . + private static Expression, bool>> Having(Aggregate aggregate, Expression>? filter, + Consumption consumption, bool negate) + { + var g = Expression.Parameter(typeof(IGrouping), "g"); + var value = consumption.Splice(aggregate.Over(g, filter)); + var body = value.Type == typeof(bool) ? value : CompositionRunner.IsNull(value); + return Expression.Lambda, bool>>(negate ? Expression.Not(body) : body, g); + } + + private static MemberInitExpression Projected(Expression id, Expression value) => + Expression.MemberInit(Expression.New(typeof(Projected)), + Expression.Bind(typeof(Projected).GetProperty(nameof(Composition.Projected.Id))!, id), + Expression.Bind(typeof(Projected).GetProperty(nameof(Composition.Projected.Value))!, + Expression.Convert(Expression.Convert(value, NullRule.Lift(value.Type)), typeof(object)))); + + /// One aggregate: the column it reads, the method, its result type, and the same aggregate in memory. + private sealed record Aggregate(string Column, MethodInfo Method, Type ResultType, Func Read, Func, object?> Memory) + { + /// source[.Where(filter)].Select(r => r.Column).Aggregate(), over a group or a query. + public Expression Over(Expression source, Expression>? filter) + { + var queryable = typeof(IQueryable).IsAssignableFrom(source.Type); + var linq = queryable ? typeof(Queryable) : typeof(Enumerable); + Expression Lambda(LambdaExpression lambda) => queryable ? Expression.Quote(lambda) : lambda; + + if (filter is not null) source = Expression.Call(linq, nameof(Enumerable.Where), [typeof(AggregateRow)], source, Lambda(filter)); + var r = Expression.Parameter(typeof(AggregateRow), "r"); + var column = Expression.Property(r, Column); + var select = Expression.Call(linq, nameof(Enumerable.Select), [typeof(AggregateRow), column.Type], source, Lambda(Expression.Lambda(column, r))); + return Expression.Call(Method, select); + } + } + + private static IEnumerable Aggregates() + { + var aggregates = typeof(RangeAggregateExtensions).GetMethods(BindingFlags.Public | BindingFlags.Static); + MethodInfo Over(string name, Type input) => aggregates.Single(m => m.Name == name && !m.IsGenericMethodDefinition + && m.GetParameters()[0].ParameterType == typeof(IEnumerable<>).MakeGenericType(input)); + MethodInfo OverSets(string name) => aggregates.Single(m => m.Name == name && m.IsGenericMethodDefinition).MakeGenericMethod(RangeType, ElementType); + + yield return new(nameof(AggregateRow.Period), Over(nameof(RangeAggregateExtensions.RangeAgg), RangeType), SetType, + r => r.Period, values => values.Cast().RangeAgg()); + yield return new(nameof(AggregateRow.Period), Over(nameof(RangeAggregateExtensions.RangeIntersectAgg), RangeType), RangeType, + r => r.Period, values => values.Cast().RangeIntersectAgg()); + yield return new(nameof(AggregateRow.Blocks), OverSets(nameof(RangeAggregateExtensions.RangeAgg)), SetType, + r => r.Blocks, values => values.Cast>().RangeAgg()); + yield return new(nameof(AggregateRow.Blocks), OverSets(nameof(RangeAggregateExtensions.RangeIntersectAgg)), SetType, + r => r.Blocks, values => values.Cast>().RangeIntersectAgg()); + } + + /// + /// What a query does with an aggregate's result: the value itself, a null test, ?? (,), and + /// every discovered member with the result as its receiver and a captured operand. + /// + private sealed record Consumption(string Name, ParameterExpression Parameter, Expression Body) + { + public Expression Splice(Expression aggregate) => new Rebind(Parameter, aggregate).Visit(Body)!; + + private sealed class Rebind(ParameterExpression from, Expression to) : ExpressionVisitor + { + protected override Expression VisitParameter(ParameterExpression node) => node == from ? to : node; + } + } + + private static IEnumerable Consumptions(Type result) + { + var a = Expression.Parameter(result, "a"); + var infinite = result == RangeType ? (Expression)Captured(DateRange.Infinite) : Captured(RangeSet.Infinite); + + yield return new("", a, a); + yield return new(" == null", a, Expression.Equal(a, Expression.Constant(null, result))); + yield return new(" ?? @(,)", a, Expression.Coalesce(a, infinite)); + + Operand[] Slot(Type type, int position) => position == 0 + ? type.IsAssignableFrom(result) && Kind(type) is "range" or "set" ? [new("", a, true)] : [] + : Kind(type) switch + { + "range" => [new("@[3,8]", Captured(Days(3, 8)), false)], + "set" => [new("@{[3,8]}", Captured(RangeSet.From(Days(3, 8))), false)], + "element" => [new("@2026-03-07", Captured(March(7)), false)], + _ => [] + }; + + foreach (var call in Calls(Slot, operands => operands[0].Value == a && operands.Count <= 2)) + yield return new(call.Name, a, call.Body); + } + + public sealed class AggregateKey + { + public int Id { get; set; } + } + + /// A key and one of its rows, or no row when the key has none. + public sealed class KeyedRow + { + public int Key { get; set; } + + public AggregateRow Row { get; set; } = null!; + } + + public sealed class AggregateRow + { + public int Id { get; set; } + + public int GroupKey { get; set; } + + public int Weight { get; set; } + + public DateRange? Period { get; set; } + + public RangeSet? Blocks { get; set; } + } + + private sealed class AggregateContext(string connectionString) : DbContext + { + public DbSet Keys => Set(); + + public DbSet Rows => Set(); + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => CompositionDatabase.Configure(optionsBuilder, connectionString); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity().Property(k => k.Id).ValueGeneratedNever(); + modelBuilder.Entity().Property(r => r.Id).ValueGeneratedNever(); + } + } +} diff --git a/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/NullableOperandPairProbeTests.cs b/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/NullableOperandPairProbeTests.cs new file mode 100644 index 00000000..8ed580a9 --- /dev/null +++ b/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/NullableOperandPairProbeTests.cs @@ -0,0 +1,186 @@ +using System.Linq.Expressions; +using CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests.Composition; +using CodoMetis.ValueRanges.Oracles; +using Microsoft.EntityFrameworkCore; +using static CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests.Composition.CompositionMembers; + +namespace CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests; + +/// +/// Every binary range, and value set member with both operands +/// able to be NULL: two nullable columns, a conditional over them (r.Flag ? r.A : r.B, a +/// CASE) and a coalesce of them (r.A ?? r.B, a COALESCE), over every pair of +/// (null, null), (null, value), (value, null) and (value, value), against over the +/// rows as loaded (#376). +/// +/// +/// +/// The nullable-column sweep (#355) pairs one nullable column with a non-null captured operand. Here EF +/// Core has to compensate for two operands, and for operands whose nullability it infers from an +/// expression rather than reads from a column. The values are a finite range, empty, (,) +/// and an unbounded one, each in a row with every other and with NULL; the element operand is a nullable +/// DateOnly column read through .Value. +/// +/// +/// Equality and CompareTo are swept beside the members, each with its own documented rule: EF +/// Core's equality, under which NULL equals NULL, and 's order, under +/// which a NULL sorts first (#261, docs/ranges.md#sort-ranges). +/// +/// +[TestClass] +public sealed class NullableOperandPairProbeTests +{ + private static DateOnly March(int day) => new(2026, 3, day); + + private static DateRange Days(int from, int to) => DateRange.CreateFinite(March(from), March(to), true, true); + + private static RangeSet Blocks(params DateRange[] ranges) => RangeSet.From(ranges); + + [TestMethod] + public async Task EveryBinaryMember_OverTwoNullableOperands_AnswersAsRuleC() + { + ContainerLifecycle.RequireServer(); + + (string Name, DateRange? Range, RangeSet? Set, StringSet? Tags)[] values = + [ + ("null", null, null, null), + ("[1,10]", Days(1, 10), Blocks(Days(1, 10)), StringSet.From(["a"])), + ("empty", DateRange.Empty, Blocks(), StringSet.Empty), + ("(,)", DateRange.Infinite, Blocks(DateRange.Infinite), StringSet.From(["a", "b"])), + ("[20,)", DateRange.CreateUnboundedEnd(March(20), true), Blocks(Days(20, 25), DateRange.CreateUnboundedEnd(March(27), true)), StringSet.From(["b"])) + ]; + DateOnly?[] days = [null, March(3), March(7), March(20), March(31)]; + string?[] labels = [null, "a", "b", "c", "a"]; + + var connectionString = await CompositionDatabase.Create("operand_pair"); + var names = new Dictionary(); + await using (var seed = new PairContext(connectionString)) + { + await seed.Database.EnsureCreatedAsync(); + for (var a = 0; a < values.Length; a++) + for (var b = 0; b < values.Length; b++) + { + var id = a * values.Length + b + 1; + names[id] = $"({values[a].Name}, {values[b].Name})"; + seed.Rows.Add(new PairRow + { + Id = id, Flag = (a + b) % 2 == 0, Day = days[(a + 2 * b) % days.Length], Label = labels[(2 * a + b) % labels.Length], + A = values[a].Range, B = values[b].Range, SetA = values[a].Set, SetB = values[b].Set, TagsA = values[a].Tags, TagsB = values[b].Tags + }); + } + + await seed.SaveChangesAsync(); + } + + var row = Expression.Parameter(typeof(PairRow), "r"); + Expression Column(string name) => Expression.Property(row, name); + var flag = Column(nameof(PairRow.Flag)); + + // Per family the two columns, the conditional between them and their coalesce, on either side. + Operand[] Forms(string a, string b) => + [ + new($"r.{a}", Column(a), true), + new($"(r.Flag ? r.{a} : r.{b})", Expression.Condition(flag, Column(a), Column(b)), true), + new($"(r.{a} ?? r.{b})", Expression.Coalesce(Column(a), Column(b)), true) + ]; + + Operand[] Slot(Type type, int position) + { + Operand[] Of(string left, string right) => position == 0 ? Forms(left, right) : Forms(right, left); + return Kind(type) switch + { + "range" => Of(nameof(PairRow.A), nameof(PairRow.B)), + "set" => Of(nameof(PairRow.SetA), nameof(PairRow.SetB)), + "values" => Of(nameof(PairRow.TagsA), nameof(PairRow.TagsB)), + "element" => [new("r.Day.Value", Expression.Property(Column(nameof(PairRow.Day)), nameof(Nullable<>.Value)), true)], + "text" => [new("r.Label", Column(nameof(PairRow.Label)), true)], + _ => [] + }; + } + + // The members of one operand are #355's and the nullable-column sweep's; here only the forms it + // does not reach, the conditional and the coalesce, are kept for them. + var calls = Calls(Slot, operands => operands.Count > 1 || operands[0].Name.StartsWith('(')); + calls.AddRange(EqualityAndOrder(Column)); + + var failures = new List(); + var untranslated = new HashSet(); + var unanswered = new HashSet(); + var judged = 0; + + await using var context = new PairContext(connectionString); + var loaded = await context.Rows.AsNoTracking().OrderBy(r => r.Id).ToListAsync(); + + foreach (var call in calls) + { + switch (await CompositionRunner.Judge("pair", call, row, context.Rows.AsNoTracking(), loaded, r => r.Id, id => names[id], failures)) + { + case Outcome.Compared: judged++; break; + case Outcome.Untranslated: untranslated.Add(call.Name); break; + case Outcome.Unanswered: unanswered.Add(call.Name); break; + } + } + + + Assert.IsGreaterThanOrEqualTo(500, judged, $"Only {judged} calls were judged; untranslated: {string.Join("; ", untranslated.Order())}; unanswered: {string.Join("; ", unanswered.Order())}"); + + Candidates.Verdict(nameof(NullableOperandPairProbeTests), failures, + "A binary member over two nullable operands: PostgreSQL and rule (c) disagree", + method: nameof(EveryBinaryMember_OverTwoNullableOperands_AnswersAsRuleC)); + } + + /// + /// ==, != and CompareTo against 0 for the two range, multirange and value set + /// columns, each under its own documented rule (). + /// + private static IEnumerable EqualityAndOrder(Func column) + { + foreach (var (a, b) in new[] { (nameof(PairRow.A), nameof(PairRow.B)), (nameof(PairRow.SetA), nameof(PairRow.SetB)), (nameof(PairRow.TagsA), nameof(PairRow.TagsB)) }) + { + yield return new Call($"r.{a} == r.{b}", Expression.Equal(column(a), column(b))); + yield return new Call($"r.{a} != r.{b}", Expression.NotEqual(column(a), column(b))); + } + + foreach (var (a, b) in new[] { (nameof(PairRow.A), nameof(PairRow.B)), (nameof(PairRow.SetA), nameof(PairRow.SetB)) }) + { + var compare = Expression.Call(column(a), column(a).Type.GetMethod(nameof(IComparable<>.CompareTo), [column(b).Type])!, column(b)); + foreach (var (name, op) in new[] { ("<", ExpressionType.LessThan), ("<=", ExpressionType.LessThanOrEqual), ("==", ExpressionType.Equal), + (">=", ExpressionType.GreaterThanOrEqual), (">", ExpressionType.GreaterThan) }) + yield return new Call($"r.{a}.CompareTo(r.{b}) {name} 0", Expression.MakeBinary(op, compare, Expression.Constant(0))); + } + } + + public sealed class PairRow + { + public int Id { get; set; } + + public bool Flag { get; set; } + + public DateOnly? Day { get; set; } + + public string? Label { get; set; } + + public DateRange? A { get; set; } + + public DateRange? B { get; set; } + + public RangeSet? SetA { get; set; } + + public RangeSet? SetB { get; set; } + + public StringSet? TagsA { get; set; } + + public StringSet? TagsB { get; set; } + } + + private sealed class PairContext(string connectionString) : DbContext + { + public DbSet Rows => Set(); + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => CompositionDatabase.Configure(optionsBuilder, connectionString); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + => modelBuilder.Entity().Property(r => r.Id).ValueGeneratedNever(); + } +} diff --git a/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/OuterJoinProbeTests.cs b/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/OuterJoinProbeTests.cs new file mode 100644 index 00000000..01a9146c --- /dev/null +++ b/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/OuterJoinProbeTests.cs @@ -0,0 +1,256 @@ +using System.Linq.Expressions; +using CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests.Composition; +using CodoMetis.ValueRanges.Oracles; +using Microsoft.EntityFrameworkCore; +using static CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests.Composition.CompositionMembers; + +namespace CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests; + +/// +/// The NULL an outer join produces from a required column: every range, +/// and value set member over the child of a parent that has none, reached through +/// GroupJoin/DefaultIfEmpty, LeftJoin and an optional navigation, against +/// over the rows as loaded (#375). +/// +/// +/// +/// The nullable-column sweeps put the NULL in a column the model declares nullable. Here the model says +/// the child's columns cannot be NULL, and only the join makes them so; EF Core reads that from the +/// join, not from the column, so a translation can be right over a nullable column and wrong here. +/// +/// +/// The calls are discovered (): the child's range, multirange, value set +/// and element columns as the receiver or the operand, the parent's required columns and captured values +/// on the other side, and the child's range coalesced with (,). On top of them, the range factories +/// over the child's element columns and the bounds through ??. The reference guards the missing +/// child explicitly: a predicate over it is and its negation +/// (rule (c)), and a bound, a value-producing member or a factory over it has no value. +/// +/// +[TestClass] +public sealed class OuterJoinProbeTests +{ + private static DateOnly March(int day) => new(2026, 3, day); + + private static DateRange Days(int from, int to) => DateRange.CreateFinite(March(from), March(to), true, true); + + private static RangeSet Blocks(params DateRange[] ranges) => RangeSet.From(ranges); + + [TestMethod] + public async Task EveryMember_OverAMissingChild_AnswersAsRuleC() + { + ContainerLifecycle.RequireServer(); + + var connectionString = await CompositionDatabase.Create("outer_join"); + await using (var seed = new OuterJoinContext(connectionString)) + { + await seed.Database.EnsureCreatedAsync(); + seed.Parents.AddRange( + new JoinParent { Id = 1, Window = Days(5, 22), WindowBlocks = Blocks(Days(3, 8)), Labels = StringSet.From(["a"]) }, + new JoinParent { Id = 2, Window = Days(5, 22), WindowBlocks = Blocks(Days(3, 8)), Labels = StringSet.From(["a"]) }, + new JoinParent { Id = 3, Window = DateRange.Infinite, WindowBlocks = Blocks(DateRange.Infinite), Labels = StringSet.Empty }, + new JoinParent { Id = 4, Window = Days(5, 22), WindowBlocks = Blocks(Days(3, 8)), Labels = StringSet.From(["b"]) }, + new JoinParent { Id = 5, Window = DateRange.Empty, WindowBlocks = Blocks(), Labels = StringSet.From(["a"]) }, + new JoinParent { Id = 6, Window = Days(5, 22), WindowBlocks = Blocks(Days(3, 8)), Labels = StringSet.From(["a"]) }, + new JoinParent { Id = 7, Window = DateRange.Infinite, WindowBlocks = Blocks(DateRange.Infinite), Labels = StringSet.Empty }); + seed.Children.AddRange( + new JoinChild { Id = 11, ParentId = 1, Period = Days(1, 10), Blocks = Blocks(Days(1, 10)), Tags = StringSet.From(["a"]), From = March(1), To = March(10), Name = "a" }, + new JoinChild { Id = 12, ParentId = 2, Period = DateRange.Empty, Blocks = Blocks(), Tags = StringSet.Empty, From = March(10), To = March(1), Name = "b" }, + new JoinChild { Id = 13, ParentId = 3, Period = DateRange.Infinite, Blocks = Blocks(DateRange.Infinite), Tags = StringSet.From(["a", "b"]), From = March(1), To = March(31), Name = "a" }, + new JoinChild { Id = 14, ParentId = 4, Period = DateRange.CreateUnboundedEnd(March(20), true), Blocks = Blocks(Days(20, 25), DateRange.CreateUnboundedEnd(March(27), true)), + Tags = StringSet.From(["b"]), From = March(20), To = March(25), Name = "c" }, + new JoinChild { Id = 15, ParentId = 5, Period = DateRange.CreateUnboundedStart(March(10), true), Blocks = Blocks(DateRange.CreateUnboundedStart(March(10), true)), + Tags = StringSet.From(["a"]), From = March(7), To = March(7), Name = "a" }); + await seed.SaveChangesAsync(); + } + + // Parents 6 and 7 have no child. The names say which shape the child holds. + var names = new Dictionary + { + [1] = "child [1,10]", [2] = "child empty", [3] = "child (,)", [4] = "child [20,)", [5] = "child (,10]", + [6] = "no child", [7] = "no child, window (,)" + }; + + var row = Expression.Parameter(typeof(Joined), "j"); + var child = Expression.Property(row, nameof(Joined.Child)); + var parent = Expression.Property(row, nameof(Joined.Parent)); + Expression Of(Expression on, string property) => Expression.Property(on, property); + + Operand[] Slot(Type type, int position) => Kind(type) switch + { + "range" => position == 0 + ? [new("c.Period", Of(child, nameof(JoinChild.Period)), true), + new("(c.Period ?? @(,))", Expression.Coalesce(Of(child, nameof(JoinChild.Period)), Captured(DateRange.Infinite)), true), + new("p.Window", Of(parent, nameof(JoinParent.Window)), false)] + : [new("c.Period", Of(child, nameof(JoinChild.Period)), true), + new("p.Window", Of(parent, nameof(JoinParent.Window)), false), + new("@[3,8]", Captured(Days(3, 8)), false)], + "set" => position == 0 + ? [new("c.Blocks", Of(child, nameof(JoinChild.Blocks)), true), new("p.WindowBlocks", Of(parent, nameof(JoinParent.WindowBlocks)), false)] + : [new("c.Blocks", Of(child, nameof(JoinChild.Blocks)), true), new("p.WindowBlocks", Of(parent, nameof(JoinParent.WindowBlocks)), false), + new("@{[3,8]}", Captured(Blocks(Days(3, 8))), false)], + "values" => position == 0 + ? [new("c.Tags", Of(child, nameof(JoinChild.Tags)), true), new("p.Labels", Of(parent, nameof(JoinParent.Labels)), false)] + : [new("c.Tags", Of(child, nameof(JoinChild.Tags)), true), new("p.Labels", Of(parent, nameof(JoinParent.Labels)), false), + new("@{a}", Captured(StringSet.From(["a"])), false)], + "element" => [new("c.From", Of(child, nameof(JoinChild.From)), true), new("@2026-03-07", Captured(March(7)), false)], + "text" => [new("c.Name", Of(child, nameof(JoinChild.Name)), true), new("@a", Captured("a"), false)], + _ => [] + }; + + // A call is this sweep's when the child reaches it, and a binary one only once with the child on + // both sides; the (c.Period ?? @(,)) receiver is for the members of one operand. + var calls = Calls(Slot, operands => + operands.Any(o => o.Nullable) + && (operands.Count == 1 || !operands[0].Name.StartsWith('('))); + + calls.AddRange(Factories(child)); + + var failures = new List(); + var untranslated = new HashSet(); + var unanswered = new HashSet(); + var judged = 0; + + await using var context = new OuterJoinContext(connectionString); + var parents = await context.Parents.AsNoTracking().ToListAsync(); + var children = await context.Children.AsNoTracking().ToListAsync(); + var loaded = parents.Select(p => new Joined { Id = p.Id, Parent = p, Child = children.SingleOrDefault(c => c.ParentId == p.Id) }) + .OrderBy(j => j.Id).ToList(); + Assert.AreEqual(2, loaded.Count(j => j.Child is null), "the parents without a child are what this sweep is about"); + + foreach (var (form, source) in JoinForms(context)) + { + foreach (var call in calls) + { + switch (await CompositionRunner.Judge(form, call, row, source, loaded, j => j.Id, id => names[id], failures)) + { + case Outcome.Compared: judged++; break; + case Outcome.Untranslated: untranslated.Add(call.Name); break; + case Outcome.Unanswered: unanswered.Add(call.Name); break; + } + } + } + + // Some 280 calls translate per join form today; the floor catches a discovery or a join form that stopped working. + Assert.IsGreaterThanOrEqualTo(3 * 250, judged, + $"Only {judged} calls were judged; untranslated: {string.Join("; ", untranslated.Order())}; unanswered: {string.Join("; ", unanswered.Order())}"); + + Candidates.Verdict(nameof(OuterJoinProbeTests), failures, + "A member over the child of an outer join: PostgreSQL and rule (c) disagree for a parent without a child", + method: nameof(EveryMember_OverAMissingChild_AnswersAsRuleC)); + } + + /// + /// The three ways a consumer reaches an optional child: an optional reference navigation, a + /// GroupJoin flattened with DefaultIfEmpty, and .NET 10's LeftJoin. + /// + private static (string Form, IQueryable Source)[] JoinForms(OuterJoinContext context) => + [ + ("navigation", context.Parents.Select(p => new Joined { Id = p.Id, Parent = p, Child = p.Child })), + ("DefaultIfEmpty", context.Parents.GroupJoin(context.Children, p => p.Id, c => c.ParentId, (p, cs) => new { p, cs }) + .SelectMany(x => x.cs.DefaultIfEmpty(), (x, c) => new Joined { Id = x.p.Id, Parent = x.p, Child = c })), + ("LeftJoin", context.Parents.LeftJoin(context.Children, p => p.Id, c => c.ParentId, + (p, c) => new Joined { Id = p.Id, Parent = p, Child = c })) + ]; + + /// + /// The range factories over the child's element columns, as values and under a predicate, and the + /// bounds of the child's range through ??: the calls that read the child's columns through + /// something other than a library member's receiver. + /// + private static IEnumerable Factories(Expression child) + { + var from = Expression.Property(child, nameof(JoinChild.From)); + var to = Expression.Property(child, nameof(JoinChild.To)); + var v = Captured(March(7)); + + var finite = Expression.Call(typeof(DateRange), nameof(DateRange.CreateFinite), null, from, to, Expression.Constant(true), Expression.Constant(true)); + var start = Expression.Call(typeof(DateRange), nameof(DateRange.CreateUnboundedStart), null, to, Expression.Constant(true)); + var end = Expression.Call(typeof(DateRange), nameof(DateRange.CreateUnboundedEnd), null, from, Expression.Constant(true)); + + foreach (var (name, factory) in new[] + { + ("DateRange.CreateFinite(c.From, c.To)", (Expression)finite), + ("DateRange.CreateUnboundedStart(c.To)", start), + ("DateRange.CreateUnboundedEnd(c.From)", end) + }) + { + yield return new Call(name, factory); + yield return new Call($"{name}.IsEmpty()", Expression.Call(typeof(RangeExtensions), nameof(RangeExtensions.IsEmpty), [typeof(DateOnly)], factory)); + yield return new Call($"{name}.Contains(@2026-03-07)", + Expression.Call(typeof(RangeExtensions).GetMethods().Single(m => m.Name == nameof(RangeExtensions.Contains) + && m.GetParameters()[1].ParameterType.IsGenericParameter).MakeGenericMethod(typeof(DateOnly)), + factory, v)); + } + + var period = Expression.Property(child, nameof(JoinChild.Period)); + var lower = Expression.Call(typeof(RangeExtensions), nameof(RangeExtensions.LowerBound), [typeof(DateOnly)], period); + var upper = Expression.Call(typeof(RangeExtensions), nameof(RangeExtensions.UpperBound), [typeof(DateOnly)], period); + yield return new Call("(c.Period.LowerBound() ?? @0001-01-01) <= @2026-03-07", + Expression.LessThanOrEqual(Expression.Coalesce(lower, Captured(DateOnly.MinValue)), Expression.Convert(v, typeof(DateOnly)))); + yield return new Call("(c.Period.UpperBound() ?? @9999-12-31) >= @2026-03-07", + Expression.GreaterThanOrEqual(Expression.Coalesce(upper, Captured(DateOnly.MaxValue)), Expression.Convert(v, typeof(DateOnly)))); + yield return new Call("c.Period.LowerBound() ?? @0001-01-01", Expression.Coalesce(lower, Captured(DateOnly.MinValue))); + yield return new Call("c.From <= c.To", Expression.LessThanOrEqual(from, to)); + } + + public sealed class JoinParent + { + public int Id { get; set; } + + public DateRange Window { get; set; } = DateRange.Empty; + + public RangeSet WindowBlocks { get; set; } = RangeSet.Empty; + + public StringSet Labels { get; set; } = StringSet.Empty; + + public JoinChild? Child { get; set; } + } + + /// The optional child: every column required, so only the join can make it NULL. + public sealed class JoinChild + { + public int Id { get; set; } + + public int ParentId { get; set; } + + public DateRange Period { get; set; } = DateRange.Empty; + + public RangeSet Blocks { get; set; } = RangeSet.Empty; + + public StringSet Tags { get; set; } = StringSet.Empty; + + public DateOnly From { get; set; } + + public DateOnly To { get; set; } + + public string Name { get; set; } = ""; + } + + /// A parent and its child, if any, as each join form returns them. + public sealed class Joined + { + public int Id { get; set; } + + public JoinParent Parent { get; set; } = null!; + + public JoinChild? Child { get; set; } + } + + private sealed class OuterJoinContext(string connectionString) : DbContext + { + public DbSet Parents => Set(); + + public DbSet Children => Set(); + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => CompositionDatabase.Configure(optionsBuilder, connectionString); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity().Property(p => p.Id).ValueGeneratedNever(); + modelBuilder.Entity().Property(c => c.Id).ValueGeneratedNever(); + modelBuilder.Entity().HasOne(p => p.Child).WithOne().HasForeignKey(c => c.ParentId); + } + } +} diff --git a/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/QuantifierProbeTests.cs b/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/QuantifierProbeTests.cs new file mode 100644 index 00000000..50cd92b8 --- /dev/null +++ b/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/QuantifierProbeTests.cs @@ -0,0 +1,394 @@ +using System.Linq.Expressions; +using System.Reflection; +using CodoMetis.ValueRanges.Core; +using CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests.Composition; +using CodoMetis.ValueRanges.Oracles; +using Microsoft.EntityFrameworkCore; +using static CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests.Composition.CompositionMembers; + +namespace CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests; + +/// +/// Quantifiers over captured lists, compound Boolean predicates and predicates over a value-producing +/// member, over nullable range, , value set and element columns, held +/// against over the rows as loaded and by metamorphic relations inside the +/// database that need no memory answer (#377). +/// +/// +/// +/// CapturedCollectionProbeTests sends a list through Contains, Any and +/// Count over required columns. Here the column can be NULL, the list can hold a NULL, and the +/// quantifier can be All, which EF Core writes as NOT EXISTS (… WHERE NOT …): a +/// three-valued NOT inside it turns a NULL predicate into a row that All keeps. The +/// relations: All(P) selects what !Any(x => !P) selects, Count(P) == 0 what +/// !Any(P) selects, !(P && Q) what !P || !Q selects and !(P || Q) what +/// !P && !Q selects; and for every predicate, Where(P) and Where(!P) partition the +/// rows and Select(P) is exactly on Where(P)'s. +/// +/// +[TestClass] +public sealed class QuantifierProbeTests +{ + private static DateOnly March(int day) => new(2026, 3, day); + + private static DateRange Days(int from, int to) => DateRange.CreateFinite(March(from), March(to), true, true); + + private static RangeSet Blocks(params DateRange[] ranges) => RangeSet.From(ranges); + + private static readonly ParameterExpression Row = Expression.Parameter(typeof(QuantRow), "r"); + + private static readonly Dictionary Names = new() + { + [1] = "null", [2] = "[1,10]", [3] = "empty", [4] = "(,)", [5] = "[20,)", [6] = "(,10]", [7] = "[5,8]" + }; + + private static async Task<(QuantContext Context, List Loaded)> Database() + { + var connectionString = await CompositionDatabase.Create("quantifier"); + await using (var seed = new QuantContext(connectionString)) + { + await seed.Database.EnsureCreatedAsync(); + seed.Rows.AddRange( + new QuantRow { Id = 1 }, + new QuantRow { Id = 2, Period = Days(1, 10), Blocks = Blocks(Days(1, 10)), Tags = StringSet.From(["a"]), Day = March(7), Label = "a" }, + new QuantRow { Id = 3, Period = DateRange.Empty, Blocks = Blocks(), Tags = StringSet.Empty, Day = March(3), Label = "b" }, + new QuantRow { Id = 4, Period = DateRange.Infinite, Blocks = Blocks(DateRange.Infinite), Tags = StringSet.From(["a", "b"]), Day = March(20) }, + new QuantRow { Id = 5, Period = DateRange.CreateUnboundedEnd(March(20), true), Blocks = Blocks(Days(20, 25), DateRange.CreateUnboundedEnd(March(27), true)), + Tags = StringSet.From(["b"]), Label = "a" }, + new QuantRow { Id = 6, Period = DateRange.CreateUnboundedStart(March(10), true), Blocks = Blocks(DateRange.CreateUnboundedStart(March(10), true)), + Tags = StringSet.From(["a"]), Day = March(31), Label = "c" }, + new QuantRow { Id = 7, Period = Days(5, 8), Blocks = Blocks(Days(3, 8)), Tags = StringSet.From(["c"]), Day = March(7), Label = "a" }); + await seed.SaveChangesAsync(); + } + + var context = new QuantContext(connectionString); + return (context, await context.Rows.AsNoTracking().OrderBy(r => r.Id).ToListAsync()); + } + + private static Expression Column(string name) => Expression.Property(Row, name); + + // The nullable column a parameter of each kind takes. + private static Operand? ColumnFor(string? kind) => kind switch + { + "range" => new("r.Period", Column(nameof(QuantRow.Period)), true), + "set" => new("r.Blocks", Column(nameof(QuantRow.Blocks)), true), + "values" => new("r.Tags", Column(nameof(QuantRow.Tags)), true), + "element" => new("r.Day.Value", Expression.Property(Column(nameof(QuantRow.Day)), nameof(Nullable<>.Value)), true), + "text" => new("r.Label", Column(nameof(QuantRow.Label)), true), + _ => null + }; + + /// + /// list.Any(x => P), list.All(x => P) and list.Count(x => P) == 0 for every + /// binary member with a nullable column on one side and the list's element on the other, over lists + /// that are empty, hold one value, a duplicate, a NULL, or one of each shape; and + /// list.Contains(r.Column) over the same lists. + /// + [TestMethod] + public async Task Quantifiers_OverCapturedLists_AnswerAsRuleCAndAgreeWithEachOther() + { + ContainerLifecycle.RequireServer(); + + var (context, loaded) = await Database(); + await using var _ = context; + + var elements = new Dictionary + { + ["range"] = Expression.Parameter(RangeType, "x"), ["set"] = Expression.Parameter(SetType, "x"), ["values"] = Expression.Parameter(ValueSetType, "x"), + ["element"] = Expression.Parameter(ElementType, "x"), ["text"] = Expression.Parameter(typeof(string), "x") + }; + var lists = new Dictionary + { + ["range"] = [("[]", new List()), ("[[3,8]]", new List { Days(3, 8) }), ("[[3,8],[3,8]]", new List { Days(3, 8), Days(3, 8) }), + ("[null,[3,8]]", new List { null!, Days(3, 8) }), + ("[empty,(,),[3,8],[20,)]", new List { DateRange.Empty, DateRange.Infinite, Days(3, 8), DateRange.CreateUnboundedEnd(March(20), true) })], + ["set"] = [("[]", new List>()), ("[{[3,8]}]", new List> { Blocks(Days(3, 8)) }), + ("[{[3,8]},{[3,8]}]", new List> { Blocks(Days(3, 8)), Blocks(Days(3, 8)) }), + ("[null,{[3,8]}]", new List> { null!, Blocks(Days(3, 8)) }), + ("[{},{(,)}]", new List> { Blocks(), Blocks(DateRange.Infinite) })], + ["values"] = [("[]", new List()), ("[{a}]", new List { StringSet.From(["a"]) }), + ("[{a},{a}]", new List { StringSet.From(["a"]), StringSet.From(["a"]) }), + ("[null,{a}]", new List { null!, StringSet.From(["a"]) }), + ("[{},{a,b}]", new List { StringSet.Empty, StringSet.From(["a", "b"]) })], + ["element"] = [("[]", new List()), ("[7]", new List { March(7) }), ("[7,7]", new List { March(7), March(7) }), + ("[3,20]", new List { March(3), March(20) })], + ["text"] = [("[]", new List()), ("[a]", new List { "a" }), ("[a,a]", new List { "a", "a" }), ("[null,a]", new List { null!, "a" })] + }; + + // The bodies: every binary member with the column on one side and x on the other. + var bodies = new List<(string Name, string Kind, Expression Body)>(); + foreach (var method in Methods) + { + var slots = Slots(method); + if (slots.Count != 2 || method.ReturnType != typeof(bool)) continue; + for (var at = 0; at < 2; at++) + { + var columnKind = Kind(slots[1 - at]); + var elementKind = Kind(slots[at]); + if (ColumnFor(columnKind) is not { } column || elementKind is null) continue; + + var x = elements[elementKind]; + var operands = at == 0 ? new[] { (Expression)x, column.Value } : [column.Value, x]; + var name = at == 0 ? $"x.{method.Name}({column.Name})" : $"{column.Name}.{method.Name}(x)"; + bodies.Add((name, elementKind, Build(method, operands))); + } + } + + var calls = new List(); + var relations = new List<(string Name, Call Left, Call Right)>(); + foreach (var (name, kind, body) in bodies) + { + var x = elements[kind]; + foreach (var (listName, list) in lists[kind]) + { + var captured = Expression.Field(Expression.Constant(Holder(list)), "Value"); + var any = Quantify(nameof(Enumerable.Any), captured, x, body); + var all = Quantify(nameof(Enumerable.All), captured, x, body); + var notAny = Expression.Not(Quantify(nameof(Enumerable.Any), captured, x, Expression.Not(body))); + var none = Expression.Equal(Quantify(nameof(Enumerable.Count), captured, x, body), Expression.Constant(0)); + + var anyCall = new Call($"@{listName}.Any(x => {name})", any); + var allCall = new Call($"@{listName}.All(x => {name})", all); + var noneCall = new Call($"@{listName}.Count(x => {name}) == 0", none); + calls.AddRange([anyCall, allCall, noneCall]); + relations.Add(("All(P) == !Any(!P)", allCall, new Call($"!@{listName}.Any(x => !{name})", notAny))); + relations.Add(("Count(P) == 0 == !Any(P)", noneCall, new Call($"!@{listName}.Any(x => {name})", Expression.Not(any)))); + } + } + + // list.Contains(column): EF Core's own operator, over a nullable column and a list that may hold NULL. + foreach (var kind in new[] { "range", "set", "values", "text" }) + { + var column = ColumnFor(kind)!; + foreach (var (listName, list) in lists[kind]) + { + var captured = Expression.Field(Expression.Constant(Holder(list)), "Value"); + var contains = typeof(Enumerable).GetMethods().Single(m => m.Name == nameof(Enumerable.Contains) && m.GetParameters().Length == 2) + .MakeGenericMethod(column.Value.Type); + calls.Add(new Call($"@{listName}.Contains({column.Name})", Expression.Call(contains, captured, column.Value))); + } + } + + var failures = new List(); + var untranslated = new HashSet(); + var unanswered = new HashSet(); + var judged = 0; + foreach (var call in calls) + { + switch (await CompositionRunner.Judge("quantifier", call, Row, context.Rows.AsNoTracking(), loaded, r => r.Id, id => Names[id], failures)) + { + case Outcome.Compared: judged++; break; + case Outcome.Untranslated: untranslated.Add(call.Name); break; + case Outcome.Unanswered: unanswered.Add(call.Name); break; + } + } + + foreach (var (relation, left, right) in relations.Where(r => !untranslated.Contains(r.Left.Name))) + await Relate(context, relation, left, right, failures); + + + Assert.IsGreaterThanOrEqualTo(300, judged, $"Only {judged} quantifiers were judged; untranslated: {string.Join("; ", untranslated.Order())}; unanswered: {string.Join("; ", unanswered.Order())}"); + + Candidates.Verdict(nameof(QuantifierProbeTests), failures, + "A quantifier over a captured list and a nullable column: PostgreSQL disagrees with rule (c) or with itself", + method: nameof(Quantifiers_OverCapturedLists_AnswerAsRuleCAndAgreeWithEachOther)); + } + + /// + /// P && Q and P || Q over every ordered pair of predicates drawn across the nullable + /// range, multirange, value set and bound, negated, and against De Morgan's forms. + /// + [TestMethod] + public async Task CompoundPredicates_OverNullableColumns_AnswerAsRuleCAndKeepDeMorgan() + { + ContainerLifecycle.RequireServer(); + + var (context, loaded) = await Database(); + await using var _ = context; + + var period = Column(nameof(QuantRow.Period)); + var blocks = Column(nameof(QuantRow.Blocks)); + var tags = Column(nameof(QuantRow.Tags)); + (string Name, Expression Body)[] predicates = + [ + ("r.Period.Overlaps(@[3,8])", Range1(nameof(RangeExtensions.Overlaps), period, Captured(Days(3, 8)))), + ("r.Period.IsEmpty()", Expression.Call(typeof(RangeExtensions), nameof(RangeExtensions.IsEmpty), [ElementType], period)), + ("r.Period.IsStrictlyLeftOf(@[20,25])", Range1(nameof(RangeExtensions.IsStrictlyLeftOf), period, Captured(Days(20, 25)))), + ("r.Period.LowerBound() == null", Expression.Equal(Expression.Call(typeof(RangeExtensions), nameof(RangeExtensions.LowerBound), [ElementType], period), + Expression.Constant(null, typeof(DateOnly?)))), + ("r.Blocks.Contains(@2026-03-07)", Expression.Call(blocks, SetType.GetMethod(nameof(RangeSet<,>.Contains), [ElementType])!, Captured(March(7)))), + ("r.Tags.Contains(@a)", Expression.Call(Methods.Single(m => m.DeclaringType == typeof(ValueSetExtensions) && m.Name == nameof(ValueSetExtensions.Contains)), tags, Captured("a"))), + ("r.Tags.IsSubsetOf(@{a,b})", Expression.Call(Methods.Single(m => m.DeclaringType == typeof(ValueSetExtensions) && m.Name == nameof(ValueSetExtensions.IsSubsetOf)), + tags, Captured(StringSet.From(["a", "b"])))), + ("r.Tags.Count > 0", Expression.GreaterThan(Expression.Property(tags, nameof(StringSet.Count)), Expression.Constant(0))) + ]; + + var calls = new List(); + var relations = new List<(string Name, Call Left, Call Right)>(); + foreach (var (p, pBody) in predicates) + foreach (var (q, qBody) in predicates.Where(q => q.Name != p)) + { + var and = new Call($"{p} && {q}", Expression.AndAlso(pBody, qBody)); + var or = new Call($"{p} || {q}", Expression.OrElse(pBody, qBody)); + calls.AddRange([and, or]); + relations.Add(("!(P && Q) == !P || !Q", new Call($"!({p} && {q})", Expression.Not(and.Body)), + new Call($"!{p} || !{q}", Expression.OrElse(Expression.Not(pBody), Expression.Not(qBody))))); + relations.Add(("!(P || Q) == !P && !Q", new Call($"!({p} || {q})", Expression.Not(or.Body)), + new Call($"!{p} && !{q}", Expression.AndAlso(Expression.Not(pBody), Expression.Not(qBody))))); + } + + var failures = new List(); + var judged = 0; + foreach (var call in calls) + { + if (await CompositionRunner.Judge("compound", call, Row, context.Rows.AsNoTracking(), loaded, r => r.Id, id => Names[id], failures) == Outcome.Compared) judged++; + } + + foreach (var (relation, left, right) in relations) + await Relate(context, relation, left, right, failures); + + + Assert.AreEqual(calls.Count, judged, "every compound predicate translates"); + + Candidates.Verdict(nameof(QuantifierProbeTests), failures, + "A compound predicate over nullable columns: PostgreSQL disagrees with rule (c) or with De Morgan", + method: nameof(CompoundPredicates_OverNullableColumns_AnswerAsRuleCAndKeepDeMorgan)); + } + + /// + /// A predicate over the result of a value-producing member: IsEmpty and an overlap or + /// containment test on every discovered member that returns a range, a multirange or a value set, + /// over a nullable column and a captured operand. + /// + [TestMethod] + public async Task NestedPredicates_OverAValueProducingMember_AnswerAsRuleC() + { + ContainerLifecycle.RequireServer(); + + var (context, loaded) = await Database(); + await using var _ = context; + + Operand[] Slot(Type type, int position) => position == 0 + ? ColumnFor(Kind(type)) is { } column ? [column] : [] + : Kind(type) switch + { + "range" => [new("@[3,8]", Captured(Days(3, 8)), false), new("@empty", Captured(DateRange.Empty), false)], + "set" => [new("@{[3,8]}", Captured(Blocks(Days(3, 8))), false)], + "values" => [new("@{a}", Captured(StringSet.From(["a"])), false)], + "element" => [new("@2026-03-07", Captured(March(7)), false)], + "text" => [new("@a", Captured("a"), false)], + _ => [] + }; + + var calls = new List(); + foreach (var inner in Calls(Slot, _ => true).Where(c => Kind(c.Body.Type) is "range" or "set" or "values")) + { + switch (Kind(inner.Body.Type)) + { + case "range": + calls.Add(new Call($"{inner.Name}.IsEmpty()", Expression.Call(typeof(RangeExtensions), nameof(RangeExtensions.IsEmpty), [ElementType], inner.Body))); + calls.Add(new Call($"{inner.Name}.Overlaps(@[3,8])", Range1(nameof(RangeExtensions.Overlaps), inner.Body, Captured(Days(3, 8))))); + break; + case "set": + calls.Add(new Call($"{inner.Name}.IsEmpty()", Expression.Call(inner.Body, SetType.GetMethod(nameof(RangeSet<,>.IsEmpty))!))); + calls.Add(new Call($"{inner.Name}.Contains(@2026-03-07)", Expression.Call(inner.Body, SetType.GetMethod(nameof(RangeSet<,>.Contains), [ElementType])!, Captured(March(7))))); + break; + default: + calls.Add(new Call($"{inner.Name}.IsEmpty", Expression.Property(inner.Body, nameof(StringSet.IsEmpty)))); + calls.Add(new Call($"{inner.Name}.Contains(@a)", + Expression.Call(Methods.Single(m => m.DeclaringType == typeof(ValueSetExtensions) && m.Name == nameof(ValueSetExtensions.Contains)), + inner.Body, Captured("a")))); + break; + } + } + + var failures = new List(); + var untranslated = new HashSet(); + var unanswered = new HashSet(); + var judged = 0; + foreach (var call in calls) + { + switch (await CompositionRunner.Judge("nested", call, Row, context.Rows.AsNoTracking(), loaded, r => r.Id, id => Names[id], failures)) + { + case Outcome.Compared: judged++; break; + case Outcome.Untranslated: untranslated.Add(call.Name); break; + case Outcome.Unanswered: unanswered.Add(call.Name); break; + } + } + + + Assert.IsGreaterThanOrEqualTo(40, judged, $"Only {judged} nested predicates were judged; untranslated: {string.Join("; ", untranslated.Order())}; unanswered: {string.Join("; ", unanswered.Order())}"); + + Candidates.Verdict(nameof(QuantifierProbeTests), failures, + "A predicate over a value-producing member of a nullable column: PostgreSQL and rule (c) disagree", + method: nameof(NestedPredicates_OverAValueProducingMember_AnswerAsRuleC)); + } + + /// Two filters that must select the same rows; one line per row they split on. + private static async Task Relate(QuantContext context, string relation, Call left, Call right, List failures) + { + try + { + var a = await Ids(context, left.Body); + var b = await Ids(context, right.Body); + foreach (var id in a.Union(b).Order().Where(id => a.Contains(id) != b.Contains(id))) + failures.Add($"{left.Name} [{relation}], row {Names[id]}: {(a.Contains(id) ? "selected" : "dropped")} by the left form, {(b.Contains(id) ? "selected" : "dropped")} by {right.Name}"); + } + catch (InvalidOperationException e) when (e.Message.Contains("could not be translated", StringComparison.Ordinal)) + { + // Counted by Judge; a right-hand form EF Core cannot translate has nothing to relate. + } + catch (Exception e) when (e is InvalidOperationException or System.Data.Common.DbException) + { + failures.Add($"{left.Name} [{relation}]: server threw {e.GetType().Name}: {CompositionRunner.FirstLine(e)}"); + } + } + + private static async Task> Ids(QuantContext context, Expression filter) => + (await context.Rows.Where(Expression.Lambda>(filter, Row)).Select(r => r.Id).ToListAsync()).ToHashSet(); + + private static Expression Quantify(string quantifier, Expression list, ParameterExpression x, Expression body) => + Expression.Call(typeof(Enumerable), quantifier, [x.Type], list, Expression.Lambda(body, x)); + + // A captured list, which EF Core sends as a parameter; typed by its runtime type. + private static object Holder(object list) => + Activator.CreateInstance(typeof(System.Runtime.CompilerServices.StrongBox<>).MakeGenericType(list.GetType()), list)!; + + private static Expression Range1(string name, Expression range, Expression operand) => + Expression.Call(typeof(RangeExtensions).GetMethods(BindingFlags.Public | BindingFlags.Static) + .Single(m => m.Name == name && m.GetParameters()[1].ParameterType.IsGenericType + && m.GetParameters()[1].ParameterType.GetGenericTypeDefinition() == typeof(IRange<>)) + .MakeGenericMethod(ElementType), + range, operand); + + private static Expression Build(MethodInfo method, IReadOnlyList operands) => + method.IsSpecialName && method.Name.StartsWith("op_", StringComparison.Ordinal) + ? throw new NotSupportedException(method.Name) + : method.IsStatic ? Expression.Call(method, operands) : Expression.Call(operands[0], method, operands.Skip(1)); + + public sealed class QuantRow + { + public int Id { get; set; } + + public DateRange? Period { get; set; } + + public RangeSet? Blocks { get; set; } + + public StringSet? Tags { get; set; } + + public DateOnly? Day { get; set; } + + public string? Label { get; set; } + } + + private sealed class QuantContext(string connectionString) : DbContext + { + public DbSet Rows => Set(); + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => CompositionDatabase.Configure(optionsBuilder, connectionString); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + => modelBuilder.Entity().Property(r => r.Id).ValueGeneratedNever(); + } +} diff --git a/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/RepeatedExecutionProbeTests.cs b/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/RepeatedExecutionProbeTests.cs new file mode 100644 index 00000000..8e02b0a0 --- /dev/null +++ b/test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/RepeatedExecutionProbeTests.cs @@ -0,0 +1,394 @@ +using System.Data.Common; +using System.Linq.Expressions; +using System.Reflection; +using System.Runtime.CompilerServices; +using CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests.Composition; +using CodoMetis.ValueRanges.Oracles; +using Microsoft.EntityFrameworkCore; +using static CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests.Composition.CompositionMembers; + +namespace CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests; + +/// +/// The same query run again and again while its parameter changes, as a captured variable and as the +/// argument of EF.CompileQuery and EF.CompileAsyncQuery, each execution against +/// over the rows as loaded (#378). +/// +/// +/// +/// Every other sweep runs a query once. EF Core caches a query by its expression's shape and builds the +/// SQL again only for a new shape of its parameters: whether each one is NULL, and how many elements a +/// collection has. A translation that reads a parameter's value while it translates (a NULL operand, an +/// empty collection) is then reused for values it was not built for. So each operand is held in one +/// captured variable that walks null → finite → empty → (,) → null and back, and each captured +/// list walks empty → one element → a duplicate → a NULL element → empty and back. +/// +/// +/// The calls are discovered (): every binary member with the nullable +/// column on one side and the captured variable on the other, in a filter, a negated filter and a +/// projection; and Any, All, Count and Contains over the captured list. The +/// compiled forms run every predicate with the value as their argument, filtered and negated. +/// +/// +[TestClass] +public sealed class RepeatedExecutionProbeTests +{ + private static DateOnly March(int day) => new(2026, 3, day); + + private static DateRange Days(int from, int to) => DateRange.CreateFinite(March(from), March(to), true, true); + + private static RangeSet Blocks(params DateRange[] ranges) => RangeSet.From(ranges); + + private static readonly ParameterExpression Row = Expression.Parameter(typeof(RepeatRow), "r"); + + private static readonly Dictionary Names = new() + { + [1] = "null", [2] = "[1,10]", [3] = "empty", [4] = "(,)", [5] = "[20,)", [6] = "(,10]", [7] = "[5,8]" + }; + + // The walk each captured value takes, forward and back: null first, last and in the middle. + private static (string Name, object? Value)[] Walk(params (string Name, object? Value)[] values) => + [.. values, .. values.Reverse().Skip(1)]; + + private static readonly Dictionary Values = new() + { + ["range"] = Walk(("null", null), ("[3,8]", Days(3, 8)), ("empty", DateRange.Empty), ("(,)", DateRange.Infinite), ("null", null)), + ["set"] = Walk(("null", null), ("{[3,8]}", Blocks(Days(3, 8))), ("{}", Blocks()), ("{(,)}", Blocks(DateRange.Infinite)), ("null", null)), + ["values"] = Walk(("null", null), ("{a}", StringSet.From(["a"])), ("{}", StringSet.Empty), ("{a,b}", StringSet.From(["a", "b"])), ("null", null)) + }; + + private static readonly Dictionary Lists = new() + { + ["range"] = Walk(("[]", new List()), ("[[3,8]]", new List { Days(3, 8) }), ("[[3,8],[3,8]]", new List { Days(3, 8), Days(3, 8) }), + ("[null,(,)]", new List { null!, DateRange.Infinite }), ("[]", new List())) + .Select(v => (v.Name, v.Value!)).ToArray(), + ["set"] = Walk(("[]", new List>()), ("[{[3,8]}]", new List> { Blocks(Days(3, 8)) }), + ("[{[3,8]},{[3,8]}]", new List> { Blocks(Days(3, 8)), Blocks(Days(3, 8)) }), + ("[null,{}]", new List> { null!, Blocks() }), ("[]", new List>())) + .Select(v => (v.Name, v.Value!)).ToArray(), + ["values"] = Walk(("[]", new List()), ("[{a}]", new List { StringSet.From(["a"]) }), + ("[{a},{a}]", new List { StringSet.From(["a"]), StringSet.From(["a"]) }), + ("[null,{}]", new List { null!, StringSet.Empty }), ("[]", new List())) + .Select(v => (v.Name, v.Value!)).ToArray() + }; + + private static async Task<(RepeatContext Context, List Loaded)> Database() + { + var connectionString = await CompositionDatabase.Create("repeated"); + await using (var seed = new RepeatContext(connectionString)) + { + await seed.Database.EnsureCreatedAsync(); + seed.Rows.AddRange( + new RepeatRow { Id = 1 }, + new RepeatRow { Id = 2, Period = Days(1, 10), Blocks = Blocks(Days(1, 10)), Tags = StringSet.From(["a"]) }, + new RepeatRow { Id = 3, Period = DateRange.Empty, Blocks = Blocks(), Tags = StringSet.Empty }, + new RepeatRow { Id = 4, Period = DateRange.Infinite, Blocks = Blocks(DateRange.Infinite), Tags = StringSet.From(["a", "b"]) }, + new RepeatRow { Id = 5, Period = DateRange.CreateUnboundedEnd(March(20), true), Blocks = Blocks(Days(20, 25), DateRange.CreateUnboundedEnd(March(27), true)), + Tags = StringSet.From(["b"]) }, + new RepeatRow { Id = 6, Period = DateRange.CreateUnboundedStart(March(10), true), Blocks = Blocks(DateRange.CreateUnboundedStart(March(10), true)), + Tags = StringSet.From(["a"]) }, + new RepeatRow { Id = 7, Period = Days(5, 8), Blocks = Blocks(Days(3, 8)), Tags = StringSet.From(["c"]) }); + await seed.SaveChangesAsync(); + } + + var context = new RepeatContext(connectionString); + return (context, await context.Rows.AsNoTracking().OrderBy(r => r.Id).ToListAsync()); + } + + private static Expression Column(string name) => Expression.Property(Row, name); + + private static Operand? ColumnFor(string? kind) => kind switch + { + "range" => new("r.Period", Column(nameof(RepeatRow.Period)), true), + "set" => new("r.Blocks", Column(nameof(RepeatRow.Blocks)), true), + "values" => new("r.Tags", Column(nameof(RepeatRow.Tags)), true), + _ => null + }; + + /// The static type a captured operand of each kind is declared as. + private static Type Declared(string kind) => kind switch + { + "range" => RangeType, + "set" => SetType, + _ => ValueSetType + }; + + /// + /// Every binary member with the nullable column on one side and one captured variable on the other, + /// run once per value of the walk, in a filter, a negated filter and a projection. + /// + [TestMethod] + public async Task CapturedOperand_WalkedThroughNullFiniteEmptyAndInfinite_AnswersAsRuleCEveryTime() + { + ContainerLifecycle.RequireServer(); + + var (context, loaded) = await Database(); + await using var _ = context; + + var failures = new List(); + var untranslated = new HashSet(); + var unanswered = new HashSet(); + var judged = 0; + + foreach (var (name, kind, body, holder) in Bodies()) + { + foreach (var (step, (valueName, value)) in Values[kind].Index()) + { + holder.Value = value; + var call = new Call(name, body); + switch (await CompositionRunner.Judge($"execution {step + 1}, @p = {valueName}", call, Row, context.Rows.AsNoTracking(), loaded, + r => r.Id, id => Names[id], failures)) + { + case Outcome.Compared: judged++; break; + case Outcome.Untranslated: untranslated.Add(name); break; + case Outcome.Unanswered: unanswered.Add(name); break; + } + } + } + + + // Some 1000 executions are judged today; the floor catches a discovery that stopped matching. + Assert.IsGreaterThanOrEqualTo(700, judged, $"Only {judged} executions were judged; untranslated: {string.Join("; ", untranslated.Order())}; unanswered: {string.Join("; ", unanswered.Order())}"); + + Candidates.Verdict(nameof(RepeatedExecutionProbeTests), failures, + "A query run again with its captured operand changed: PostgreSQL and rule (c) disagree on a later execution", + method: nameof(CapturedOperand_WalkedThroughNullFiniteEmptyAndInfinite_AnswersAsRuleCEveryTime)); + } + + /// + /// Any, All and Count of every binary member over one captured list, and + /// Contains of the column, run once per list of the walk. + /// + [TestMethod] + public async Task CapturedList_WalkedThroughEmptyOneDuplicateAndNull_AnswersAsRuleCEveryTime() + { + ContainerLifecycle.RequireServer(); + + var (context, loaded) = await Database(); + await using var _ = context; + + var calls = new List<(Call Call, string Kind, IStrongBox Holder)>(); + foreach (var kind in Lists.Keys) + { + var listType = typeof(List<>).MakeGenericType(Declared(kind)); + var holder = (IStrongBox)Activator.CreateInstance(typeof(StrongBox<>).MakeGenericType(listType))!; + var list = Expression.Field(Expression.Constant(holder), nameof(StrongBox<>.Value)); + var x = Expression.Parameter(Declared(kind), "x"); + + foreach (var (name, body) in PredicatesOver(kind, x)) + { + foreach (var quantifier in new[] { nameof(Enumerable.Any), nameof(Enumerable.All) }) + calls.Add((new Call($"@list.{quantifier}(x => {name})", Expression.Call(typeof(Enumerable), quantifier, [x.Type], list, Expression.Lambda(body, x))), kind, holder)); + calls.Add((new Call($"@list.Count(x => {name}) == 0", + Expression.Equal(Expression.Call(typeof(Enumerable), nameof(Enumerable.Count), [x.Type], list, Expression.Lambda(body, x)), Expression.Constant(0))), + kind, holder)); + } + + calls.Add((new Call($"@list.Contains({ColumnFor(kind)!.Name})", + Expression.Call(typeof(Enumerable), nameof(Enumerable.Contains), [x.Type], list, ColumnFor(kind)!.Value)), kind, holder)); + } + + var failures = new List(); + var untranslated = new HashSet(); + var unanswered = new HashSet(); + var judged = 0; + foreach (var (call, kind, holder) in calls) + { + foreach (var (step, (listName, list)) in Lists[kind].Index()) + { + holder.Value = list; + switch (await CompositionRunner.Judge($"execution {step + 1}, @list = {listName}", call, Row, context.Rows.AsNoTracking(), loaded, + r => r.Id, id => Names[id], failures)) + { + case Outcome.Compared: judged++; break; + case Outcome.Untranslated: untranslated.Add(call.Name); break; + case Outcome.Unanswered: unanswered.Add(call.Name); break; + } + } + } + + + Assert.IsGreaterThanOrEqualTo(500, judged, $"Only {judged} executions were judged; untranslated: {string.Join("; ", untranslated.Order())}; unanswered: {string.Join("; ", unanswered.Order())}"); + + Candidates.Verdict(nameof(RepeatedExecutionProbeTests), failures, + "A query run again with its captured list changed: PostgreSQL and rule (c) disagree on a later execution", + method: nameof(CapturedList_WalkedThroughEmptyOneDuplicateAndNull_AnswersAsRuleCEveryTime)); + } + + /// + /// Every binary predicate of the first test as an EF.CompileQuery and an + /// EF.CompileAsyncQuery delegate taking the operand as its argument, filtered and negated, run + /// once per value of the walk. + /// + [TestMethod] + public async Task CompiledQuery_CalledWithNullFiniteEmptyAndInfinite_AnswersAsRuleCEveryTime() + { + ContainerLifecycle.RequireServer(); + + var (context, loaded) = await Database(); + await using var _ = context; + + var failures = new List(); + var compiled = 0; + var calls = 0; + + foreach (var (name, kind, body, holder) in Bodies().Where(b => b.Body.Type == typeof(bool))) + { + var argument = Expression.Parameter(Declared(kind), "p"); + var over = new Substitute(holder, argument).Visit(body)!; + + foreach (var (form, filter) in new[] { ("Where", over), ("Where(!)", Expression.Not(over)) }) + { + var c = Expression.Parameter(typeof(RepeatContext), "c"); + var query = Expression.Lambda( + typeof(Func<,,>).MakeGenericType(typeof(RepeatContext), argument.Type, typeof(IQueryable)), + Expression.Call(typeof(Queryable), nameof(Queryable.Select), [typeof(RepeatRow), typeof(int)], + Expression.Call(typeof(Queryable), nameof(Queryable.Where), [typeof(RepeatRow)], + Expression.Property(c, nameof(RepeatContext.Rows)), Expression.Quote(Expression.Lambda(filter, Row))), + Expression.Quote((Expression>)(r => r.Id))), + c, argument); + + var syncQuery = Compile(nameof(EF.CompileQuery), argument.Type, query); + var asyncQuery = Compile(nameof(EF.CompileAsyncQuery), argument.Type, query); + var reference = NullRule.Reference(form == "Where" ? body : Expression.Not(body), Row); + var translated = true; + + foreach (var (variant, run) in new (string, Func>>)[] + { + ("EF.CompileQuery", value => Task.FromResult(((IEnumerable)syncQuery.DynamicInvoke(context, value)!).ToList())), + ("EF.CompileAsyncQuery", async value => + { + var ids = new List(); + await foreach (var id in (IAsyncEnumerable)asyncQuery.DynamicInvoke(context, value)!) ids.Add(id); + return ids; + }) + }) + { + foreach (var (step, (valueName, value)) in Values[kind].Index()) + { + if (!translated) break; + holder.Value = value; + var expected = loaded.Where(r => reference(r) is true).Select(r => r.Id).ToHashSet(); + try + { + var server = (await run(value)).ToHashSet(); + compiled++; + foreach (var id in loaded.Select(r => r.Id).Where(id => server.Contains(id) != expected.Contains(id))) + failures.Add($"{name} [{variant}, call {step + 1}, p = {valueName}] {form}, row {Names[id]}: server " + + $"{(server.Contains(id) ? "selects" : "drops")} it, rule (c) {(expected.Contains(id) ? "selects" : "drops")} it"); + } + catch (TargetInvocationException e) when (e.InnerException is InvalidOperationException { Message: var message } + && message.Contains("could not be translated", StringComparison.Ordinal)) + { + translated = false; + } + catch (Exception e) when (e is InvalidOperationException or DbException or TargetInvocationException) + { + var inner = e is TargetInvocationException { InnerException: { } i } ? i : e; + failures.Add($"{name} [{variant}, call {step + 1}, p = {valueName}] {form}: server threw {inner.GetType().Name}: {CompositionRunner.FirstLine(inner)}"); + } + } + } + + if (translated) calls++; + } + } + + + Assert.IsGreaterThanOrEqualTo(1000, compiled, $"Only {compiled} compiled executions ran, over {calls} compiled queries"); + + Candidates.Verdict(nameof(RepeatedExecutionProbeTests), failures, + "A compiled query called with a changed argument: PostgreSQL and rule (c) disagree", + method: nameof(CompiledQuery_CalledWithNullFiniteEmptyAndInfinite_AnswersAsRuleCEveryTime)); + } + + /// + /// Every binary member with the nullable column on one side and a captured variable of its own on + /// the other, the variable's kind, and the variable. + /// + private static List<(string Name, string Kind, Expression Body, IStrongBox Holder)> Bodies() + { + var bodies = new List<(string, string, Expression, IStrongBox)>(); + foreach (var method in Methods.Where(m => Slots(m).Count == 2)) + { + var slots = Slots(method); + for (var at = 0; at < 2; at++) + { + var column = ColumnFor(Kind(slots[1 - at])); + var kind = Kind(slots[at]); + if (column is null || kind is not ("range" or "set" or "values")) continue; + + var holder = (IStrongBox)Activator.CreateInstance(typeof(StrongBox<>).MakeGenericType(Declared(kind)))!; + var captured = Expression.Field(Expression.Constant(holder), nameof(StrongBox<>.Value)); + Expression[] operands = at == 0 ? [captured, column.Value] : [column.Value, captured]; + Expression body = method.IsSpecialName + ? Expression.MakeBinary(method.Name switch { "op_BitwiseOr" => ExpressionType.Or, "op_BitwiseAnd" => ExpressionType.And, _ => ExpressionType.Subtract }, + operands[0], operands[1], false, method) + : method.IsStatic ? Expression.Call(method, operands) : Expression.Call(operands[0], method, operands[1]); + var name = method.IsSpecialName + ? at == 0 ? $"@p {method.Name} {column.Name}" : $"{column.Name} {method.Name} @p" + : at == 0 ? $"@p.{method.Name}({column.Name})" : $"{column.Name}.{method.Name}(@p)"; + bodies.Add((name, kind, body, holder)); + } + } + + return bodies; + } + + /// Every binary predicate with the nullable column of 's family on one side and on the other. + private static IEnumerable<(string Name, Expression Body)> PredicatesOver(string kind, ParameterExpression x) + { + foreach (var method in Methods.Where(m => m.ReturnType == typeof(bool) && Slots(m).Count == 2)) + { + var slots = Slots(method); + for (var at = 0; at < 2; at++) + { + if (Kind(slots[at]) != kind || ColumnFor(Kind(slots[1 - at])) is not { } column) continue; + Expression[] operands = at == 0 ? [x, column.Value] : [column.Value, x]; + yield return (at == 0 ? $"x.{method.Name}({column.Name})" : $"{column.Name}.{method.Name}(x)", + method.IsStatic ? Expression.Call(method, operands) : Expression.Call(operands[0], method, operands[1])); + } + } + } + + private static Delegate Compile(string compiler, Type argument, LambdaExpression query) => + (Delegate)typeof(EF).GetMethods() + .Single(m => m.Name == compiler && m.GetGenericArguments().Length == 3 + && m.GetParameters()[0].ParameterType.GetGenericArguments()[0].GetGenericArguments()[^1].IsGenericType + && m.GetParameters()[0].ParameterType.GetGenericArguments()[0].GetGenericArguments()[^1].GetGenericTypeDefinition() == typeof(IQueryable<>)) + .MakeGenericMethod(typeof(RepeatContext), argument, typeof(int)) + .Invoke(null, [query])!; + + /// + /// Replaces the read of a captured variable with the compiled query's argument. EF Core evaluates a + /// compiled query's closures once, when it first runs, so a captured variable left in it would keep + /// its first value. + /// + private sealed class Substitute(IStrongBox holder, Expression argument) : ExpressionVisitor + { + protected override Expression VisitMember(MemberExpression node) => + node.Expression is ConstantExpression { Value: var value } && ReferenceEquals(value, holder) ? argument : base.VisitMember(node); + } + + public sealed class RepeatRow + { + public int Id { get; set; } + + public DateRange? Period { get; set; } + + public RangeSet? Blocks { get; set; } + + public StringSet? Tags { get; set; } + } + + private sealed class RepeatContext(string connectionString) : DbContext + { + public DbSet Rows => Set(); + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + => CompositionDatabase.Configure(optionsBuilder, connectionString); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + => modelBuilder.Entity().Property(r => r.Id).ValueGeneratedNever(); + } +} diff --git a/test/oracles/axes.md b/test/oracles/axes.md index f0cdc86f..aae8b757 100644 --- a/test/oracles/axes.md +++ b/test/oracles/axes.md @@ -52,6 +52,9 @@ request culture and configuration × every type are varied by `HostCultureProbeT | `Npgsql.DisableDateTimeInfinityConversions` | off, on | every integration project (off); `InfinityConversionsDisabledTests` (on, temporal extremes only — per process, hence its own project) | | Temporal value | sub-microsecond, the extremes, each `DateTimeKind` | `TemporalStoreParityTests`, `RangeRoundTripTests`, `ProviderConversionTests` | | Composition on a translated `Union` | every operator and clause the guard knows | `UnionCanonicalityTests` (results, not SQL) | +| Where a NULL comes from | a nullable column; the optional side of an outer join (navigation, `GroupJoin` with `DefaultIfEmpty`, `LeftJoin`) over required columns; two nullable operands, a `CASE` and a `COALESCE` of them; a nullable element read through `.Value`; a captured `null`; a list holding `null`; an aggregate over no input | `ExecutedQueryTests` (one nullable column); `OuterJoinProbeTests`; `NullableOperandPairProbeTests`; `QuantifierProbeTests` (lists); `RepeatedExecutionProbeTests` (captured `null`); `FilteredAggregateProbeTests`; `DeclaredNullabilityTests` (every emitted function, typed `NULL`s) | +| Query composition | filter, negated filter, projection; `Any`, `All`, `Count`, `Contains` over a captured list; `P && Q`, `P \|\| Q`, negated, De Morgan; a predicate over a value-producing member; `HAVING`, `HAVING NOT`, ordering by an aggregate's bound | `QuantifierProbeTests`, `FilteredAggregateProbeTests`, and `CompositionRunner`'s partition check in every composition sweep | +| Execution | once; the same query again with its parameter walked null → finite → empty → `(,)` → null and back, a list walked empty → one → duplicate → `null` → empty; `EF.CompileQuery`, `EF.CompileAsyncQuery` | every sweep (once); `RepeatedExecutionProbeTests` | | Rows written by other tools | non-canonical arrays; `infinity` spelled by another writer | `SetIntegrationTests` (arrays, E21); #117 (ranges, found once) | | Model | runtime model, compiled model | `TemporalStoreTextTests` (compiled model, wrapper family) | | Server | `postgres:18-alpine`; the surface listing for 17 and 18 | `ContainerLifecycle`; `postgres-surface.catalog.txt` | @@ -59,7 +62,8 @@ request culture and configuration × every type are varied by `HostCultureProbeT Pairs no test varies: `DisableDateTimeInfinityConversions` × JSON document × NodaTime; shape pair × every range type against PostgreSQL (three domains today; the fuzz sweep samples the rest); another writer's spelling × every read path (column, JSON document, `ExecuteUpdate`); compiled model × every wrapper family; -a second PostgreSQL major in CI. +a second PostgreSQL major in CI; the composition axes above × every range and set type (the composition +sweeps run over `DateRange`, its `RangeSet` and `StringSet`). ## Hosting and AOT (A01–A03, J31) diff --git a/test/oracles/candidates.json b/test/oracles/candidates.json index 6d457c67..95dc3d56 100644 --- a/test/oracles/candidates.json +++ b/test/oracles/candidates.json @@ -54,5 +54,313 @@ "family": "a null receiver answers in memory where SQL answers NULL", "issue": "#350", "method": "PredicatesOverANullableColumn_SelectTheRowsMemorySelects" + }, + { + "oracle": "OuterJoinProbeTests", + "row": "p.Window.Contains(c.From) [", + "family": "a NULL element makes Contains NULL, not false", + "issue": "#381", + "method": "EveryMember_OverAMissingChild_AnswersAsRuleC" + }, + { + "oracle": "OuterJoinProbeTests", + "row": "p.WindowBlocks.Contains(c.From) [", + "family": "a NULL element makes Contains NULL, not false", + "issue": "#381", + "method": "EveryMember_OverAMissingChild_AnswersAsRuleC" + }, + { + "oracle": "OuterJoinProbeTests", + "row": "p.Labels.Remove(c.Name) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "EveryMember_OverAMissingChild_AnswersAsRuleC" + }, + { + "oracle": "OuterJoinProbeTests", + "row": "c.Tags.Union(p.Labels) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "EveryMember_OverAMissingChild_AnswersAsRuleC" + }, + { + "oracle": "OuterJoinProbeTests", + "row": "c.Tags.Union(@{a}) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "EveryMember_OverAMissingChild_AnswersAsRuleC" + }, + { + "oracle": "OuterJoinProbeTests", + "row": "p.Labels.Union(c.Tags) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "EveryMember_OverAMissingChild_AnswersAsRuleC" + }, + { + "oracle": "OuterJoinProbeTests", + "row": "DateRange.CreateFinite(c.From, c.To) [", + "family": "a range factory over a NULL element builds a range, not NULL", + "issue": "#382", + "method": "EveryMember_OverAMissingChild_AnswersAsRuleC" + }, + { + "oracle": "OuterJoinProbeTests", + "row": "DateRange.CreateFinite(c.From, c.To).IsEmpty() [", + "family": "a range factory over a NULL element builds a range, not NULL", + "issue": "#382", + "method": "EveryMember_OverAMissingChild_AnswersAsRuleC" + }, + { + "oracle": "OuterJoinProbeTests", + "row": "DateRange.CreateUnboundedStart(c.To) [", + "family": "a range factory over a NULL element builds a range, not NULL", + "issue": "#382", + "method": "EveryMember_OverAMissingChild_AnswersAsRuleC" + }, + { + "oracle": "OuterJoinProbeTests", + "row": "DateRange.CreateUnboundedStart(c.To).Contains(@2026-03-07) [", + "family": "a range factory over a NULL element builds a range, not NULL", + "issue": "#382", + "method": "EveryMember_OverAMissingChild_AnswersAsRuleC" + }, + { + "oracle": "OuterJoinProbeTests", + "row": "DateRange.CreateUnboundedEnd(c.From) [", + "family": "a range factory over a NULL element builds a range, not NULL", + "issue": "#382", + "method": "EveryMember_OverAMissingChild_AnswersAsRuleC" + }, + { + "oracle": "OuterJoinProbeTests", + "row": "DateRange.CreateUnboundedEnd(c.From).Contains(@2026-03-07) [", + "family": "a range factory over a NULL element builds a range, not NULL", + "issue": "#382", + "method": "EveryMember_OverAMissingChild_AnswersAsRuleC" + }, + { + "oracle": "NullableOperandPairProbeTests", + "row": "r.A.Contains(r.Day.Value) [", + "family": "a NULL element makes Contains NULL, not false", + "issue": "#381", + "method": "EveryBinaryMember_OverTwoNullableOperands_AnswersAsRuleC" + }, + { + "oracle": "NullableOperandPairProbeTests", + "row": "(r.Flag ? r.A : r.B).Contains(r.Day.Value) [", + "family": "a NULL element makes Contains NULL, not false", + "issue": "#381", + "method": "EveryBinaryMember_OverTwoNullableOperands_AnswersAsRuleC" + }, + { + "oracle": "NullableOperandPairProbeTests", + "row": "(r.A ?? r.B).Contains(r.Day.Value) [", + "family": "a NULL element makes Contains NULL, not false", + "issue": "#381", + "method": "EveryBinaryMember_OverTwoNullableOperands_AnswersAsRuleC" + }, + { + "oracle": "NullableOperandPairProbeTests", + "row": "r.SetA.Contains(r.Day.Value) [", + "family": "a NULL element makes Contains NULL, not false", + "issue": "#381", + "method": "EveryBinaryMember_OverTwoNullableOperands_AnswersAsRuleC" + }, + { + "oracle": "NullableOperandPairProbeTests", + "row": "(r.Flag ? r.SetA : r.SetB).Contains(r.Day.Value) [", + "family": "a NULL element makes Contains NULL, not false", + "issue": "#381", + "method": "EveryBinaryMember_OverTwoNullableOperands_AnswersAsRuleC" + }, + { + "oracle": "NullableOperandPairProbeTests", + "row": "(r.SetA ?? r.SetB).Contains(r.Day.Value) [", + "family": "a NULL element makes Contains NULL, not false", + "issue": "#381", + "method": "EveryBinaryMember_OverTwoNullableOperands_AnswersAsRuleC" + }, + { + "oracle": "NullableOperandPairProbeTests", + "row": "r.TagsA.Remove(r.Label) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "EveryBinaryMember_OverTwoNullableOperands_AnswersAsRuleC" + }, + { + "oracle": "NullableOperandPairProbeTests", + "row": "(r.Flag ? r.TagsA : r.TagsB).Remove(r.Label) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "EveryBinaryMember_OverTwoNullableOperands_AnswersAsRuleC" + }, + { + "oracle": "NullableOperandPairProbeTests", + "row": "(r.TagsA ?? r.TagsB).Remove(r.Label) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "EveryBinaryMember_OverTwoNullableOperands_AnswersAsRuleC" + }, + { + "oracle": "NullableOperandPairProbeTests", + "row": "r.TagsA.Union(r.TagsB) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "EveryBinaryMember_OverTwoNullableOperands_AnswersAsRuleC" + }, + { + "oracle": "NullableOperandPairProbeTests", + "row": "r.TagsA.Union((r.Flag ? r.TagsB : r.TagsA)) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "EveryBinaryMember_OverTwoNullableOperands_AnswersAsRuleC" + }, + { + "oracle": "NullableOperandPairProbeTests", + "row": "r.TagsA.Union((r.TagsB ?? r.TagsA)) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "EveryBinaryMember_OverTwoNullableOperands_AnswersAsRuleC" + }, + { + "oracle": "NullableOperandPairProbeTests", + "row": "(r.Flag ? r.TagsA : r.TagsB).Union(r.TagsB) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "EveryBinaryMember_OverTwoNullableOperands_AnswersAsRuleC" + }, + { + "oracle": "NullableOperandPairProbeTests", + "row": "(r.Flag ? r.TagsA : r.TagsB).Union((r.Flag ? r.TagsB : r.TagsA)) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "EveryBinaryMember_OverTwoNullableOperands_AnswersAsRuleC" + }, + { + "oracle": "NullableOperandPairProbeTests", + "row": "(r.Flag ? r.TagsA : r.TagsB).Union((r.TagsB ?? r.TagsA)) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "EveryBinaryMember_OverTwoNullableOperands_AnswersAsRuleC" + }, + { + "oracle": "NullableOperandPairProbeTests", + "row": "(r.TagsA ?? r.TagsB).Union(r.TagsB) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "EveryBinaryMember_OverTwoNullableOperands_AnswersAsRuleC" + }, + { + "oracle": "NullableOperandPairProbeTests", + "row": "(r.TagsA ?? r.TagsB).Union((r.Flag ? r.TagsB : r.TagsA)) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "EveryBinaryMember_OverTwoNullableOperands_AnswersAsRuleC" + }, + { + "oracle": "QuantifierProbeTests", + "row": "@[[3,8]].All(x => x.Contains(r.Day.Value)) [", + "family": "a NULL element makes Contains NULL, not false", + "issue": "#381", + "method": "Quantifiers_OverCapturedLists_AnswerAsRuleCAndAgreeWithEachOther" + }, + { + "oracle": "QuantifierProbeTests", + "row": "@[[3,8],[3,8]].All(x => x.Contains(r.Day.Value)) [", + "family": "a NULL element makes Contains NULL, not false", + "issue": "#381", + "method": "Quantifiers_OverCapturedLists_AnswerAsRuleCAndAgreeWithEachOther" + }, + { + "oracle": "QuantifierProbeTests", + "row": "@[empty,(,),[3,8],[20,)].All(x => x.Contains(r.Day.Value)) [", + "family": "a NULL element makes Contains NULL, not false", + "issue": "#381", + "method": "Quantifiers_OverCapturedLists_AnswerAsRuleCAndAgreeWithEachOther" + }, + { + "oracle": "QuantifierProbeTests", + "row": "@[{[3,8]}].All(x => x.Contains(r.Day.Value)) [", + "family": "a NULL element makes Contains NULL, not false", + "issue": "#381", + "method": "Quantifiers_OverCapturedLists_AnswerAsRuleCAndAgreeWithEachOther" + }, + { + "oracle": "QuantifierProbeTests", + "row": "@[{[3,8]},{[3,8]}].All(x => x.Contains(r.Day.Value)) [", + "family": "a NULL element makes Contains NULL, not false", + "issue": "#381", + "method": "Quantifiers_OverCapturedLists_AnswerAsRuleCAndAgreeWithEachOther" + }, + { + "oracle": "QuantifierProbeTests", + "row": "@[{},{(,)}].All(x => x.Contains(r.Day.Value)) [", + "family": "a NULL element makes Contains NULL, not false", + "issue": "#381", + "method": "Quantifiers_OverCapturedLists_AnswerAsRuleCAndAgreeWithEachOther" + }, + { + "oracle": "RepeatedExecutionProbeTests", + "row": "@p.Union(r.Tags) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "CapturedOperand_WalkedThroughNullFiniteEmptyAndInfinite_AnswersAsRuleCEveryTime" + }, + { + "oracle": "RepeatedExecutionProbeTests", + "row": "r.Tags.Union(@p) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "CapturedOperand_WalkedThroughNullFiniteEmptyAndInfinite_AnswersAsRuleCEveryTime" + }, + { + "oracle": "DeclaredNullabilityTests", + "row": "r.Tags.Remove(r.Label) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "EveryTranslatedValue_IsNullInAFilterExactlyWhereItIsNullInAProjection" + }, + { + "oracle": "DeclaredNullabilityTests", + "row": "r.RequiredTags.Remove(r.Label) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "EveryTranslatedValue_IsNullInAFilterExactlyWhereItIsNullInAProjection" + }, + { + "oracle": "DeclaredNullabilityTests", + "row": "r.Tags.Union(@{a}) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "EveryTranslatedValue_IsNullInAFilterExactlyWhereItIsNullInAProjection" + }, + { + "oracle": "DeclaredNullabilityTests", + "row": "r.Tags.Union(@null) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "EveryTranslatedValue_IsNullInAFilterExactlyWhereItIsNullInAProjection" + }, + { + "oracle": "DeclaredNullabilityTests", + "row": "r.RequiredTags.Union(@null) [", + "family": "value set Union or Remove over a NULL operand projects a value where its NULL test answers NULL", + "issue": "#360", + "method": "EveryTranslatedValue_IsNullInAFilterExactlyWhereItIsNullInAProjection" + }, + { + "oracle": "DeclaredNullabilityTests", + "row": "array_cat(text[], text[]):", + "family": "value set Union or Remove is declared NULL for a NULL argument, but array_cat and array_remove are not strict", + "issue": "#360", + "method": "EveryEmittedFunction_IsDeclaredAsPostgresEvaluatesIt" + }, + { + "oracle": "DeclaredNullabilityTests", + "row": "array_remove(text[], text):", + "family": "value set Union or Remove is declared NULL for a NULL argument, but array_cat and array_remove are not strict", + "issue": "#360", + "method": "EveryEmittedFunction_IsDeclaredAsPostgresEvaluatesIt" } ] diff --git a/test/oracles/claims.json b/test/oracles/claims.json index a5fc1863..194f8d91 100644 --- a/test/oracles/claims.json +++ b/test/oracles/claims.json @@ -183,7 +183,8 @@ "heldBy": [ "RangeAggregateParityTests", "RangeAggregateTests", - "ExecutedQueryTests" + "ExecutedQueryTests", + "FilteredAggregateProbeTests" ] }, { @@ -1220,7 +1221,9 @@ "RangeSet.UpperBound" ], "heldBy": [ - "ExecutedQueryTests" + "ExecutedQueryTests", + "OuterJoinProbeTests", + "DeclaredNullabilityTests" ] }, { @@ -1237,7 +1240,11 @@ ], "heldBy": [ "ExecutedQueryTests", - "NullOperandTranslationTests" + "NullOperandTranslationTests", + "OuterJoinProbeTests", + "NullableOperandPairProbeTests", + "QuantifierProbeTests", + "RepeatedExecutionProbeTests" ] }, { @@ -1253,7 +1260,8 @@ "RangeSet.UpperBound" ], "heldBy": [ - "ExecutedQueryTests" + "ExecutedQueryTests", + "FilteredAggregateProbeTests" ] }, { diff --git a/test/oracles/oracles.json b/test/oracles/oracles.json index ee376c7d..da19820e 100644 --- a/test/oracles/oracles.json +++ b/test/oracles/oracles.json @@ -358,5 +358,59 @@ "needs": [ "docker" ] + }, + { + "class": "OuterJoinProbeTests", + "project": "CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests", + "kind": "differential", + "against": "rule (c) over the rows as loaded, with the missing child guarded explicitly (NullRule), for every discovered range, RangeSet and value set member over the child of a parent without one × a navigation, GroupJoin with DefaultIfEmpty and LeftJoin × a filter, a negated filter and a projection", + "needs": [ + "docker" + ] + }, + { + "class": "NullableOperandPairProbeTests", + "project": "CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests", + "kind": "differential", + "against": "rule (c) over the rows as loaded (NullRule), for every discovered binary range, RangeSet and value set member over two nullable columns, a CASE and a COALESCE of them, on every (null, value) pair, with equality and CompareTo under their own documented rules", + "needs": [ + "docker" + ] + }, + { + "class": "QuantifierProbeTests", + "project": "CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests", + "kind": "metamorphic", + "against": "rule (c) over the rows as loaded (NullRule), and inside the database All(P) against !Any(!P), Count(P) == 0 against !Any(P) and De Morgan's forms, for Any, All, Count and Contains over captured lists (empty, one, duplicate, holding NULL, every shape) and nullable columns, compound predicates and predicates over value-producing members", + "needs": [ + "docker" + ] + }, + { + "class": "RepeatedExecutionProbeTests", + "project": "CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests", + "kind": "differential", + "against": "rule (c) over the rows as loaded (NullRule) on every execution, for every discovered binary member and quantifier with its captured operand or list walked through null, finite, empty and infinite and back, and as EF.CompileQuery and EF.CompileAsyncQuery delegates", + "needs": [ + "docker" + ] + }, + { + "class": "FilteredAggregateProbeTests", + "project": "CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests", + "kind": "differential", + "against": "RangeAgg and RangeIntersectAgg in memory over the same inputs with E07 mapped, for groups that are ordinary, filtered empty, all NULL, holding empty or (,), and a key left-joined to no rows × every discovered consumption of the result in a projection, HAVING, HAVING NOT and an ordering by a bound", + "needs": [ + "docker" + ] + }, + { + "class": "DeclaredNullabilityTests", + "project": "CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests", + "kind": "differential", + "against": "pg_proc.proisstrict and the server's answer over typed NULLs and samples, for every function the translators emit; and EF Core's null test against the projected value, for every translated value over nullable and required columns and a captured null", + "needs": [ + "docker" + ] } ] diff --git a/test/oracles/seeds/aggregates-fold-a-null-input.patch b/test/oracles/seeds/aggregates-fold-a-null-input.patch index 1e68bf94..eab5e2eb 100644 --- a/test/oracles/seeds/aggregates-fold-a-null-input.patch +++ b/test/oracles/seeds/aggregates-fold-a-null-input.patch @@ -1,6 +1,6 @@ Seed: RangeAgg and RangeIntersectAgg fold a null input in instead of skipping it, as range_agg and range_intersect_agg skip a NULL row -Oracles: RangeAggregateParityTests -Fails: Aggregates_OverANullableColumn_SkipTheNullRows, for every group with its NULL row after another input +Oracles: RangeAggregateParityTests, FilteredAggregateProbeTests +Fails: Aggregates_OverANullableColumn_SkipTheNullRows, for every group with its NULL row after another input; FilteredAggregateProbeTests, every consumption of RangeIntersectAgg over the groups holding a NULL after another input diff --git a/src/CodoMetis.ValueRanges/RangeAggregateExtensions.cs b/src/CodoMetis.ValueRanges/RangeAggregateExtensions.cs --- a/src/CodoMetis.ValueRanges/RangeAggregateExtensions.cs diff --git a/test/oracles/seeds/bound-declared-null-only-for-a-null-argument.patch b/test/oracles/seeds/bound-declared-null-only-for-a-null-argument.patch index 385c98bf..e5176e21 100644 --- a/test/oracles/seeds/bound-declared-null-only-for-a-null-argument.patch +++ b/test/oracles/seeds/bound-declared-null-only-for-a-null-argument.patch @@ -1,6 +1,6 @@ Seed: lower() and upper() are declared NULL only for a NULL argument again, so EF Core takes the bound of a required column as never NULL (#349) -Oracles: ExecutedQueryTests -Fails: BoundsComparedWithNull_SelectTheRowsMemorySelects on every operation and column, and OrderingByABound_PutsTheRowsWithoutItWherePostgresPutsNulls on the Bound() != null key, which folds to TRUE +Oracles: ExecutedQueryTests, OuterJoinProbeTests, NullableOperandPairProbeTests, FilteredAggregateProbeTests, DeclaredNullabilityTests +Fails: BoundsComparedWithNull_SelectTheRowsMemorySelects on every operation and column, and OrderingByABound_PutsTheRowsWithoutItWherePostgresPutsNulls on the Bound() != null key, which folds to TRUE; OuterJoinProbeTests, LowerBound() and UpperBound() == null on the child ranges without that bound; NullableOperandPairProbeTests, the same over a CASE and a COALESCE of two columns; FilteredAggregateProbeTests, the bounds of an aggregate holding empty; DeclaredNullabilityTests, lower(empty) and upper(empty) against the declaration, and LowerBound() == null on the required column diff --git a/src/CodoMetis.ValueRanges.EFCore.PostgreSQL/Query/ValueRangesMethodCallTranslator.cs b/src/CodoMetis.ValueRanges.EFCore.PostgreSQL/Query/ValueRangesMethodCallTranslator.cs --- a/src/CodoMetis.ValueRanges.EFCore.PostgreSQL/Query/ValueRangesMethodCallTranslator.cs diff --git a/test/oracles/seeds/range-agg-declared-never-null.patch b/test/oracles/seeds/range-agg-declared-never-null.patch new file mode 100644 index 00000000..0d5172db --- /dev/null +++ b/test/oracles/seeds/range-agg-declared-never-null.patch @@ -0,0 +1,17 @@ +Seed: range_agg is declared never NULL, so EF Core drops the NULL test of a RangeAgg over a group whose inputs are all filtered away, NULL or absent +Oracles: FilteredAggregateProbeTests +Fails: RangeAgg() == null, ?? and every predicate under HAVING NOT on the groups with no non-NULL input + +diff --git a/src/CodoMetis.ValueRanges.EFCore.PostgreSQL/Query/ValueRangesAggregateMethodCallTranslator.cs b/src/CodoMetis.ValueRanges.EFCore.PostgreSQL/Query/ValueRangesAggregateMethodCallTranslator.cs +index 950f649..bcde0ee 100644 +--- a/src/CodoMetis.ValueRanges.EFCore.PostgreSQL/Query/ValueRangesAggregateMethodCallTranslator.cs ++++ b/src/CodoMetis.ValueRanges.EFCore.PostgreSQL/Query/ValueRangesAggregateMethodCallTranslator.cs +@@ -49,7 +49,7 @@ internal sealed class ValueRangesAggregateMethodCallTranslator( + "range_agg", + [argument], + source, +- nullable: true, ++ nullable: false, + argumentsPropagateNullability: [false], + definition.RangeSetClrType, + definition.RangeSetTypeMapping), diff --git a/test/oracles/seeds/range-predicate-over-a-null-column-is-null.patch b/test/oracles/seeds/range-predicate-over-a-null-column-is-null.patch index 2f8ec286..4adb0faf 100644 --- a/test/oracles/seeds/range-predicate-over-a-null-column-is-null.patch +++ b/test/oracles/seeds/range-predicate-over-a-null-column-is-null.patch @@ -1,6 +1,6 @@ Seed: a range or multirange predicate over a NULL column is NULL in SQL, so its negation drops the row EF Core keeps for its own types -Oracles: ExecutedQueryTests -Fails: EveryPredicate_OverANullableColumn_IsFalseForTheNullRow, the negated filter and the projection of every predicate on the NULL row of Period and Blocks +Oracles: ExecutedQueryTests, OuterJoinProbeTests, NullableOperandPairProbeTests, QuantifierProbeTests, RepeatedExecutionProbeTests +Fails: EveryPredicate_OverANullableColumn_IsFalseForTheNullRow, the negated filter and the projection of every predicate on the NULL row of Period and Blocks; OuterJoinProbeTests, the negated filter and the projection of every range and RangeSet predicate on a parent without a child; NullableOperandPairProbeTests, the same on every row with a NULL operand; QuantifierProbeTests, the quantifiers and compound predicates over the NULL row; RepeatedExecutionProbeTests, every execution with a NULL operand diff --git a/src/CodoMetis.ValueRanges.EFCore.PostgreSQL/Query/ValueRangesMethodCallTranslator.cs b/src/CodoMetis.ValueRanges.EFCore.PostgreSQL/Query/ValueRangesMethodCallTranslator.cs --- a/src/CodoMetis.ValueRanges.EFCore.PostgreSQL/Query/ValueRangesMethodCallTranslator.cs diff --git a/test/oracles/seeds/sql-processor-reuses-its-first-output.patch b/test/oracles/seeds/sql-processor-reuses-its-first-output.patch new file mode 100644 index 00000000..4f9c8121 --- /dev/null +++ b/test/oracles/seeds/sql-processor-reuses-its-first-output.patch @@ -0,0 +1,21 @@ +Seed: the parameter-based SQL processor keeps its first output per query, so a query run again with a NULL, an empty list or a value where it first saw another reuses the SQL built for the first +Oracles: RepeatedExecutionProbeTests +Fails: every captured operand and list after its first execution, wherever the first value was NULL or empty and a later one is not, and the other way + +diff --git a/src/CodoMetis.ValueRanges.EFCore.PostgreSQL/Query/ValueRangesParameterBasedSqlProcessorFactory.cs b/src/CodoMetis.ValueRanges.EFCore.PostgreSQL/Query/ValueRangesParameterBasedSqlProcessorFactory.cs +index 3a2ead0..2f0f5cf 100644 +--- a/src/CodoMetis.ValueRanges.EFCore.PostgreSQL/Query/ValueRangesParameterBasedSqlProcessorFactory.cs ++++ b/src/CodoMetis.ValueRanges.EFCore.PostgreSQL/Query/ValueRangesParameterBasedSqlProcessorFactory.cs +@@ -78,8 +78,10 @@ internal sealed class ValueRangesParameterBasedSqlProcessorFactory( + // EF Core's SELECT of bare NULLs under WHERE FALSE, which PostgreSQL types as text, so the + // NULLs are given the column type every operator on them needs before the guard judges the + // final tree (#290). ++ private readonly System.Collections.Concurrent.ConcurrentDictionary _processed = new(ReferenceEqualityComparer.Instance); ++ + public override Expression Process(Expression queryExpression, ParametersCacheDecorator parametersDecorator) +- => ValueSetSqlGuard.Check( +- EmptyCollectionRewriter.Rewrite(inner.Process(queryExpression, parametersDecorator), Dependencies.SqlExpressionFactory)); ++ => _processed.GetOrAdd(queryExpression, query => ValueSetSqlGuard.Check( ++ EmptyCollectionRewriter.Rewrite(inner.Process(query, parametersDecorator), Dependencies.SqlExpressionFactory))); + } + } diff --git a/test/oracles/seeds/value-set-predicate-over-a-null-column-is-null.patch b/test/oracles/seeds/value-set-predicate-over-a-null-column-is-null.patch index 9af91a16..6ed13a28 100644 --- a/test/oracles/seeds/value-set-predicate-over-a-null-column-is-null.patch +++ b/test/oracles/seeds/value-set-predicate-over-a-null-column-is-null.patch @@ -1,6 +1,6 @@ Seed: a value set predicate over a NULL column is NULL in SQL, so its negation drops the row EF Core keeps for string[] -Oracles: ExecutedQueryTests -Fails: EveryPredicate_OverANullableColumn_IsFalseForTheNullRow, the negated filter and the projection of every predicate on the NULL row of Tags +Oracles: ExecutedQueryTests, OuterJoinProbeTests, NullableOperandPairProbeTests, QuantifierProbeTests, RepeatedExecutionProbeTests +Fails: EveryPredicate_OverANullableColumn_IsFalseForTheNullRow, the negated filter and the projection of every predicate on the NULL row of Tags; OuterJoinProbeTests, the negated filter and the projection of every value set predicate on a parent without a child; NullableOperandPairProbeTests, the same on every row with a NULL value set; QuantifierProbeTests, the value set quantifiers and compound predicates over the NULL row; RepeatedExecutionProbeTests, every execution with a NULL value set diff --git a/src/CodoMetis.ValueRanges.EFCore.PostgreSQL/Query/ValueSetsMethodCallTranslator.cs b/src/CodoMetis.ValueRanges.EFCore.PostgreSQL/Query/ValueSetsMethodCallTranslator.cs --- a/src/CodoMetis.ValueRanges.EFCore.PostgreSQL/Query/ValueSetsMethodCallTranslator.cs