test: composition sweeps — outer joins, nullable operand pairs, quantifiers, repeated execution, filtered aggregates, declared nullability (#375–#380)

docs/oracles.md

@@ -265,6 +265,32 @@ range collection no `Contains` reads also fails loudly, as `42809` for `Any(==)`
 found #137: `Any` or `Count` with a predicate over an empty collection the plugin sends one parameter per
 element throws.
 
+### The composition sweeps
+
+The sweeps above vary shapes, types and operand kinds, mostly in a projection over one nullable column at
+most. #349, #350, #355, #360 and #361 came from how a translated call *composes*: where a NULL comes from,
+what a filter, a negation or EF Core's null semantics do around it, and what the next execution reuses. Six
+classes in the integration project (#362) vary that instead, each over `DateRange`, its `RangeSet` and
+`StringSet`, with the members discovered (`Composition/CompositionMembers.cs`):
+
+| Class | What it composes |
+|---|---|
+| `OuterJoinProbeTests` (#375) | every member over the child of a parent without one: a navigation, `GroupJoin` with `DefaultIfEmpty`, `LeftJoin`; the range factories over the child's element columns |
+| `NullableOperandPairProbeTests` (#376) | every binary member over two nullable columns, a `CASE` and a `COALESCE` of them, on every (null, value) pair; equality and `CompareTo` under their own rules |
+| `QuantifierProbeTests` (#377) | `Any`, `All`, `Count`, `Contains` over captured lists holding a NULL; `P && Q`, `P \|\| Q` against De Morgan; a predicate over a value-producing member |
+| `RepeatedExecutionProbeTests` (#378) | the same query run again with its parameter walked null → finite → empty → `(,)` → null and back; `EF.CompileQuery` and `EF.CompileAsyncQuery` |
+| `FilteredAggregateProbeTests` (#379) | `RangeAgg` and `RangeIntersectAgg` over groups a filter empties, all NULL, holding `empty`, or reached by a `LeftJoin` with no rows, consumed by every member, `HAVING` and an ordering |
+| `DeclaredNullabilityTests` (#380) | every function the translators emit, its declaration against `pg_proc.proisstrict` and the server; and every translated value's null test against its projection |
+
+Their reference is `Composition/NullRule.cs`: the expression rewritten so that a NULL never reaches the
+library, the explicit guard a consumer writes over loaded rows. A predicate with a null on either side is
+`false` and its negation `true` (rule (c)); a value-producing member over a null, or over a missing element,
+has no value; equality and `CompareTo` keep their documented rules; the aggregates skip a null. So what the
+library does *today* for a null in memory (#350's 11.0.0 half) is never what a row is judged against. Beside
+the reference, `CompositionRunner` checks every call without one: `Where(P)` and `Where(!P)` partition the
+rows, and `Select(P)` is `true` exactly on `Where(P)`'s; for a value, the same with `== null`. A
+declaration that disagrees with the server shows there even where memory has no answer.
+
 ## How to use it
 
 **Run the oracles.** They are ordinary tests; `dotnet test` runs them all. The registered classes are

test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/Composition/CompositionDatabase.cs

@@ -0,0 +1,27 @@
+using Microsoft.EntityFrameworkCore;
+using Npgsql;
+
+namespace CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests.Composition;
+
+/// <summary>
+/// A database of its own per composition sweep, so its nullable columns and joined tables never change
+/// the shared schema and its rows never meet another test's.
+/// </summary>
+internal static class CompositionDatabase
+{
+    public static async Task<string> Create(string prefix)
+    {
+        var name = $"{prefix}_{Guid.NewGuid():N}";
+        await using (var connection = new NpgsqlConnection(ContainerLifecycle.ServerConnectionString))
+        {
+            await connection.OpenAsync();
+            await using var command = new NpgsqlCommand($"CREATE DATABASE {name}", connection);
+            await command.ExecuteNonQueryAsync();
+        }
+
+        return new NpgsqlConnectionStringBuilder(ContainerLifecycle.ServerConnectionString) { Database = name }.ConnectionString;
+    }
+
+    public static void Configure(DbContextOptionsBuilder optionsBuilder, string connectionString)
+        => optionsBuilder.UseNpgsql(connectionString, npgsql => npgsql.UseValueRanges());
+}

test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/Composition/CompositionMembers.cs

@@ -0,0 +1,171 @@
+using System.Linq.Expressions;
+using System.Reflection;
+
+namespace CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests.Composition;
+
+/// <summary>An operand a call can be given: its readable name, the expression, and whether it can be NULL.</summary>
+internal sealed record Operand(string Name, Expression Value, bool Nullable);
+
+/// <summary>One call of a library member with its operands spliced in, written as C#.</summary>
+internal sealed record Call(string Name, Expression Body)
+{
+    public bool Predicate => Body.Type == typeof(bool);
+}
+
+/// <summary>
+/// The members of the library a query can call, discovered rather than listed, closed over one family:
+/// <see cref="DateRange"/>, its <see cref="RangeSet{TRange,T}"/> and <see cref="StringSet"/>. The
+/// composition sweeps vary how a call is composed (the join, the operands' nullability, the quantifier,
+/// the execution, the aggregate), not the type; the type axis is the other sweeps'.
+/// </summary>
+/// <remarks>
+/// Every public method of <see cref="RangeExtensions"/>, of <see cref="RangeSet{TRange,T}"/> (with its
+/// operators) and of <see cref="ValueSetExtensions"/>, plus the value set's <c>Count</c> and <c>IsEmpty</c>
+/// properties. A member EF Core cannot translate is found by the sweep (it throws "could not be
+/// translated") and counted, not judged.
+/// </remarks>
+internal static class CompositionMembers
+{
+    public static readonly Type RangeType    = typeof(DateRange);
+    public static readonly Type ElementType  = typeof(DateOnly);
+    public static readonly Type SetType      = typeof(RangeSet<DateRange, DateOnly>);
+    public static readonly Type ValueSetType = typeof(StringSet);
+
+    private static readonly string[] NotQueryable =
+        [nameof(Equals), nameof(GetHashCode), nameof(ToString), "GetEnumerator", "CompareTo", "Parse", "TryParse", "Create", "From", "op_Equality", "op_Inequality"];
+
+    /// <summary>The members, closed over the family; a static one takes its receiver first.</summary>
+    public static IReadOnlyList<MethodInfo> Methods { get; } = Discover();
+
+    private static List<MethodInfo> Discover()
+    {
+        var methods = new List<MethodInfo>();
+
+        foreach (var (declaring, one, two) in new[]
+                 {
+                     (typeof(RangeExtensions), new[] { ElementType }, new[] { RangeType, ElementType }),
+                     (typeof(ValueSetExtensions), new[] { typeof(string) }, new[] { ValueSetType, typeof(string) })
+                 })
+        {
+            methods.AddRange(declaring.GetMethods(BindingFlags.Public | BindingFlags.Static)
+                                      .Select(method => Close(method, one, two))
+                                      .OfType<MethodInfo>()
+                                      .Where(method => method.GetParameters().Length is 1 or 2));
+        }
+
+        methods.AddRange(SetType.GetMethods(BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static | BindingFlags.DeclaredOnly)
+                            .Where(method => !method.IsSpecialName || method.Name.StartsWith("op_", StringComparison.Ordinal))
+                            .Where(method => method.GetParameters().Length <= (method.IsStatic ? 2 : 1)));
+
+        return [.. methods.Where(method => !NotQueryable.Contains(method.Name))
+                          .Where(method => method.ReturnType != typeof(void))
+                          .OrderBy(method => method.DeclaringType!.Name, StringComparer.Ordinal)
+                          .ThenBy(method => method.Name, StringComparer.Ordinal)
+                          .ThenBy(method => string.Join(",", method.GetParameters().Select(p => p.ParameterType.Name)), StringComparer.Ordinal)];
+    }
+
+    private static MethodInfo? Close(MethodInfo method, Type[] one, Type[] two)
+    {
+        try
+        {
+            return method.GetGenericArguments().Length switch
+            {
+                0 => method,
+                1 => method.MakeGenericMethod(one),
+                2 => method.MakeGenericMethod(two),
+                _ => null
+            };
+        }
+        catch (ArgumentException)
+        {
+            return null; // a constraint the family does not meet
+        }
+    }
+
+    /// <summary>
+    /// The parameters of <paramref name="method"/> in call order, the receiver first: the instance for
+    /// an instance method, the first argument otherwise.
+    /// </summary>
+    public static IReadOnlyList<Type> Slots(MethodInfo method) =>
+        [.. method.IsStatic ? [] : new[] { method.DeclaringType! }, .. method.GetParameters().Select(p => p.ParameterType)];
+
+    /// <summary>
+    /// Every call of every member with the operands <paramref name="operands"/> offers per slot, kept
+    /// when <paramref name="keep"/> accepts its operands (typically: one of them can be NULL), plus the
+    /// value set's <c>Count</c> and <c>IsEmpty</c> on each operand the first slot of a value set takes.
+    /// </summary>
+    public static List<Call> Calls(Func<Type, int, IEnumerable<Operand>> operands, Func<IReadOnlyList<Operand>, bool> keep)
+    {
+        var calls = new List<Call>();
+
+        foreach (var method in Methods)
+        {
+            var slots = Slots(method);
+            foreach (var combination in Product(slots.Select((type, index) => operands(type, index).ToList()).ToList()))
+            {
+                if (!keep(combination)) continue;
+                calls.Add(new Call(Name(method, combination), Build(method, combination.Select(o => o.Value).ToList())));
+            }
+        }
+
+        foreach (var receiver in operands(ValueSetType, 0).Where(o => keep([o])))
+        {
+            calls.Add(new Call($"{receiver.Name}.Count", Expression.Property(receiver.Value, nameof(StringSet.Count))));
+            calls.Add(new Call($"{receiver.Name}.IsEmpty", Expression.Property(receiver.Value, nameof(StringSet.IsEmpty))));
+        }
+
+        return calls;
+    }
+
+    private static Expression Build(MethodInfo method, IReadOnlyList<Expression> values)
+    {
+        if (method.IsSpecialName && method.Name.StartsWith("op_", StringComparison.Ordinal))
+        {
+            var type = method.Name switch
+            {
+                "op_BitwiseOr"  => ExpressionType.Or,
+                "op_BitwiseAnd" => ExpressionType.And,
+                "op_Subtraction" => ExpressionType.Subtract,
+                _ => throw new NotSupportedException(method.Name)
+            };
+            return Expression.MakeBinary(type, values[0], values[1], false, method);
+        }
+
+        return method.IsStatic
+            ? Expression.Call(method, values)
+            : Expression.Call(values[0], method, values.Skip(1));
+    }
+
+    private static string Name(MethodInfo method, IReadOnlyList<Operand> operands) =>
+        method.Name switch
+        {
+            "op_BitwiseOr"   => $"{operands[0].Name} | {operands[1].Name}",
+            "op_BitwiseAnd"  => $"{operands[0].Name} & {operands[1].Name}",
+            "op_Subtraction" => $"{operands[0].Name} - {operands[1].Name}",
+            _                => $"{operands[0].Name}.{method.Name}({string.Join(", ", operands.Skip(1).Select(o => o.Name))})"
+        };
+
+    private static IEnumerable<IReadOnlyList<Operand>> Product(IReadOnlyList<List<Operand>> slots)
+    {
+        IEnumerable<IReadOnlyList<Operand>> result = [[]];
+        foreach (var slot in slots)
+            result = result.SelectMany(prefix => slot.Select(operand => (IReadOnlyList<Operand>)[.. prefix, operand])).ToList();
+        return result;
+    }
+
+    /// <summary>A captured variable, which EF Core sends as a parameter.</summary>
+    public static MemberExpression Captured<T>(T value) =>
+        Expression.Field(Expression.Constant(new System.Runtime.CompilerServices.StrongBox<T>(value)), nameof(System.Runtime.CompilerServices.StrongBox<>.Value));
+
+    /// <summary>
+    /// Which family a parameter of type <paramref name="parameter"/> takes, if any: a range, a multirange,
+    /// a value set, or an element of either.
+    /// </summary>
+    public static string? Kind(Type parameter) =>
+        parameter == ElementType                    ? "element"
+      : parameter == typeof(string)                 ? "text"
+      : parameter.IsAssignableFrom(RangeType)       ? "range"
+      : parameter == SetType                        ? "set"
+      : parameter.IsAssignableFrom(ValueSetType)    ? "values"
+      :                                               null;
+}

test/CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests/Composition/CompositionRunner.cs

@@ -0,0 +1,137 @@
+using System.Data.Common;
+using System.Linq.Expressions;
+using Microsoft.EntityFrameworkCore;
+
+namespace CodoMetis.ValueRanges.EFCore.PostgreSQL.IntegrationTests.Composition;
+
+/// <summary>A row's id and one projected value, as the server returns them.</summary>
+public sealed class Projected
+{
+    public int Id { get; set; }
+
+    public object? Value { get; set; }
+}
+
+/// <summary>What became of one call: translated and compared with the reference on at least one row, translated
+/// with no row the reference could answer (only the reference-free checks ran), or not translated.</summary>
+internal enum Outcome
+{
+    Compared,
+    Unanswered,
+    Untranslated
+}
+
+/// <summary>
+/// Runs one call in the query contexts a consumer puts it in, on the server and against
+/// <see cref="NullRule"/> over the rows as loaded, and writes one line per row that disagrees.
+/// </summary>
+internal static class CompositionRunner
+{
+    /// <summary>
+    /// A predicate in <c>Where(P)</c>, <c>Where(!P)</c> and <c>Select(P)</c>; a value tested against
+    /// <see langword="null"/> both ways and projected. Each form is compared with the reference, and the
+    /// three with each other, which needs no reference: the two filters partition the rows, and the
+    /// projection says which side each row is on. Each line starts with the call's name, then
+    /// <paramref name="context"/>, the query form and the row. A call EF Core cannot translate is counted
+    /// rather than judged. Without <paramref name="reference"/> only the reference-free checks run.
+    /// </summary>
+    public static async Task<Outcome> Judge<TRow>(
+        string context, Call call, ParameterExpression row, IQueryable<TRow> source, IReadOnlyList<TRow> loaded,
+        Expression<Func<TRow, int>> id, Func<int, string> rowName, List<string> failures, bool reference = true)
+    {
+        var idOf = id.Compile();
+
+        // A row whose reference throws has no answer to compare (a library member refusing a non-null
+        // operand), and is left out of every form.
+        var expected = new Dictionary<int, object?>();
+        if (reference)
+        {
+            var answer = NullRule.Reference<TRow>(call.Body, row);
+            foreach (var loadedRow in loaded)
+            {
+                try { expected[idOf(loadedRow)] = answer(loadedRow); }
+                catch (Exception e) when (e is ArgumentException or InvalidOperationException or OverflowException) { }
+            }
+        }
+
+        var forms = call.Predicate
+            ? new (string Form, Expression Filter, Func<object?, bool> Selected)[]
+            {
+                ("Where", call.Body, value => value is true),
+                ("Where(!)", Expression.Not(call.Body), value => value is false)
+            }
+            : [
+                ("Where(== null)", IsNull(call.Body), value => value is null),
+                ("Where(!= null)", Expression.Not(IsNull(call.Body)), value => value is not null)
+            ];
+
+        try
+        {
+            var filtered = new List<HashSet<int>>();
+            foreach (var (form, filter, selected) in forms)
+            {
+                var server = (await source.Where(Expression.Lambda<Func<TRow, bool>>(filter, row)).Select(id).ToListAsync()).ToHashSet();
+                filtered.Add(server);
+                foreach (var (rowId, value) in expected.OrderBy(pair => pair.Key))
+                {
+                    if (server.Contains(rowId) != selected(value))
+                        failures.Add($"{call.Name} [{context}] {form}, row {rowName(rowId)}: server {Selects(server.Contains(rowId))} it, rule (c) {Selects(selected(value))} it");
+                }
+            }
+
+            var projection = Expression.MemberInit(
+                Expression.New(typeof(Projected)),
+                Expression.Bind(typeof(Projected).GetProperty(nameof(Projected.Id))!, Splice(id, row)),
+                Expression.Bind(typeof(Projected).GetProperty(nameof(Projected.Value))!,
+                                Expression.Convert(Expression.Convert(call.Body, NullRule.Lift(call.Body.Type)), typeof(object))));
+            var projected = await source.Select(Expression.Lambda<Func<TRow, Projected>>(projection, row)).ToListAsync();
+            foreach (var item in projected.OrderBy(p => p.Id))
+            {
+                if (expected.TryGetValue(item.Id, out var value) && !Equals(item.Value, value))
+                    failures.Add($"{call.Name} [{context}] Select, row {rowName(item.Id)}: server {Show(item.Value)}, rule (c) {Show(value)}");
+
+                var (first, second) = (filtered[0].Contains(item.Id), filtered[1].Contains(item.Id));
+                if (first == second || first != forms[0].Selected(item.Value))
+                    failures.Add($"{call.Name} [{context}] partition, row {rowName(item.Id)}: {forms[0].Form} {Selects(first)} it, "
+                               + $"{forms[1].Form} {Selects(second)} it, Select {Show(item.Value)}");
+            }
+
+            return expected.Count > 0 || !reference ? Outcome.Compared : Outcome.Unanswered;
+        }
+        catch (InvalidOperationException e) when (e.Message.Contains("could not be translated", StringComparison.Ordinal))
+        {
+            return Outcome.Untranslated;
+        }
+        catch (Exception e) when (e is InvalidOperationException or DbException or InvalidCastException)
+        {
+            failures.Add($"{call.Name} [{context}]: server threw {e.GetType().Name}: {FirstLine(e)}");
+            return Outcome.Compared;
+        }
+    }
+
+    /// <summary><paramref name="value"/> <c>== null</c>, lifting a value type so the test can be written.</summary>
+    internal static Expression IsNull(Expression value)
+    {
+        var type = NullRule.Lift(value.Type);
+        return Expression.Equal(type == value.Type ? value : Expression.Convert(value, type), Expression.Constant(null, type));
+    }
+
+    internal static Expression Splice<TRow, TOut>(Expression<Func<TRow, TOut>> lambda, ParameterExpression row) =>
+        new Rebind(lambda.Parameters[0], row).Visit(lambda.Body);
+
+    internal static string Show(object? value) => value switch
+    {
+        null   => "NULL",
+        bool b => b ? "true" : "false",
+        _      => value.ToString() ?? "NULL"
+    };
+
+    internal static string FirstLine(Exception e) => e.Message.Split('\n')[0];
+
+    private static string Selects(bool selected) => selected ? "selects" : "drops";
+
+    private sealed class Rebind(ParameterExpression from, Expression to) : ExpressionVisitor
+    {
+        protected override Expression VisitParameter(ParameterExpression node) => node == from ? to : node;
+    }
+}