Handle scalar rows in CSV export

CosmosDBShell.Fuzzer/ImportFuzzer.cs

@@ -116,7 +116,7 @@ private static async Task DriveParsersAsync(string content)
             var pk = ImportCommand.ParsePartitionKeySegments(RandomPartitionKey());
             for (var r = 1; r < records.Count; r++)
             {
-                _ = ImportCommand.BuildCsvObject(headers, records[r], pk);
+                _ = ImportCommand.BuildCsvObject(headers, records[r], pk, r + 1);
             }
         }
     }

CosmosDBShell.Tests/CommandTests/CsvScalarColumnEnvironmentTests.cs

@@ -0,0 +1,74 @@
+// ------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+// ------------------------------------------------------------
+
+namespace CosmosShell.Tests.CommandTests;
+
+using System.Text.Json;
+using Azure.Data.Cosmos.Shell.Commands;
+
+// Environment changes must not race with other CSV exports.
+[CollectionDefinition(Name, DisableParallelization = true)]
+public sealed class CsvScalarColumnEnvironmentTestCollection
+{
+    public const string Name = "CSV scalar column environment tests";
+}
+
+[Collection(CsvScalarColumnEnvironmentTestCollection.Name)]
+public class CsvScalarColumnEnvironmentTests
+{
+    private const string EnvironmentVariable = "COSMOSDB_SHELL_CSV_SCALAR_COLUMN";
+
+    [Theory]
+    [InlineData(null, "\"\"")]
+    [InlineData("scalar", "\"scalar\"")]
+    [InlineData("scalar,\"name", "\"scalar,\"\"name\"")]
+    public async Task WriteCsvAsync_OmittedScalarColumn_UsesEnvironmentOrEmptyDefault(string? configuredHeader, string expectedHeader)
+    {
+        var previous = Environment.GetEnvironmentVariable(EnvironmentVariable);
+        try
+        {
+            Environment.SetEnvironmentVariable(EnvironmentVariable, configuredHeader);
+            using var writer = new StringWriter { NewLine = "\n" };
+
+            var count = await ExportCommand.WriteCsvAsync(
+                ScalarRowsAsync(), writer, ',', TestContext.Current.CancellationToken);
+
+            Assert.Equal(1, count);
+            Assert.Equal($"{expectedHeader}\n\"text\"\n", writer.ToString());
+        }
+        finally
+        {
+            Environment.SetEnvironmentVariable(EnvironmentVariable, previous);
+        }
+    }
+
+    [Theory]
+    [InlineData("", "\"\"")]
+    [InlineData("explicit", "\"explicit\"")]
+    public async Task WriteCsvAsync_ExplicitScalarColumn_OverridesEnvironment(string explicitHeader, string expectedHeader)
+    {
+        var previous = Environment.GetEnvironmentVariable(EnvironmentVariable);
+        try
+        {
+            Environment.SetEnvironmentVariable(EnvironmentVariable, "configured");
+            using var writer = new StringWriter { NewLine = "\n" };
+
+            var count = await ExportCommand.WriteCsvAsync(
+                ScalarRowsAsync(), writer, ',', TestContext.Current.CancellationToken, explicitHeader);
+
+            Assert.Equal(1, count);
+            Assert.Equal($"{expectedHeader}\n\"text\"\n", writer.ToString());
+        }
+        finally
+        {
+            Environment.SetEnvironmentVariable(EnvironmentVariable, previous);
+        }
+    }
+
+    private static async IAsyncEnumerable<JsonElement> ScalarRowsAsync()
+    {
+        yield return JsonSerializer.SerializeToElement("text");
+        await Task.CompletedTask;
+    }
+}

CosmosDBShell.Tests/CommandTests/ExportCommandTests.cs

@@ -10,6 +10,7 @@ namespace CosmosShell.Tests.CommandTests;
 using System.Threading;
 using System.Threading.Tasks;
 using Azure.Data.Cosmos.Shell.Commands;
+using Azure.Data.Cosmos.Shell.Core;
 using Azure.Data.Cosmos.Shell.Util;
 using Microsoft.Azure.Cosmos;
 using NSubstitute;
@@ -204,6 +205,119 @@ public async Task WriteCsvAsync_NestedValuesWrittenAsCompactJson()
         Assert.Contains("\"{\"\"a\"\":\"\"b\"\"}\"", output);
     }
 
+    [Fact]
+    public async Task WriteCsvAsync_ObjectRowsKeepRawJsonValues()
+    {
+        var items = ToAsyncEnumerableAsync(
+            JsonSerializer.SerializeToElement(new { id = "1", flag = true, missing = (object?)null }));
+
+        using var writer = new StringWriter();
+        writer.NewLine = "\n";
+
+        var count = await ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None);
+
+        Assert.Equal(1, count);
+        var lines = writer.ToString().TrimEnd('\n').Split('\n');
+        Assert.Equal("\"id\",\"flag\",\"missing\"", lines[0]);
+        Assert.Equal("\"1\",\"true\",\"null\"", lines[1]);
+    }
+
+    [Fact]
+    public async Task WriteCsvAsync_ScalarStringsUseEmptyHeaderAndEscapeCsv()
+    {
+        var items = ToAsyncEnumerableAsync(
+            JsonSerializer.SerializeToElement("a,\"b"));
+
+        using var writer = new StringWriter();
+        writer.NewLine = "\n";
+
+        var count = await ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None, string.Empty);
+
+        Assert.Equal(1, count);
+        Assert.Equal("\"\"\n\"a,\"\"b\"\n", writer.ToString());
+    }
+
+    [Fact]
+    public async Task WriteCsvAsync_ScalarNumbersBoolsAndNullUseEmptyHeader()
+    {
+        var items = ToAsyncEnumerableAsync(
+            JsonSerializer.SerializeToElement(42),
+            JsonSerializer.SerializeToElement(true),
+            JsonSerializer.SerializeToElement(false),
+            JsonSerializer.SerializeToElement((object?)null));
+
+        using var writer = new StringWriter();
+        writer.NewLine = "\n";
+
+        var count = await ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None, string.Empty);
+
+        Assert.Equal(4, count);
+        Assert.Equal("\"\"\n\"42\"\n\"True\"\n\"False\"\n\"\"\n", writer.ToString());
+    }
+
+    [Fact]
+    public async Task WriteCsvAsync_ArrayRowsUseEmptyHeader()
+    {
+        var items = ToAsyncEnumerableAsync(
+            JsonSerializer.SerializeToElement(new[] { 1, 2 }));
+
+        using var writer = new StringWriter();
+        writer.NewLine = "\n";
+
+        var count = await ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None, string.Empty);
+
+        Assert.Equal(1, count);
+        Assert.Equal("\"\"\n\"[1,2]\"\n", writer.ToString());
+    }
+
+    [Fact]
+    public async Task WriteCsvAsync_MixedObjectAndScalarRowsPreserveScalarValue()
+    {
+        var items = ToAsyncEnumerableAsync(
+            JsonSerializer.SerializeToElement(new { id = "1" }),
+            JsonSerializer.SerializeToElement("a,b"),
+            JsonSerializer.SerializeToElement(new { id = "2", value = 99 }));
+
+        using var writer = new StringWriter();
+        writer.NewLine = "\n";
+
+        var count = await ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None, string.Empty);
+
+        Assert.Equal(3, count);
+        Assert.Equal("\"id\",\"\",\"value\"\n\"1\",\"\",\"\"\n\"\",\"a,b\",\"\"\n\"2\",\"\",\"99\"\n", writer.ToString());
+    }
+
+    [Fact]
+    public async Task WriteCsvAsync_CustomScalarHeaderPreservesObjectValueColumn()
+    {
+        var items = ToAsyncEnumerableAsync(
+            JsonSerializer.SerializeToElement(new { value = 99 }),
+            JsonSerializer.SerializeToElement("text"));
+        using var writer = new StringWriter();
+        writer.NewLine = "\n";
+
+        var count = await ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None, "scalar");
+
+        Assert.Equal(2, count);
+        Assert.Equal("\"value\",\"scalar\"\n\"99\",\"\"\n\"\",\"text\"\n", writer.ToString());
+    }
+
+    [Theory]
+    [InlineData("", "{\"\":1}")]
+    [InlineData("value", "{\"value\":1}")]
+    public async Task WriteCsvAsync_ScalarHeaderConflictingWithObjectPropertyFails(string header, string objectJson)
+    {
+        using var document = JsonDocument.Parse(objectJson);
+        var items = ToAsyncEnumerableAsync(JsonSerializer.SerializeToElement(42), document.RootElement.Clone());
+        using var writer = new StringWriter();
+
+        var error = await Assert.ThrowsAsync<CommandException>(() =>
+            ExportCommand.WriteCsvAsync(items, writer, ',', CancellationToken.None, header));
+
+        Assert.Contains("COSMOSDB_SHELL_CSV_SCALAR_COLUMN", error.Message);
+        Assert.Equal(string.Empty, writer.ToString());
+    }
+
     [Fact]
     public async Task WriteCsvAsync_WithNoItems_ProducesEmptyOutput()
     {

CosmosDBShell.Tests/CommandTests/ImportCommandTests.cs

@@ -340,13 +340,67 @@ public void BuildCsvObject_MapsColumnsToStringProperties()
         var element = ImportCommand.BuildCsvObject(
             new[] { "id", "name" },
             new[] { "1", "Alice" },
-            partitionKeySegments: null);
+            partitionKeySegments: null,
+            lineNumber: 2);
 
         Assert.Equal(JsonValueKind.Object, element.ValueKind);
         Assert.Equal("1", element.GetProperty("id").GetString());
         Assert.Equal("Alice", element.GetProperty("name").GetString());
     }
 
+    [Fact]
+    public void BuildCsvObject_EmptyUnnamedColumnIsIgnored()
+    {
+        var element = ImportCommand.BuildCsvObject(
+            new[] { "id", "" },
+            new[] { "1", "" },
+            partitionKeySegments: null,
+            lineNumber: 2);
+
+        Assert.Equal("1", element.GetProperty("id").GetString());
+        Assert.False(element.TryGetProperty("", out _));
+    }
+
+    [Fact]
+    public void BuildCsvObject_PopulatedUnnamedColumnFailsWithLocation()
+    {
+        var error = Assert.Throws<CommandException>(() => ImportCommand.BuildCsvObject(
+            new[] { "id", "" },
+            new[] { "1", "lost" },
+            partitionKeySegments: null,
+            lineNumber: 7));
+
+        Assert.Contains("7", error.Message);
+        Assert.Contains("2", error.Message);
+        Assert.Contains("header", error.Message);
+    }
+
+    [Fact]
+    public async Task EnumerateCsvAsync_PopulatedUnnamedColumnReportsPhysicalStartLine()
+    {
+        var separator = Azure.Data.Cosmos.Shell.Core.ShellInterpreter.CSVSeparator;
+        var filePath = Path.GetTempFileName();
+        try
+        {
+            await File.WriteAllTextAsync(filePath, $"id{separator}\"\"\n\"a\nb\"{separator}\"\"\n2{separator}lost\n", TestContext.Current.CancellationToken);
+
+            var error = await Assert.ThrowsAsync<CommandException>(async () =>
+            {
+                await foreach (var _ in ImportCommand.EnumerateCsvAsync(filePath, null, TestContext.Current.CancellationToken))
+                {
+                    // Enumerate to trigger parsing and validation errors.
+                }
+            });
+
+            Assert.Contains("4", error.Message);
+            Assert.Contains("2", error.Message);
+        }
+        finally
+        {
+            File.Delete(filePath);
+        }
+    }
+
     [Fact]
     public void ReadCsvRecords_ReadsValidRowsBeforeReportingMalformedRecord()
     {
@@ -464,7 +518,8 @@ public void BuildCsvObject_SingleSegmentPartitionKey_StaysTopLevel()
         var element = ImportCommand.BuildCsvObject(
             new[] { "id", "city" },
             new[] { "1", "Seattle" },
-            new[] { "city" });
+            new[] { "city" },
+            lineNumber: 2);
 
         Assert.Equal("Seattle", element.GetProperty("city").GetString());
     }
@@ -475,7 +530,8 @@ public void BuildCsvObject_NestedPartitionKey_NestsMatchingColumn()
         var element = ImportCommand.BuildCsvObject(
             new[] { "id", "city" },
             new[] { "1", "Seattle" },
-            new[] { "address", "city" });
+            new[] { "address", "city" },
+            lineNumber: 2);
 
         Assert.False(element.TryGetProperty("city", out _));
         Assert.Equal("Seattle", element.GetProperty("address").GetProperty("city").GetString());
@@ -488,7 +544,8 @@ public void BuildCsvObject_NestedPartitionKey_ConflictingScalarColumn_Throws()
         var ex = Assert.Throws<CommandException>(() => ImportCommand.BuildCsvObject(
             new[] { "id", "address", "city" },
             new[] { "1", "123 Main St", "Seattle" },
-            new[] { "address", "city" }));
+            new[] { "address", "city" },
+            lineNumber: 2));
 
         Assert.Contains("address", ex.Message, StringComparison.Ordinal);
     }
@@ -499,7 +556,8 @@ public void BuildCsvObject_MissingValues_FillWithEmptyString()
         var element = ImportCommand.BuildCsvObject(
             new[] { "id", "name", "extra" },
             new[] { "1" },
-            partitionKeySegments: null);
+            partitionKeySegments: null,
+            lineNumber: 2);
 
         Assert.Equal("1", element.GetProperty("id").GetString());
         Assert.Equal(string.Empty, element.GetProperty("name").GetString());