Fix four scripting parser and value-conversion bugs

CHANGELOG.md

@@ -26,6 +26,10 @@
 
 ### Fixes
 
+- `for` and `do` loops now reject misspelled `in` and `while` keywords before any statements in the input execute. They previously accepted any identifier in those positions.
+- `filter` now rejects array index literals larger than `2147483647`, including optional access, instead of silently selecting element zero. Representable indexes beyond the array length still return `null`.
+- Decimal arithmetic and comparisons involving JSON numeric strings now use the same culture-invariant `double` conversion as shell strings instead of failing with a `Decimal`-to-`Double` cast error. String concatenation with `+` is unchanged.
+- `exec` now accepts normal command options, negative arguments, URLs, and file patterns while preserving variable, JSON, and parenthesized-expression arguments. Options previously failed to parse, including the documented `exec $cmd -m 5` form.
 - `mkdb`, `mkcon`, `create database`, and `create container` now work on serverless accounts. They previously requested autoscale throughput even when `--scale` and `--ru` were omitted, which serverless accounts reject. Omitting both options now creates the resource without throughput settings; supplying either option on a serverless account fails with an explanation. Provisioned accounts keep the existing autoscale default of 1000 RU/s. ([#218](https://github.com/Azure/CosmosDBShell/issues/218))
 - Vector `ORDER BY`, `ORDER BY RANK` relevance ranking, and object-shaped `DISTINCT` projections no longer fail with a continuation-token error. These query pipelines execute successfully but cannot export a resumable token, which was previously reported as a command failure. Such queries now return their documents; through MCP they keep reading until the requested limit instead of stopping after one page, and a truncated result is reported as `resultIncomplete` rather than as an exhausted result set. ([#219](https://github.com/Azure/CosmosDBShell/issues/219))
 - Local emulator outages are now detected across Cosmos DB commands. Requests fail promptly with an error and return the shell to its disconnected state instead of leaving an unresponsive session labeled as connected.

CosmosDBShell.Tests/CommandTests/FilterCommandTests.cs

@@ -11,6 +11,23 @@ namespace CosmosShell.Tests.CommandTests;
 
 public class FilterCommandTests
 {
+    [Theory]
+    [InlineData(".[2147483648]")]
+    [InlineData(".[2147483648]?")]
+    public async Task ExecuteAsync_OverflowingIndex_RejectsInputWithoutChangingResult(string expression)
+    {
+        using var shell = ShellInterpreter.CreateInstance();
+        var input = new ShellJson(JsonSerializer.SerializeToElement(new[] { 1, 2, 3 }));
+        var state = new CommandState { Result = input };
+        var command = new FilterCommand { ExpressionText = expression };
+
+        var error = await Assert.ThrowsAsync<CommandException>(() =>
+            command.ExecuteAsync(shell, state, string.Empty, TestContext.Current.CancellationToken));
+
+        Assert.Contains("2147483648", error.Message);
+        Assert.Same(input, state.Result);
+    }
+
     [Fact]
     public async Task ExecuteAsync_AppliesPathExpression_AndPreservesStructuredResult()
     {

CosmosDBShell.Tests/Parser/CommandExecutionTests.cs

@@ -92,6 +92,74 @@ public async Task Exec_VariableCommand_RunsResolvedCommand()
         Assert.Equal("hello world", Assert.IsType<ShellText>(state.Result).Text);
     }
 
+    [Theory]
+    [InlineData("-5", "-5")]
+    [InlineData("-1.5 +2", "-1.5 +2")]
+    [InlineData("\"--format=json\"", "--format=json")]
+    [InlineData("https://localhost:8081 *.json", "https://localhost:8081 *.json")]
+    public async Task Exec_ShellWords_RunLikeDirectArguments(string arguments, string expected)
+    {
+        var state = await Shell.RunCommandAsync(new(), $"exec \"echo\" {arguments}", TestContext.Current.CancellationToken);
+
+        Assert.False(state.IsError);
+        Assert.Equal(expected, Assert.IsType<ShellText>(state.Result).Text);
+    }
+
+    [Theory]
+    [InlineData("-directory . -l")]
+    [InlineData("--directory=. --list")]
+    [InlineData("--directory:. --list=false")]
+    [InlineData("--directory=$path --list=true")]
+    public async Task Exec_Options_BindAndRunResolvedCommand(string arguments)
+    {
+        SetVariable("cmd", new ShellText("dir"));
+        SetVariable("path", new ShellText("."));
+
+        var state = await Shell.RunCommandAsync(
+            new(), $"exec $cmd \"*.missing-regression-file\" {arguments}", TestContext.Current.CancellationToken);
+
+        Assert.False(state.IsError);
+        Assert.Equal(0, Assert.IsType<ShellJson>(state.Result).Value.GetProperty("values").GetArrayLength());
+    }
+
+    [Fact]
+    public async Task Exec_FunctionArguments_PreserveTypesAndParenthesizedExpressions()
+    {
+        var state = await Shell.RunCommandAsync(
+            new(),
+            "def capture [object flag number] { return [$object,$flag,$number] }; " +
+            "exec \"capture\" {\"id\":1} true (1.5 + 2)",
+            TestContext.Current.CancellationToken);
+
+        Assert.False(state.IsError);
+        Assert.Equal("[{\"id\":1},true,3.5]", Assert.IsType<ShellJson>(state.Result).Value.GetRawText());
+    }
+
+    [Fact]
+    public async Task Exec_UnknownOption_Throws()
+    {
+        await Assert.ThrowsAsync<UnknownOptionException>(() => RunSingleAsync("exec \"dir\" --bogus=1"));
+    }
+
+    [Fact]
+    public async Task Exec_OptionMissingValue_Throws()
+    {
+        await Assert.ThrowsAsync<CommandException>(() => RunSingleAsync("exec \"dir\" --directory"));
+    }
+
+    [Theory]
+    [InlineData("exec \"echo\" first; echo after")]
+    [InlineData("exec \"echo\" first\necho after")]
+    [InlineData("{ exec \"echo\" first }; echo after")]
+    [InlineData("exec \"echo\" first | echo after")]
+    public async Task Exec_ArgumentTerminators_PreserveFollowingStatements(string source)
+    {
+        var state = await Shell.RunCommandAsync(new(), source, TestContext.Current.CancellationToken);
+
+        Assert.False(state.IsError);
+        Assert.Equal("after", Assert.IsType<ShellText>(state.Result).Text);
+    }
+
     [Fact]
     public async Task Exec_EmptyCommandPath_Throws()
     {

CosmosDBShell.Tests/Parser/CultureInvariantConversionTests.cs

@@ -5,6 +5,7 @@
 namespace CosmosShell.Tests.Parser;
 
 using System.Globalization;
+using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -51,6 +52,30 @@ public async Task ExpressionEvaluation_TextDecimalArithmetic_UsesInvariantCultur
         Assert.Equal(3.0d, value.Value);
     }
 
+    [Theory]
+    [InlineData("1.5", 1.5)]
+    [InlineData("\"1.5\"", 1.5)]
+    [InlineData("1e100", 1e100)]
+    [InlineData("\"1e100\"", 1e100)]
+    public void ShellJson_DecimalConversion_ReturnsInvariantDouble(string source, double expected)
+    {
+        using var document = JsonDocument.Parse(source);
+        var value = WithGermanCulture(() =>
+            Assert.IsType<double>(new ShellJson(document.RootElement).ConvertShellObject(DataType.Decimal)));
+
+        Assert.Equal(expected, value);
+    }
+
+    [Fact]
+    public void ShellJson_DecimalConversion_RejectsLocalizedDecimalSeparator()
+    {
+        using var document = JsonDocument.Parse("\"1,5\"");
+
+        WithGermanCulture(() =>
+            Assert.Throws<InvalidOperationException>(() =>
+                new ShellJson(document.RootElement).ConvertShellObject(DataType.Decimal)));
+    }
+
     private static T WithGermanCulture<T>(Func<T> action)
     {
         var originalCulture = CultureInfo.CurrentCulture;

CosmosDBShell.Tests/Parser/FilterPathExpressionTests.cs

@@ -70,6 +70,35 @@ public async Task Index_OutOfRange_ReturnsNull()
         Assert.Equal(JsonValueKind.Null, Assert.IsType<ShellJson>(result).Value.ValueKind);
     }
 
+    [Theory]
+    [InlineData(".[2147483647]")]
+    [InlineData(".[2147483647]?")]
+    public async Task Index_MaximumRepresentableIndex_ReturnsNull(string expression)
+    {
+        var result = await EvalAsync(expression, new[] { 10, 20, 30 });
+
+        Assert.Equal(JsonValueKind.Null, Assert.IsType<ShellJson>(result).Value.ValueKind);
+    }
+
+    [Theory]
+    [InlineData(".[2147483648]", "2147483648")]
+    [InlineData(".[2147483648]?", "2147483648")]
+    [InlineData(".items[999999999999999999999999]", "999999999999999999999999")]
+    public void Index_Overflow_ReportsErrorAtIndex(string expression, string index)
+    {
+        var lexer = new Lexer(expression);
+        var parser = new ExpressionParser(lexer);
+
+        parser.ParseFilterExpression();
+
+        var error = Assert.Single(lexer.Errors);
+        Assert.Equal(expression.IndexOf(index, StringComparison.Ordinal), error.Start);
+        Assert.Equal(index.Length, error.Length);
+        Assert.Equal(
+            Azure.Data.Cosmos.Shell.Util.MessageService.GetArgsString("expression_error_invalid_number", "value", index),
+            error.Message);
+    }
+
     [Fact]
     public async Task Index_OnNonArrayWithoutOptional_Throws()
         => await Assert.ThrowsAsync<CommandException>(() => EvalAsync(".[0]", new { a = 1 }));

CosmosDBShell.Tests/Parser/ShellObjectConversionTests.cs

@@ -116,7 +116,7 @@ public void ShellJson_NumberToAllTypes()
         var json = Json("42");
         Assert.Equal("42", json.ConvertShellObject(DataType.Text));
         Assert.Equal(42, json.ConvertShellObject(DataType.Number));
-        Assert.Equal(42m, json.ConvertShellObject(DataType.Decimal));
+        Assert.Equal(42d, Assert.IsType<double>(json.ConvertShellObject(DataType.Decimal)));
         Assert.Equal(true, json.ConvertShellObject(DataType.Boolean));
     }
 
@@ -170,7 +170,7 @@ public void ShellJson_StringToNumber(string raw, int expected)
     [Fact]
     public void ShellJson_StringToDecimal()
     {
-        Assert.Equal(15m, Json("\"15\"").ConvertShellObject(DataType.Decimal));
+        Assert.Equal(15d, Assert.IsType<double>(Json("\"15\"").ConvertShellObject(DataType.Decimal)));
     }
 
     [Fact]

CosmosDBShell.Tests/Parser/StatementExecutionTests.cs

@@ -96,6 +96,46 @@ public async Task SyntaxError_PreventsEarlierAssignment()
         Assert.Equal(1, GetInt("value"));
     }
 
+    [Theory]
+    [InlineData("for $x typo [1,2] { $value = 3 }")]
+    [InlineData("do { $value = 3 } typo false")]
+    public async Task InvalidLoopKeyword_PreventsEarlierAssignmentAndLoopBody(string invalid)
+    {
+        SetVariable("value", new ShellNumber(1));
+
+        var state = await Shell.RunCommandAsync(new(), $"$value = 2; {invalid}", TestContext.Current.CancellationToken);
+
+        Assert.True(state.IsError);
+        Assert.Equal(1, GetInt("value"));
+    }
+
+    [Theory]
+    [InlineData("-", "0.5")]
+    [InlineData("*", "7.5")]
+    [InlineData("/", "1.2")]
+    [InlineData("%", "0.5")]
+    [InlineData("**", "15.588457268119896")]
+    [InlineData("<", "false")]
+    [InlineData("<=", "false")]
+    [InlineData(">", "true")]
+    [InlineData(">=", "true")]
+    public async Task JsonNumericStrings_MatchTextAcrossValueOrigins(string operation, string expected)
+    {
+        var state = await Shell.RunCommandAsync(
+            new(),
+            "$source = {\"value\":\"2.5\"}; def identity [value] { return $value }; " +
+            $"$direct = 3.0 {operation} \"2.5\"; $json = 3.0 {operation} $source.value; " +
+            $"$function = 3.0 {operation} (identity $source.value); " +
+            $"for $item in [$source.value] {{ $loop = 3.0 {operation} $item }}",
+            TestContext.Current.CancellationToken);
+
+        Assert.False(state.IsError);
+        foreach (var name in new[] { "direct", "json", "function", "loop" })
+        {
+            Assert.Equal(expected, GetVariable(name)!.ConvertShellObject(DataType.Text));
+        }
+    }
+
     [Theory]
     [InlineData("break")]
     [InlineData("continue")]

CosmosDBShell.Tests/Parser/StatementParserEdgeTests.cs

@@ -49,6 +49,65 @@ public void Exec_WithArguments_ProducesExecStatement()
         Assert.IsType<ExecStatement>(Assert.Single(statements));
     }
 
+    [Theory]
+    [InlineData("for $x typo [1,2] {}", "typo", "statement_error_expected_in")]
+    [InlineData("do {} typo false", "typo", "statement_error_expected_while")]
+    public void Loop_InvalidKeyword_ReportsErrorAtKeyword(string source, string keyword, string messageKey)
+    {
+        var (_, parser) = ParseWithParser(source);
+
+        var error = Assert.Single(parser.Errors);
+        Assert.Equal(source.IndexOf(keyword, StringComparison.Ordinal), error.Start);
+        Assert.Equal(keyword.Length, error.Length);
+        Assert.Equal(Azure.Data.Cosmos.Shell.Util.MessageService.GetString(messageKey), error.Message);
+    }
+
+    [Theory]
+    [InlineData("for $x IN [1,2] {}")]
+    [InlineData("do {} WHILE false")]
+    public void Loop_KeywordsRemainCaseInsensitive(string source)
+    {
+        var (statements, parser) = ParseWithParser(source);
+
+        Assert.Single(statements);
+        Assert.Empty(parser.Errors);
+    }
+
+    [Theory]
+    [InlineData("-directory . -r")]
+    [InlineData("--directory=. --recursive")]
+    [InlineData("--directory:. --recursive=false")]
+    [InlineData("--directory=$path")]
+    [InlineData("-5 -1.5 +2")]
+    [InlineData("https://localhost:8081 *.json \"two words\"")]
+    [InlineData("{\"id\":1} [1,2] true $value (1 + 2)")]
+    public void Exec_ArgumentsMatchDirectCommandParsing(string arguments)
+    {
+        var (directStatements, directParser) = ParseWithParser($"dir {arguments}");
+        var (execStatements, execParser) = ParseWithParser($"exec $cmd {arguments}");
+
+        Assert.Empty(directParser.Errors);
+        Assert.Empty(execParser.Errors);
+        var direct = Assert.IsType<CommandStatement>(Assert.Single(directStatements));
+        var exec = Assert.IsType<ExecStatement>(Assert.Single(execStatements));
+        Assert.Equal(direct.Arguments.Count, exec.Arguments.Count);
+        for (int i = 0; i < direct.Arguments.Count; i++)
+        {
+            Assert.Equal(direct.Arguments[i].GetType(), exec.Arguments[i].GetType());
+            Assert.Equal(CommandArgumentFormatter.Format(direct.Arguments[i]), CommandArgumentFormatter.Format(exec.Arguments[i]));
+        }
+    }
+
+    [Theory]
+    [InlineData("exec \"echo\" --format=")]
+    [InlineData("exec \"echo\" --format=; echo after")]
+    public void Exec_MissingInlineOptionValue_ReportsError(string source)
+    {
+        var (_, parser) = ParseWithParser(source);
+
+        Assert.NotEmpty(parser.Errors);
+    }
+
     [Fact]
     public void Def_WithParenParameters_ParsesParameters()
     {

CosmosDBShell.Tests/ScriptExecutionScopeTests.cs

@@ -213,16 +213,20 @@ public async Task RecursiveScript_StopsAtCallLimit_AndRestoresScope()
     }
 
     [Theory]
-    [InlineData(false)]
-    [InlineData(true)]
-    public async Task InvalidScript_DoesNotRegisterFunctions_AndRestoresScope(bool expression)
+    [InlineData(false, "if true {")]
+    [InlineData(true, "if true {")]
+    [InlineData(false, "for $x typo [1,2] { $value = 3 }")]
+    [InlineData(true, "for $x typo [1,2] { $value = 3 }")]
+    [InlineData(false, "do { $value = 3 } typo false")]
+    [InlineData(true, "do { $value = 3 } typo false")]
+    public async Task InvalidScript_DoesNotRegisterFunctions_AndRestoresScope(bool expression, string invalid)
     {
         var shell = ShellInterpreter.CreateInstance();
         shell.SetVariable("value", new ShellNumber(1));
         var path = Path.GetTempFileName();
         try
         {
-            await File.WriteAllTextAsync(path, "def mustNotRegister { return 1 }; if true {", TestContext.Current.CancellationToken);
+            await File.WriteAllTextAsync(path, $"def mustNotRegister {{ return 1 }}; {invalid}", TestContext.Current.CancellationToken);
             var token = new Token(TokenType.Identifier, path, 0, path.Length);
             var state = expression
                 ? await new CommandExpression(token).RunScriptAsync(shell, new(), CancellationToken.None)

CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ExpressionParser.cs

@@ -1722,7 +1722,7 @@ private Expression ParseFilterPathExpression(Token firstToken)
                 }
 
                 var indexToken = this.Consume(TokenType.Number, MessageService.GetString("expression_error_expected_array_index"));
-                int index = int.TryParse(indexToken.Value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedIndex) ? parsedIndex : 0;
+                bool validIndex = int.TryParse(indexToken.Value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var index);
                 var indexedCloseBracket = this.Consume(TokenType.CloseBracket, MessageService.GetString("expression_error_expected_close_bracket"));
                 var indexQuestionToken = this.TryConsumeQuestion();
                 end = indexedCloseBracket.Start + indexedCloseBracket.Length;
@@ -1731,6 +1731,12 @@ private Expression ParseFilterPathExpression(Token firstToken)
                     end = indexQuestionToken.Start + indexQuestionToken.Length;
                 }
 
+                if (!validIndex)
+                {
+                    this.ReportError(MessageService.GetArgsString("expression_error_invalid_number", "value", indexToken.Value), indexToken);
+                    return new ErrorExpression(firstToken.Start, end - firstToken.Start);
+                }
+
                 segments.Add(new FilterIndexSegment(index, indexQuestionToken != null));
                 continue;
             }

CosmosDBShell/Azure.Data.Cosmos.Shell.Parser/ShellObject/ShellJson.cs

@@ -91,13 +91,13 @@ public override object ConvertShellObject(DataType type)
             case DataType.Decimal:
                 if (this.Value.ValueKind == JsonValueKind.Number)
                 {
-                    return this.Value.GetDecimal();
+                    return this.Value.GetDouble();
                 }
 
                 if (this.Value.ValueKind == JsonValueKind.String)
                 {
                     var strValue = this.Value.GetString() ?? string.Empty;
-                    if (decimal.TryParse(strValue, NumberStyles.Float, CultureInfo.InvariantCulture, out decimal decimalValue))
+                    if (double.TryParse(strValue, NumberStyles.Float, CultureInfo.InvariantCulture, out double decimalValue))
                     {
                         return decimalValue;
                     }