Unified: Fix locations of various tokens

Handles things like `try!` (which is represented as two separate tokens
-- we explicitly union their ranges) and "let" binding modifiers (where
we reuse the bindingSpecifier, getting its location and string value for
free).

Author: tausbn

unified/extractor/src/languages/swift/swift.rs

@@ -100,7 +100,9 @@ fn and_chain(
     conds
         .into_iter()
         .reduce(|acc, elem| {
-            tree!((binary_expr operator: (infix_operator "&&") left: {acc} right: {elem}))
+            let operator_range = ctx.empty_source_range_between(acc, elem);
+            let operator = ctx.literal_with_source_range("infix_operator", "&&", operator_range);
+            tree!((binary_expr operator: {operator} left: {acc} right: {elem}))
         })
         .expect("control-flow statement must have at least one condition")
 }
@@ -126,21 +128,15 @@ fn member_chain(
     ctx: &mut yeast::build::BuildCtx<'_, SwiftContext>,
     parts: Vec<yeast::Id>,
 ) -> yeast::Id {
-    // `member_chain` builds the imported expression inside the larger import
-    // declaration rule. The imported expression should span the import path,
-    // not the whole declaration including the `import` keyword.
-    let source_range = ctx.source_range.take();
     let mut iter = parts.into_iter();
     let first = iter
         .next()
         .expect("identifier with `part:` must have at least one part");
     let init = tree!((identifier #{first}));
-    let result = iter.fold(
+    iter.fold(
         init,
         |acc, elem| tree!((member_access_expr base: {acc} member_name_node: (identifier #{elem}))),
-    );
-    ctx.source_range = source_range;
-    result
+    )
 }
 
 /// Compound-assignment operator spellings (`+=`, `<<=`, ...). Used to tell a
@@ -865,6 +861,7 @@ fn translation_rules() -> Vec<Rule<SwiftContext>> {
         // form is matched first.
         rule!(
             (optionalBindingCondition
+                bindingSpecifier: @@spec
                 pattern: (identifierPattern identifier: @name)
                 initializer: (initializerClause value: @val))
             =>
@@ -873,18 +870,20 @@ fn translation_rules() -> Vec<Rule<SwiftContext>> {
                 pattern: (call_expr
                     callee: (member_access_expr base: (identifier "Optional") member_name_node: (identifier "some"))
                     argument: (argument value: (expr_pattern
-                        modifier: (modifier "let")
+                        modifier: (modifier #{spec})
                         expr: (identifier #{name})))))
         ),
         rule!(
-            (optionalBindingCondition pattern: (identifierPattern identifier: @name))
+            (optionalBindingCondition
+                bindingSpecifier: @@spec
+                pattern: (identifierPattern identifier: @name))
             =>
             (pattern_guard_expr
                 value: (identifier #{name})
                 pattern: (call_expr
                     callee: (member_access_expr base: (identifier "Optional") member_name_node: (identifier "some"))
                     argument: (argument value: (expr_pattern
-                        modifier: (modifier "let")
+                        modifier: (modifier #{spec})
                         expr: (identifier #{name})))))
         ),
         // A single condition in an `if`/`while`/`guard` condition list unwraps to
@@ -968,11 +967,18 @@ fn translation_rules() -> Vec<Rule<SwiftContext>> {
         }),
         // try/try?/try! expr → unary_expr with operator "try", "try?" or "try!"
         rule!(
-            (tryExpr questionOrExclamationMark: _? @@m expression: @e)
+            (tryExpr
+                tryKeyword: @@keyword
+                questionOrExclamationMark: _? @@m
+                expression: @e)
             =>
             expr {
                 let op = format!("try{}", m.map(|m| ctx.source_text(m)).unwrap_or_default());
-                tree!((unary_expr operator: (prefix_operator #{op}) operand: {e}))
+                let operator_range =
+                    ctx.source_range_of_all(std::iter::once(keyword).chain(m));
+                let operator =
+                    ctx.literal_with_source_range("prefix_operator", &op, operator_range);
+                tree!((unary_expr operator: {operator} operand: {e}))
             }
         ),
         // Do-catch → try_expr
@@ -1006,17 +1012,27 @@ fn translation_rules() -> Vec<Rule<SwiftContext>> {
         // Catch block without error binding
         rule!((catchClause body: @body) => (catch_clause body: {body})),
         // As expression (type cast) — as?, as!
-        rule!((asExpr expression: @val questionOrExclamationMark: _? @@mark type: @ty) => type_cast_expr {
+        rule!((asExpr expression: @val asKeyword: @@keyword questionOrExclamationMark: _? @@mark type: @ty) => type_cast_expr {
             let op = format!("as{}", mark.map(|m| ctx.source_text(m)).unwrap_or_default());
-            tree!((type_cast_expr expr: {val} operator: (infix_operator #{op}) type: {ty}))
+            let operator_range =
+                ctx.source_range_of_all(std::iter::once(keyword).chain(mark));
+            let operator = ctx.literal_with_source_range("infix_operator", &op, operator_range);
+            tree!((type_cast_expr expr: {val} operator: {operator} type: {ty}))
         }),
         // Check expression (`x is T`) → type_test_expr
-        rule!((isExpr expression: @val type: @ty) => (type_test_expr expr: {val} operator: (infix_operator "is") type: {ty})),
+        rule!((isExpr expression: @val isKeyword: @@keyword type: @ty) => (type_test_expr
+            expr: {val}
+            operator: {ctx.literal_with_source_range_of("infix_operator", "is", keyword)}
+            type: {ty})),
         // Await expression → unary_expr with operator "await"
-        rule!((awaitExpr expression: @val) => (unary_expr operator: (prefix_operator "await") operand: {val})),
+        rule!((awaitExpr awaitKeyword: @@keyword expression: @val) => (unary_expr
+            operator: {ctx.literal_with_source_range_of("prefix_operator", "await", keyword)}
+            operand: {val})),
         // Force-unwrap (`x!`) → postfix unary_expr, via swift-syntax's dedicated
         // `forceUnwrapExpr` node.
-        rule!((forceUnwrapExpr expression: @e) => (unary_expr operator: (postfix_operator "!") operand: {e})),
+        rule!((forceUnwrapExpr expression: @e exclamationMark: @@mark) => (unary_expr
+            operator: {ctx.literal_with_source_range_of("postfix_operator", "!", mark)}
+            operand: {e})),
         // ---- Imports ----
         // An import declaration. The dotted path (a list of
         // `importPathComponent`s) becomes a `name_node`/`member_access_expr`
@@ -1036,9 +1052,14 @@ fn translation_rules() -> Vec<Rule<SwiftContext>> {
             import_declaration {
                 let last = *parts.last().ok_or("import has no path")?;
                 let pattern = match kind {
-                    None => tree!((named_pattern
-                        name_node: (identifier #{last})
-                        sub_pattern: (bulk_importing_pattern))),
+                    None => {
+                        let bulk = ctx.with_node_source_range(ctx.matched_source_range, |ctx| {
+                            tree!(ctx, (bulk_importing_pattern))
+                        });
+                        tree!((named_pattern
+                            name_node: (identifier #{last})
+                            sub_pattern: {bulk}))
+                    }
                     Some(_) => tree!((identifier #{last})),
                 };
                 tree!((import_declaration

unified/extractor/tests/corpus/swift/desugar/import-with-deeply-nested-path-three-parts.output

@@ -38,4 +38,4 @@ top_level
           pattern:
             named_pattern
               name_node: identifier "URLSession"
-              sub_pattern: bulk_importing_pattern
+              sub_pattern: bulk_importing_pattern "import Foundation.Networking.URLSession"

unified/extractor/tests/corpus/swift/desugar/import-with-dotted-path-two-parts.output

@@ -32,4 +32,4 @@ top_level
           pattern:
             named_pattern
               name_node: identifier "Networking"
-              sub_pattern: bulk_importing_pattern
+              sub_pattern: bulk_importing_pattern "import Foundation.Networking"

unified/extractor/tests/corpus/swift/desugar/simple-import-with-single-name.output

@@ -26,4 +26,4 @@ top_level
           pattern:
             named_pattern
               name_node: identifier "Foundation"
-              sub_pattern: bulk_importing_pattern
+              sub_pattern: bulk_importing_pattern "import Foundation"

unified/extractor/tests/location_tests.rs

@@ -71,13 +71,46 @@ fn generic_type_children_have_local_ranges() {
     assert_has_span(&ast, source, "identifier", Some("Foo"), "Foo");
 }
 
+#[test]
+fn synthesized_condition_and_switch_nodes_use_child_ranges() {
+    let source = "if a, b { c }\nswitch x { case a, b: c }";
+    let ast = desugar(source);
+
+    assert_has_span(&ast, source, "binary_expr", None, "a, b");
+    assert_has_empty_span(
+        &ast,
+        "infix_operator",
+        Some("&&"),
+        source.find(',').unwrap(),
+    );
+    assert_has_span(&ast, source, "or_pattern", None, "a, b");
+    assert_has_span(&ast, source, "block", None, "c");
+}
+
 #[test]
 fn declaration_and_operator_tokens_keep_precise_ranges() {
-    let source = "func f() { return x }";
+    let source = "func f() { return x }\nlet y = try? await value! as? T\nlet z = value is T";
     let ast = desugar(source);
 
     assert_has_span(&ast, source, "block", None, "{ return x }");
     assert_has_span(&ast, source, "return_expr", None, "return x");
+    assert_has_span(&ast, source, "prefix_operator", Some("try?"), "try?");
+    assert_has_span(&ast, source, "prefix_operator", Some("await"), "await");
+    assert_has_span(&ast, source, "postfix_operator", Some("!"), "!");
+    assert_has_span(&ast, source, "infix_operator", Some("as?"), "as?");
+    assert_has_span(&ast, source, "infix_operator", Some("is"), "is");
+}
+
+#[test]
+fn synthetic_optional_binding_nodes_anchor_to_binding_keyword() {
+    let source = "if let value = optional {}";
+    let ast = desugar(source);
+    let binding_start = source.find("let").unwrap();
+
+    assert_has_empty_span(&ast, "member_access_expr", None, binding_start);
+    assert_has_empty_span(&ast, "identifier", Some("Optional"), binding_start);
+    assert_has_empty_span(&ast, "identifier", Some("some"), binding_start);
+    assert_has_span(&ast, source, "modifier", Some("let"), "let");
 }
 
 #[test]