Skip to content

Commit d0e54fd

Browse files
renecannaorenecannao
committed
feat: preserve MySQL JSON, window and aggregate grammar
MySQL JSON arrows were split into subtraction/comparison or shift tokens, allowing a successful complete parse to change extraction into another operation during emission. Tokenize -> and ->> together and retain a structured extraction node with a column reference and literal path. Support database-qualified and delimited column names, preserve path string delimiters, and reject nonliteral paths and malformed arrow forms. Share named/inherited window AST productions with MySQL, including its nonreserved keyword names. Parse GROUP_CONCAT DISTINCT, multiple arguments, ORDER BY and string/hex/bit SEPARATOR operands explicitly. Retain MySQL GROUP BY ... WITH ROLLUP as a suffix instead of dropping the grouping semantics. Reject empty SELECT targets and incomplete new productions; keep PostgreSQL-only NULLS ordering out of MySQL window/aggregate syntax. Append token/node kinds without renumbering existing values or changing AST layout. Guard local execution of unsupported JSON/aggregate/grouping semantics and three-part column references, because the local resolver only handles table.column. Parameterization stays conservative for the new JSON and GROUP_CONCAT nodes rather than binding syntax-only literals. Validation: forced full rebuild and corpus-test build (1,530 tests pass; 37 external-database tests skipped); 14 focused positive, negative, exact-emission, reparse and planner-guard regressions; ASan/UBSan with leak detection passes all 14 focused tests. Native isolated MySQL 8.4.8 and 9.7.2 checks confirm the representative grammar and malformed controls. The full pinned PostgreSQL baseline and 87 compatibility-harness tests remain unchanged. Independent review verified the corrected quoting, keyword-name, literal separator and qualification edge cases. No benchmark artifacts included.
1 parent 456c743 commit d0e54fd

10 files changed

Lines changed: 462 additions & 34 deletions

File tree

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ TEST_SRCS = $(TEST_DIR)/test_main.cpp \
6464
$(TEST_DIR)/test_pg_table_constraints.cpp \
6565
$(TEST_DIR)/test_pg_maintenance.cpp \
6666
$(TEST_DIR)/test_review_expressions.cpp \
67+
$(TEST_DIR)/test_mysql_grammar.cpp \
6768
$(TEST_DIR)/test_review_ast.cpp \
6869
$(TEST_DIR)/test_review_transactions.cpp \
6970
$(TEST_DIR)/test_review_runtime.cpp \

‎include/sql_engine/plan_builder.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,14 @@ class PlanBuilder {
7070
bool dml_values = false) {
7171
using sql_parser::NodeType;
7272
switch (node->type) {
73+
case NodeType::NODE_QUALIFIED_NAME:
74+
// The local resolver combines only table.column; it would
75+
// otherwise silently drop the final part of db.table.column.
76+
if constexpr (D == sql_parser::Dialect::MySQL) {
77+
if (node->first_child && node->first_child->next_sibling &&
78+
node->first_child->next_sibling->next_sibling) return true;
79+
}
80+
break;
7381
// PG_CONT_EXPRESSION_GUARD
7482
case NodeType::NODE_LITERAL_BIT:
7583
case NodeType::NODE_LITERAL_HEX:
@@ -168,6 +176,9 @@ class PlanBuilder {
168176
break;
169177
case NodeType::NODE_NAMED_ARGUMENT:
170178
case NodeType::NODE_TYPE_CAST:
179+
case NodeType::NODE_MYSQL_JSON_EXTRACT:
180+
case NodeType::NODE_MYSQL_GROUP_CONCAT:
181+
case NodeType::NODE_MYSQL_SEPARATOR:
171182
case NodeType::NODE_DISTINCT_ON:
172183
case NodeType::NODE_AGGREGATE_ORDER_BY:
173184
case NodeType::NODE_AGGREGATE_FILTER:

‎include/sql_parser/common.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -344,6 +344,9 @@ enum class NodeType : uint16_t {
344344
NODE_PG_SELECT_INTO,
345345
NODE_PG_ROW_LOCK,
346346
NODE_PG_COMMAND_STMT, // validated command with structural operands/clauses
347+
NODE_MYSQL_JSON_EXTRACT, // value=-> or ->>; column reference, literal path
348+
NODE_MYSQL_GROUP_CONCAT, // arguments, optional aggregate ORDER BY and separator
349+
NODE_MYSQL_SEPARATOR, // literal string child
347350

348351
};
349352

‎include/sql_parser/emitter.h‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,8 +161,15 @@ class Emitter {
161161
break;
162162
// ---- Expressions ----
163163
case NodeType::NODE_BINARY_OP: emit_binary_op(node); break;
164+
case NodeType::NODE_MYSQL_JSON_EXTRACT:
165+
emit_node(node->first_child); sb_.append_char(' '); emit_value(node); sb_.append_char(' ');
166+
emit_mysql_syntax_string(node->first_child ? node->first_child->next_sibling : nullptr);
167+
break;
164168
case NodeType::NODE_UNARY_OP: emit_unary_op(node); break;
165169
case NodeType::NODE_FUNCTION_CALL: emit_function_call(node); break;
170+
case NodeType::NODE_MYSQL_GROUP_CONCAT: emit_function_call(node); break;
171+
case NodeType::NODE_MYSQL_SEPARATOR:
172+
sb_.append(" SEPARATOR "); emit_mysql_syntax_string(node->first_child); break;
166173
case NodeType::NODE_IS_NULL: emit_is_null(node); break;
167174
case NodeType::NODE_IS_NOT_NULL: emit_is_not_null(node); break;
168175
case NodeType::NODE_BETWEEN: emit_between(node); break;
@@ -836,13 +843,18 @@ class Emitter {
836843

837844
void emit_group_by(const AstNode* node) {
838845
sb_.append(" GROUP BY ");
839-
if (!node->value().empty()) { emit_value(node); sb_.append_char(' '); }
846+
if constexpr (D == Dialect::PostgreSQL) {
847+
if (!node->value().empty()) { emit_value(node); sb_.append_char(' '); }
848+
}
840849
bool first = true;
841850
for (const AstNode* child = node->first_child; child; child = child->next_sibling) {
842851
if (!first) sb_.append(", ");
843852
first = false;
844853
emit_node(child);
845854
}
855+
if constexpr (D == Dialect::MySQL) {
856+
if (!node->value().empty()) { sb_.append_char(' '); emit_value(node); }
857+
}
846858
}
847859

848860
void emit_having(const AstNode* node) {
@@ -1535,6 +1547,14 @@ class Emitter {
15351547

15361548
// ---- Expressions ----
15371549

1550+
void emit_mysql_syntax_string(const AstNode* node) {
1551+
// A double-quoted literal can contain unescaped apostrophes. Preserve
1552+
// its delimiter instead of turning a valid path/separator into bad SQL.
1553+
if (node && mode_ == EmitMode::NORMAL && !node->source().empty())
1554+
sb_.append(node->source_ptr, node->source_len);
1555+
else emit_node(node);
1556+
}
1557+
15381558
void emit_binary_op(const AstNode* node) {
15391559
const AstNode* left = node->first_child;
15401560
const AstNode* right = left ? left->next_sibling : nullptr;
@@ -1617,7 +1637,12 @@ class Emitter {
16171637
else if (node->flags & FLAG_FUNCTION_ALL) sb_.append("ALL ");
16181638
bool first = true;
16191639
const AstNode* order = nullptr;
1640+
const AstNode* separator = nullptr;
16201641
for (; arg; arg = arg->next_sibling) {
1642+
if (arg->type == NodeType::NODE_MYSQL_SEPARATOR) {
1643+
separator = arg;
1644+
continue;
1645+
}
16211646
if (arg->type == NodeType::NODE_AGGREGATE_ORDER_BY) {
16221647
order = arg;
16231648
continue;
@@ -1627,6 +1652,7 @@ class Emitter {
16271652
emit_node(arg);
16281653
}
16291654
if (order && !(node->flags & FLAG_FUNCTION_WITHIN_GROUP)) emit_node(order);
1655+
if (separator) emit_node(separator);
16301656
sb_.append_char(')');
16311657
if (order && (node->flags & FLAG_FUNCTION_WITHIN_GROUP)) {
16321658
sb_.append(" WITHIN GROUP (ORDER BY ");

‎include/sql_parser/expression_parser.h‎

Lines changed: 133 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,39 @@ class ExpressionParser {
7070
token.text.equals_ci(word.data(), static_cast<uint32_t>(word.size()));
7171
}
7272

73+
// Window names are identifiers, including nonreserved MySQL keyword tokens.
74+
// Token classification checked against MySQL 8.4.8 and 9.7.2 KEYWORDS.
75+
static bool window_name_token(const Token& token) {
76+
if (token.type == TokenType::TK_IDENTIFIER) return true;
77+
if constexpr (D == Dialect::MySQL) {
78+
switch (token.type) {
79+
case TokenType::TK_ARRAY: case TokenType::TK_AVG: case TokenType::TK_BEGIN:
80+
case TokenType::TK_BUFFERS: case TokenType::TK_CHARSET: case TokenType::TK_COLUMNS:
81+
case TokenType::TK_COMMIT: case TokenType::TK_COMMITTED: case TokenType::TK_CONCURRENT:
82+
case TokenType::TK_CONFLICT: case TokenType::TK_COSTS: case TokenType::TK_COUNT:
83+
case TokenType::TK_DATA: case TokenType::TK_DEALLOCATE: case TokenType::TK_DO:
84+
case TokenType::TK_DUMPFILE: case TokenType::TK_DUPLICATE: case TokenType::TK_END:
85+
case TokenType::TK_EXECUTE: case TokenType::TK_FIELDS: case TokenType::TK_FORMAT:
86+
case TokenType::TK_FULL: case TokenType::TK_GLOBAL: case TokenType::TK_ISOLATION:
87+
case TokenType::TK_LEVEL: case TokenType::TK_LOCAL: case TokenType::TK_LOCKED:
88+
case TokenType::TK_MAX: case TokenType::TK_MEMBER: case TokenType::TK_MIN:
89+
case TokenType::TK_NAMES: case TokenType::TK_NOTHING: case TokenType::TK_NOWAIT:
90+
case TokenType::TK_OFFSET: case TokenType::TK_ONLY: case TokenType::TK_PERSIST:
91+
case TokenType::TK_PREPARE: case TokenType::TK_QUICK: case TokenType::TK_REPEATABLE:
92+
case TokenType::TK_RESET: case TokenType::TK_RETURNING: case TokenType::TK_ROLLBACK:
93+
case TokenType::TK_SAVEPOINT: case TokenType::TK_SERIALIZABLE: case TokenType::TK_SESSION:
94+
case TokenType::TK_SETTINGS: case TokenType::TK_SHARE: case TokenType::TK_SKIP:
95+
case TokenType::TK_SOUNDS: case TokenType::TK_START: case TokenType::TK_SUM:
96+
case TokenType::TK_SUMMARY: case TokenType::TK_TIMING: case TokenType::TK_TRANSACTION:
97+
case TokenType::TK_TRUNCATE: case TokenType::TK_UNCOMMITTED: case TokenType::TK_VERBOSE:
98+
case TokenType::TK_VIEW: case TokenType::TK_WAL:
99+
return true;
100+
default: break;
101+
}
102+
}
103+
return false;
104+
}
105+
73106
AstNode* syntax_error() {
74107
StringRef source = tok_.peek().source;
75108
if (source.empty() && tok_.input_end() > tok_.input_begin())
@@ -697,7 +730,7 @@ class ExpressionParser {
697730
item->add_child(ordering);
698731
}
699732
}
700-
if (keyword(tok_.peek(), "NULLS")) {
733+
if (D == Dialect::PostgreSQL && keyword(tok_.peek(), "NULLS")) {
701734
tok_.skip();
702735
Token placement = tok_.peek();
703736
if (!keyword(placement, "FIRST") && !keyword(placement, "LAST")) return syntax_error();
@@ -716,6 +749,10 @@ class ExpressionParser {
716749
}
717750

718751
AstNode* parse_identifier_or_function(const Token& name_token) {
752+
if constexpr (D == Dialect::MySQL) {
753+
if (tok_.peek().type == TokenType::TK_LPAREN && keyword(name_token, "GROUP_CONCAT"))
754+
return parse_mysql_group_concat();
755+
}
719756
if constexpr (D == Dialect::PostgreSQL) {
720757
const bool has_arguments = tok_.peek().type == TokenType::TK_LPAREN;
721758
// PG_GAPS_JSON_XML_EXPRESSION_HOOK
@@ -936,6 +973,10 @@ class ExpressionParser {
936973
Token col = tok_.next_token();
937974
if constexpr (D == Dialect::PostgreSQL) {
938975
if (!pg_column_label(col) && col.type != TokenType::TK_ASTERISK) return syntax_error();
976+
} else {
977+
if (!(col.type == TokenType::TK_IDENTIFIER || col.type == TokenType::TK_ASTERISK ||
978+
(col.type >= TokenType::TK_SELECT && col.type <= TokenType::TK_RECURSIVE)))
979+
return syntax_error();
939980
}
940981
AstNode* qname = make_node(arena_, NodeType::NODE_QUALIFIED_NAME);
941982
AstNode* schema_node = make_node_from_token(arena_, NodeType::NODE_IDENTIFIER, name_token);
@@ -947,15 +988,93 @@ class ExpressionParser {
947988
col_node->flags |= FLAG_IDENT_DELIMITED;
948989
qname->add_child(schema_node);
949990
qname->add_child(col_node);
991+
if constexpr (D == Dialect::MySQL) {
992+
// MySQL simple_ident permits column, table.column and db.table.column.
993+
if (tok_.peek().type == TokenType::TK_DOT) {
994+
tok_.skip();
995+
Token field = tok_.next_token();
996+
if (col.type == TokenType::TK_ASTERISK ||
997+
!(field.type == TokenType::TK_IDENTIFIER || field.type == TokenType::TK_ASTERISK ||
998+
(field.type >= TokenType::TK_SELECT && field.type <= TokenType::TK_RECURSIVE)))
999+
return syntax_error();
1000+
auto* field_node = make_node_from_token(arena_, NodeType::NODE_IDENTIFIER, field,
1001+
token_was_delimited_(field) ? FLAG_IDENT_DELIMITED : 0);
1002+
if (!field_node) return syntax_error();
1003+
qname->add_child(field_node);
1004+
}
1005+
return parse_mysql_json_extract(qname);
1006+
}
9501007
return qname;
9511008
}
9521009

9531010
AstNode* col_ref = make_node_from_token(arena_, NodeType::NODE_COLUMN_REF, name_token);
9541011
if (col_ref && token_was_delimited_(name_token))
9551012
col_ref->flags |= FLAG_IDENT_DELIMITED;
1013+
if constexpr (D == Dialect::MySQL) return parse_mysql_json_extract(col_ref);
9561014
return col_ref;
9571015
}
9581016

1017+
AstNode* parse_mysql_group_concat() {
1018+
tok_.skip(); // (
1019+
auto* node = make_node(arena_, NodeType::NODE_MYSQL_GROUP_CONCAT, {"GROUP_CONCAT", 12});
1020+
if (!node) return syntax_error();
1021+
if (tok_.peek().type == TokenType::TK_DISTINCT) {
1022+
tok_.skip();
1023+
node->flags |= FLAG_FUNCTION_DISTINCT;
1024+
}
1025+
do {
1026+
auto* arg = parse_complete();
1027+
if (!arg || arg->type == NodeType::NODE_ASTERISK) return syntax_error();
1028+
if (arg->type == NodeType::NODE_QUALIFIED_NAME) {
1029+
for (auto* part = arg->first_child; part; part = part->next_sibling)
1030+
if (!(part->flags & FLAG_IDENT_DELIMITED) && part->value().equals_ci("*", 1))
1031+
return syntax_error();
1032+
}
1033+
node->add_child(arg);
1034+
if (tok_.peek().type != TokenType::TK_COMMA) break;
1035+
tok_.skip();
1036+
} while (true);
1037+
if (tok_.peek().type == TokenType::TK_ORDER) {
1038+
auto* order = parse_aggregate_order();
1039+
if (!order) return syntax_error();
1040+
node->add_child(order);
1041+
}
1042+
if (keyword(tok_.peek(), "SEPARATOR")) {
1043+
tok_.skip();
1044+
const Token text = tok_.next_token();
1045+
NodeType literal_type;
1046+
if (text.type == TokenType::TK_STRING) literal_type = NodeType::NODE_LITERAL_STRING;
1047+
else if (text.type == TokenType::TK_HEX_LITERAL) literal_type = NodeType::NODE_LITERAL_HEX;
1048+
else if (text.type == TokenType::TK_BIT_LITERAL) literal_type = NodeType::NODE_LITERAL_BIT;
1049+
else return syntax_error();
1050+
auto* separator = make_node(arena_, NodeType::NODE_MYSQL_SEPARATOR);
1051+
auto* literal = make_node_from_token(arena_, literal_type, text);
1052+
if (!separator || !literal) return syntax_error();
1053+
separator->add_child(literal);
1054+
node->add_child(separator);
1055+
}
1056+
if (tok_.peek().type != TokenType::TK_RPAREN) return syntax_error();
1057+
tok_.skip();
1058+
return tok_.peek().type == TokenType::TK_OVER ? parse_window_function(node) : node;
1059+
}
1060+
1061+
AstNode* parse_mysql_json_extract(AstNode* column) {
1062+
if (tok_.peek().type != TokenType::TK_MYSQL_JSON_ARROW) return column;
1063+
Token op = tok_.next_token();
1064+
if (!column || tok_.peek().type != TokenType::TK_STRING) return syntax_error();
1065+
if (column->type == NodeType::NODE_QUALIFIED_NAME) {
1066+
for (auto* part = column->first_child; part; part = part->next_sibling)
1067+
if (!(part->flags & FLAG_IDENT_DELIMITED) && part->value().equals_ci("*", 1)) return syntax_error();
1068+
}
1069+
Token path = tok_.next_token();
1070+
AstNode* node = make_node(arena_, NodeType::NODE_MYSQL_JSON_EXTRACT, op.text);
1071+
AstNode* literal = make_node_from_token(arena_, NodeType::NODE_LITERAL_STRING, path);
1072+
if (!node || !literal) return syntax_error();
1073+
node->add_child(column);
1074+
node->add_child(literal);
1075+
return node;
1076+
}
1077+
9591078
// True iff `t` is a TK_IDENTIFIER token whose source bytes were
9601079
// surrounded by backticks (MySQL) or double quotes (PostgreSQL).
9611080
// Uses lookback against the tokenizer's input buffer so the check is
@@ -1788,11 +1907,13 @@ class ExpressionParser {
17881907
AstNode* win = make_node(arena_, NodeType::NODE_WINDOW_FUNCTION);
17891908
if (!win) return syntax_error();
17901909
win->add_child(func);
1791-
if constexpr (D == Dialect::PostgreSQL) {
1792-
if (tok_.peek().type == TokenType::TK_IDENTIFIER) {
1910+
{
1911+
if (window_name_token(tok_.peek())) {
17931912
Token name = tok_.next_token();
1794-
win->add_child(make_node(arena_, NodeType::NODE_WINDOW_REFERENCE,
1795-
name.source.empty() ? name.text : name.source));
1913+
auto* reference = make_node(arena_, NodeType::NODE_WINDOW_REFERENCE,
1914+
name.source.empty() ? name.text : name.source);
1915+
if (!reference) return syntax_error();
1916+
win->add_child(reference);
17961917
return win;
17971918
}
17981919
}
@@ -1809,13 +1930,15 @@ class ExpressionParser {
18091930
if (!spec) return syntax_error();
18101931
if (tok_.peek().type != TokenType::TK_LPAREN) return syntax_error();
18111932
tok_.skip();
1812-
if constexpr (D == Dialect::PostgreSQL) {
1933+
{
18131934
Token name = tok_.peek();
1814-
if (name.type == TokenType::TK_IDENTIFIER &&
1935+
if (window_name_token(name) &&
18151936
!keyword(name, "ROWS") && !keyword(name, "RANGE") && !keyword(name, "GROUPS")) {
18161937
tok_.skip();
1817-
spec->add_child(make_node(arena_, NodeType::NODE_WINDOW_REFERENCE,
1818-
name.source.empty() ? name.text : name.source));
1938+
auto* reference = make_node(arena_, NodeType::NODE_WINDOW_REFERENCE,
1939+
name.source.empty() ? name.text : name.source);
1940+
if (!reference) return syntax_error();
1941+
spec->add_child(reference);
18191942
}
18201943
}
18211944
if (tok_.peek().type == TokenType::TK_PARTITION) {
@@ -1858,7 +1981,7 @@ class ExpressionParser {
18581981
item->add_child(ordering);
18591982
}
18601983
}
1861-
if (keyword(tok_.peek(), "NULLS")) {
1984+
if (D == Dialect::PostgreSQL && keyword(tok_.peek(), "NULLS")) {
18621985
tok_.skip();
18631986
item->flags |= FLAG_ORDER_NULLS;
18641987
Token placement = tok_.peek();

0 commit comments

Comments
 (0)