Skip to content

Commit c1e2d17

Browse files
renecannaorenecannao
committed
feat: parse MySQL table DDL, JSON_TABLE and procedure bodies
Replace the remaining classification-only table and routine witnesses with bounded native grammar and traversable ASTs. Add a shared MySQL field-type parser for table columns, JSON_TABLE columns and procedure parameters, including native modifiers, character sets, numeric token bounds and the 9.7 VECTOR type. Unknown types and unsupported clause tails remain errors. Parse CREATE [TEMPORARY] TABLE [IF NOT EXISTS] definitions, common column attributes, generated/default expressions, keys, CHECK/foreign constraints and table options. Parse ALTER ADD/DROP/MODIFY/CHANGE/RENAME actions and supported algorithm/lock options. Keep NOW() adjacent during emission and reject wildcard/DEFAULT operands where native value syntax excludes them. Represent JSON_TABLE documents, paths, typed/ordinality/EXISTS columns, nested COLUMNS and ON EMPTY/ON ERROR responses separately. Preserve native literal spelling, require table aliases, and bound nested column lists. Parse typed CREATE PROCEDURE parameters, SQL characteristics and supported single-statement or nested BEGIN/END bodies. Reuse strict query/DML grammar, validate SET operands, and use routine grammar to find supported procedure boundaries in parse_all without confusing CASE END or quoted semicolons. Retain optimizer-hint comments immediately after hintable statement keywords. Preserve exact hint contents without interpreting their language; ignore comments after qualified keyword identifiers and REPLACE functions as native MySQL does. Keep new syntax out of unsupported local execution and conservative parameterization, including direct UPDATE/DELETE plans. Append enum kinds and retain the 48-byte AST layout. Add positive, malformed-input, AST ownership/mutation, stable-emission, batch-boundary, nesting-limit and planner regressions. Independent review identified and verified fixes for NOW spacing, qualified hint comments, procedure SET stars/collation names and DDL value-expression validation. Validation: forced full build and make test pass 1,669 tests with 37 external-database skips; 153 focused MySQL tests pass ASan/UBSan with leak detection. Seven parser/emitter benchmark smoke cases pass; benchmark code and results are deliberately excluded. This implements bounded feature subsets, not complete MySQL DDL or stored-program grammar.
1 parent 798c6dc commit c1e2d17

22 files changed

Lines changed: 1799 additions & 0 deletions

‎Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,10 @@ TEST_SRCS = $(TEST_DIR)/test_main.cpp \
7474
$(TEST_DIR)/test_mysql_conversion_expressions.cpp \
7575
$(TEST_DIR)/test_mysql_insert_sources.cpp \
7676
$(TEST_DIR)/test_mysql_json_aggregates.cpp \
77+
$(TEST_DIR)/test_mysql_optimizer_hints.cpp \
78+
$(TEST_DIR)/test_mysql_json_table.cpp \
79+
$(TEST_DIR)/test_mysql_table_ddl.cpp \
80+
$(TEST_DIR)/test_mysql_procedure.cpp \
7781
$(TEST_DIR)/test_mysql_grouping.cpp \
7882
$(TEST_DIR)/test_mysql_explain.cpp \
7983
$(TEST_DIR)/test_review_ast.cpp \

‎include/sql_engine/dml_plan_builder.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,8 @@ class DmlPlanBuilder {
204204
private:
205205
static bool supported(const sql_parser::AstNode* ast) {
206206
if (!ast) return false;
207+
if constexpr (D == sql_parser::Dialect::MySQL)
208+
if (has_mysql_metadata(ast)) return false;
207209
if constexpr (D == sql_parser::Dialect::PostgreSQL) {
208210
if (!PlanBuilder<D>::supports_dml_features(ast) || unsupported_dml(ast)) return false;
209211
} else if (ast->type == sql_parser::NodeType::NODE_INSERT_STMT) {
@@ -214,6 +216,17 @@ class DmlPlanBuilder {
214216
return true;
215217
}
216218

219+
// These newly structured nodes have no local execution semantics. Keep
220+
// the existing multi-table UPDATE/DELETE routing policy otherwise intact.
221+
static bool has_mysql_metadata(const sql_parser::AstNode* ast) {
222+
using sql_parser::NodeType;
223+
if (ast->type == NodeType::NODE_MYSQL_OPTIMIZER_HINT ||
224+
ast->type == NodeType::NODE_MYSQL_JSON_TABLE) return true;
225+
for (auto* child = ast->first_child; child; child = child->next_sibling)
226+
if (has_mysql_metadata(child)) return true;
227+
return false;
228+
}
229+
217230
static bool unsupported_dml(const sql_parser::AstNode* ast) {
218231
using sql_parser::NodeType;
219232
if (ast->type == NodeType::NODE_CTE || ast->type == NodeType::NODE_COMPOUND_QUERY ||

‎include/sql_engine/plan_builder.h‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,25 @@ class PlanBuilder {
197197
case NodeType::NODE_MYSQL_INSERT_ALIAS:
198198
case NodeType::NODE_MYSQL_JSON_AGGREGATE:
199199
case NodeType::NODE_MYSQL_JSON_AGG_ARGUMENT:
200+
case NodeType::NODE_MYSQL_CREATE_TABLE:
201+
case NodeType::NODE_MYSQL_ALTER_TABLE:
202+
case NodeType::NODE_MYSQL_COLUMN_DEF:
203+
case NodeType::NODE_MYSQL_DDL_CLAUSE:
204+
case NodeType::NODE_MYSQL_DDL_LIST:
205+
case NodeType::NODE_MYSQL_DDL_SYNTAX:
206+
case NodeType::NODE_MYSQL_ALTER_ACTIONS:
207+
case NodeType::NODE_MYSQL_JSON_TABLE:
208+
case NodeType::NODE_MYSQL_JSON_TABLE_COLUMNS:
209+
case NodeType::NODE_MYSQL_JSON_TABLE_COLUMN:
210+
case NodeType::NODE_MYSQL_JSON_TABLE_NESTED:
211+
case NodeType::NODE_MYSQL_JSON_TABLE_RESPONSE:
212+
case NodeType::NODE_MYSQL_JSON_TABLE_LITERAL:
213+
case NodeType::NODE_MYSQL_CREATE_PROCEDURE:
214+
case NodeType::NODE_MYSQL_PROCEDURE_PARAMS:
215+
case NodeType::NODE_MYSQL_PROCEDURE_PARAM:
216+
case NodeType::NODE_MYSQL_PROCEDURE_CHARACTERISTIC:
217+
case NodeType::NODE_MYSQL_PROCEDURE_BLOCK:
218+
case NodeType::NODE_MYSQL_OPTIMIZER_HINT:
200219
case NodeType::NODE_MYSQL_EXPLAIN_INTO:
201220
case NodeType::NODE_MYSQL_EXTRACT:
202221
case NodeType::NODE_MYSQL_SUBSTRING:

‎include/sql_parser/common.h‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -367,6 +367,27 @@ enum class NodeType : uint16_t {
367367
NODE_MYSQL_JSON_AGG_ARGUMENT, // explicit ALL argument; expression child
368368
NODE_MYSQL_EXPLAIN_INTO, // user-variable destination child
369369

370+
// Structured MySQL table, routine, JSON_TABLE and hint syntax.
371+
NODE_MYSQL_CREATE_TABLE, // command value; validated syntax, name, definition list, options
372+
NODE_MYSQL_ALTER_TABLE, // command value; TABLE, name, comma-separated actions
373+
NODE_MYSQL_COLUMN_DEF, // name, type, structured attributes
374+
NODE_MYSQL_DDL_CLAUSE, // optional keyword prefix; space-separated operands
375+
NODE_MYSQL_DDL_LIST, // parenthesized comma-separated operands
376+
NODE_MYSQL_DDL_SYNTAX, // grammar-validated fixed token, not an arbitrary SQL tail
377+
NODE_MYSQL_ALTER_ACTIONS, // comma-separated action clauses
378+
NODE_MYSQL_JSON_TABLE, // document expression, path literal, columns
379+
NODE_MYSQL_JSON_TABLE_COLUMNS, // nonempty column/nested definitions
380+
NODE_MYSQL_JSON_TABLE_COLUMN, // name/type/path/responses; flags 1=ordinality, 2=exists
381+
NODE_MYSQL_JSON_TABLE_NESTED, // path literal, nested columns
382+
NODE_MYSQL_JSON_TABLE_RESPONSE, // validated action phrase; DEFAULT has literal child
383+
NODE_MYSQL_JSON_TABLE_LITERAL, // validated native literal source spelling
384+
NODE_MYSQL_CREATE_PROCEDURE, // name, parameters, characteristics, structured body
385+
NODE_MYSQL_PROCEDURE_PARAMS, // possibly empty typed parameter list
386+
NODE_MYSQL_PROCEDURE_PARAM, // direction value; name and validated type children
387+
NODE_MYSQL_PROCEDURE_CHARACTERISTIC, // validated phrase; COMMENT has string child
388+
NODE_MYSQL_PROCEDURE_BLOCK, // statement children; owns their semicolons
389+
NODE_MYSQL_OPTIMIZER_HINT, // lossless hint comment metadata; no hint semantic analysis
390+
370391
};
371392

372393
} // namespace sql_parser

‎include/sql_parser/delete_parser.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,15 @@ class DeleteParser {
3737
AstNode* root = make_node(arena_, NodeType::NODE_DELETE_STMT);
3838
if (!root) return nullptr;
3939

40+
if constexpr (D == Dialect::MySQL) {
41+
if (tok_.peek().type == TokenType::TK_MYSQL_OPTIMIZER_HINT) {
42+
auto token = tok_.next_token();
43+
auto* hint = make_node_from_token(arena_, NodeType::NODE_MYSQL_OPTIMIZER_HINT, token);
44+
if (!hint) return nullptr;
45+
root->add_child(hint);
46+
}
47+
}
48+
4049
if constexpr (D == Dialect::MySQL) {
4150
return parse_mysql(root);
4251
} else {

‎include/sql_parser/emitter.h‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,67 @@ class Emitter {
201201
sb_.append_char(')'); break;
202202
case NodeType::NODE_MYSQL_JSON_AGG_ARGUMENT:
203203
sb_.append("ALL "); emit_node(node->first_child); break;
204+
case NodeType::NODE_MYSQL_CREATE_TABLE:
205+
case NodeType::NODE_MYSQL_ALTER_TABLE:
206+
case NodeType::NODE_MYSQL_COLUMN_DEF:
207+
case NodeType::NODE_MYSQL_DDL_CLAUSE:
208+
case NodeType::NODE_MYSQL_PROCEDURE_PARAM:
209+
case NodeType::NODE_MYSQL_PROCEDURE_CHARACTERISTIC:
210+
emit_value(node);
211+
if (!node->value().empty() && node->first_child) sb_.append_char(' ');
212+
emit_list(node, " "); break;
213+
case NodeType::NODE_MYSQL_DDL_SYNTAX:
214+
case NodeType::NODE_MYSQL_JSON_TABLE_LITERAL:
215+
case NodeType::NODE_MYSQL_OPTIMIZER_HINT:
216+
emit_value(node); break;
217+
case NodeType::NODE_MYSQL_DDL_LIST:
218+
case NodeType::NODE_MYSQL_PROCEDURE_PARAMS:
219+
sb_.append_char('('); emit_list(node, ", "); sb_.append_char(')'); break;
220+
case NodeType::NODE_MYSQL_ALTER_ACTIONS:
221+
emit_list(node, ", "); break;
222+
case NodeType::NODE_MYSQL_JSON_TABLE: {
223+
sb_.append("JSON_TABLE(");
224+
const auto* c = node->first_child;
225+
emit_node(c); sb_.append(", ");
226+
c = c ? c->next_sibling : nullptr; emit_node(c); sb_.append_char(' ');
227+
emit_node(c ? c->next_sibling : nullptr); sb_.append_char(')'); break;
228+
}
229+
case NodeType::NODE_MYSQL_JSON_TABLE_COLUMNS:
230+
sb_.append("COLUMNS ("); emit_list(node, ", "); sb_.append_char(')'); break;
231+
case NodeType::NODE_MYSQL_JSON_TABLE_COLUMN: {
232+
const auto* c = node->first_child; emit_node(c);
233+
if (node->flags & 1) { sb_.append(" FOR ORDINALITY"); break; }
234+
c = c ? c->next_sibling : nullptr; sb_.append_char(' '); emit_node(c);
235+
sb_.append(node->flags & 2 ? " EXISTS PATH " : " PATH ");
236+
for (c = c ? c->next_sibling : nullptr; c; c = c->next_sibling) {
237+
emit_node(c); if (c->next_sibling) sb_.append_char(' ');
238+
}
239+
break;
240+
}
241+
case NodeType::NODE_MYSQL_JSON_TABLE_NESTED:
242+
sb_.append("NESTED PATH "); emit_list(node, " "); break;
243+
case NodeType::NODE_MYSQL_JSON_TABLE_RESPONSE:
244+
if (node->first_child) {
245+
sb_.append("DEFAULT "); emit_node(node->first_child);
246+
auto v = node->value();
247+
if (v.len >= 7) sb_.append(v.ptr + 7, v.len - 7);
248+
} else emit_value(node);
249+
break;
250+
case NodeType::NODE_MYSQL_CREATE_PROCEDURE: {
251+
sb_.append("CREATE PROCEDURE ");
252+
auto* c = node->first_child; emit_node(c);
253+
c = c ? c->next_sibling : nullptr; emit_node(c);
254+
for (c = c ? c->next_sibling : nullptr; c; c = c->next_sibling) {
255+
sb_.append_char(' '); emit_node(c);
256+
}
257+
break;
258+
}
259+
case NodeType::NODE_MYSQL_PROCEDURE_BLOCK:
260+
sb_.append("BEGIN ");
261+
for (auto* c = node->first_child; c; c = c->next_sibling) {
262+
emit_node(c); sb_.append("; ");
263+
}
264+
sb_.append("END"); break;
204265
case NodeType::NODE_MYSQL_EXPLAIN_INTO:
205266
sb_.append("INTO "); emit_node(node->first_child); break;
206267
case NodeType::NODE_MYSQL_CHARSET_LITERAL:
@@ -817,6 +878,7 @@ class Emitter {
817878
sb_.append("SELECT ");
818879
for (const AstNode* child = node->first_child; child; child = child->next_sibling) {
819880
emit_node(child);
881+
if (child->type == NodeType::NODE_MYSQL_OPTIMIZER_HINT) sb_.append_char(' ');
820882
}
821883
}
822884

‎include/sql_parser/expression_parser.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -809,6 +809,10 @@ class ExpressionParser {
809809

810810
AstNode* parse_identifier_or_function(const Token& name_token) {
811811
if constexpr (D == Dialect::MySQL) {
812+
// REPLACE is both a hintable statement keyword and a native
813+
// string function. MySQL discards its hint payload in the latter.
814+
if (name_token.type == TokenType::TK_REPLACE &&
815+
tok_.peek().type == TokenType::TK_MYSQL_OPTIMIZER_HINT) tok_.skip();
812816
if (tok_.peek().type == TokenType::TK_LPAREN &&
813817
(keyword(name_token, "CAST") || keyword(name_token, "CONVERT")))
814818
return parse_mysql_conversion(keyword(name_token, "CONVERT"));

‎include/sql_parser/insert_parser.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,15 @@ class InsertParser {
3636
auto* root = make_node(arena_, NodeType::NODE_INSERT_STMT, {},
3737
is_replace_ ? FLAG_REPLACE : uint16_t(0));
3838
if (!root) return error();
39+
if constexpr (D == Dialect::MySQL) {
40+
if (tok_.peek().type == TokenType::TK_MYSQL_OPTIMIZER_HINT) {
41+
auto token = tok_.next_token();
42+
auto* hint = make_node_from_token(arena_, NodeType::NODE_MYSQL_OPTIMIZER_HINT, token);
43+
if (!hint) return nullptr;
44+
root->add_child(hint);
45+
}
46+
}
47+
3948
if (auto* options = parse_stmt_options()) root->add_child(options);
4049
take(TokenType::TK_INTO);
4150

0 commit comments

Comments
 (0)