Skip to content

Commit ba8e3e7

Browse files
renecannaorenecannao
committed
feat: add real MySQL + PostgreSQL backend connections via libmysqlclient/libpq
Implements MySQLRemoteExecutor, PgSQLRemoteExecutor, and MultiRemoteExecutor for executing queries against real database backends. Includes datetime parsing utilities, Docker-based test infrastructure, and 37 integration tests that skip gracefully when backends are not available.
1 parent 1ae5c66 commit ba8e3e7

17 files changed

Lines changed: 1684 additions & 5 deletions

‎Makefile.new‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@ CXX = g++
22
CXXFLAGS = -std=c++17 -Wall -Wextra -g -O2
33
CPPFLAGS = -I./include -I./third_party/googletest/googletest/include
44

5+
# MySQL and PostgreSQL client libraries
6+
MYSQL_CFLAGS = $(shell mysql_config --cflags 2>/dev/null)
7+
MYSQL_LIBS = $(shell mysql_config --libs 2>/dev/null)
8+
PG_CFLAGS = -I$(shell pg_config --includedir 2>/dev/null || echo /usr/include/postgresql)
9+
PG_LIBS = -L$(shell pg_config --libdir 2>/dev/null || echo /usr/lib/x86_64-linux-gnu) -lpq
10+
511
PROJECT_ROOT = .
612
SRC_DIR = $(PROJECT_ROOT)/src/sql_parser
713
ENGINE_SRC_DIR = $(PROJECT_ROOT)/src/sql_engine
@@ -15,7 +21,11 @@ LIB_TARGET = $(PROJECT_ROOT)/libsqlparser.a
1521

1622
# SQL Engine sources
1723
ENGINE_SRCS = $(ENGINE_SRC_DIR)/function_registry.cpp \
18-
$(ENGINE_SRC_DIR)/in_memory_catalog.cpp
24+
$(ENGINE_SRC_DIR)/in_memory_catalog.cpp \
25+
$(ENGINE_SRC_DIR)/datetime_parse.cpp \
26+
$(ENGINE_SRC_DIR)/mysql_remote_executor.cpp \
27+
$(ENGINE_SRC_DIR)/pgsql_remote_executor.cpp \
28+
$(ENGINE_SRC_DIR)/multi_remote_executor.cpp
1929
ENGINE_OBJS = $(ENGINE_SRCS:.cpp=.o)
2030

2131
# Google Test library
@@ -59,7 +69,10 @@ TEST_SRCS = $(TEST_DIR)/test_main.cpp \
5969
$(TEST_DIR)/test_optimizer.cpp \
6070
$(TEST_DIR)/test_distributed_planner.cpp \
6171
$(TEST_DIR)/test_dml.cpp \
62-
$(TEST_DIR)/test_distributed_dml.cpp
72+
$(TEST_DIR)/test_distributed_dml.cpp \
73+
$(TEST_DIR)/test_mysql_executor.cpp \
74+
$(TEST_DIR)/test_pgsql_executor.cpp \
75+
$(TEST_DIR)/test_distributed_real.cpp
6376
TEST_OBJS = $(TEST_SRCS:.cpp=.o)
6477
TEST_TARGET = $(PROJECT_ROOT)/run_tests
6578

@@ -95,21 +108,21 @@ $(SRC_DIR)/%.o: $(SRC_DIR)/%.cpp
95108

96109
# SQL Engine objects
97110
$(ENGINE_SRC_DIR)/%.o: $(ENGINE_SRC_DIR)/%.cpp
98-
$(CXX) $(CXXFLAGS) $(CPPFLAGS) -c $< -o $@
111+
$(CXX) $(CXXFLAGS) $(CPPFLAGS) $(MYSQL_CFLAGS) $(PG_CFLAGS) -c $< -o $@
99112

100113
# Google Test object
101114
$(GTEST_OBJ): $(GTEST_SRC)
102115
$(CXX) $(CXXFLAGS) $(GTEST_CPPFLAGS) -c $< -o $@
103116

104117
# Test objects
105118
$(TEST_DIR)/%.o: $(TEST_DIR)/%.cpp
106-
$(CXX) $(CXXFLAGS) $(CPPFLAGS) $(GTEST_CPPFLAGS) -c $< -o $@
119+
$(CXX) $(CXXFLAGS) $(CPPFLAGS) $(GTEST_CPPFLAGS) $(MYSQL_CFLAGS) $(PG_CFLAGS) -c $< -o $@
107120

108121
test: $(TEST_TARGET)
109122
./$(TEST_TARGET)
110123

111124
$(TEST_TARGET): $(TEST_OBJS) $(GTEST_OBJ) $(LIB_TARGET) $(ENGINE_OBJS)
112-
$(CXX) $(CXXFLAGS) -o $@ $(TEST_OBJS) $(GTEST_OBJ) $(ENGINE_OBJS) -L$(PROJECT_ROOT) -lsqlparser -lpthread
125+
$(CXX) $(CXXFLAGS) -o $@ $(TEST_OBJS) $(GTEST_OBJ) $(ENGINE_OBJS) -L$(PROJECT_ROOT) -lsqlparser -lpthread $(MYSQL_LIBS) $(PG_LIBS)
113126

114127
# Benchmark objects
115128
$(GBENCH_DIR)/src/%.o: $(GBENCH_DIR)/src/%.cc

‎include/sql_engine/backend_config.h‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#ifndef SQL_ENGINE_BACKEND_CONFIG_H
2+
#define SQL_ENGINE_BACKEND_CONFIG_H
3+
4+
#include "sql_parser/common.h"
5+
#include <cstdint>
6+
#include <string>
7+
8+
namespace sql_engine {
9+
10+
struct BackendConfig {
11+
std::string name; // logical name: "shard_1", "analytics_db"
12+
std::string host;
13+
uint16_t port = 0;
14+
std::string user;
15+
std::string password;
16+
std::string database;
17+
sql_parser::Dialect dialect = sql_parser::Dialect::MySQL;
18+
};
19+
20+
} // namespace sql_engine
21+
22+
#endif // SQL_ENGINE_BACKEND_CONFIG_H

‎include/sql_engine/datetime_parse.h‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#ifndef SQL_ENGINE_DATETIME_PARSE_H
2+
#define SQL_ENGINE_DATETIME_PARSE_H
3+
4+
#include <cstdint>
5+
6+
namespace sql_engine {
7+
namespace datetime_parse {
8+
9+
// Parse "YYYY-MM-DD" to days since 1970-01-01.
10+
int32_t parse_date(const char* s);
11+
12+
// Parse "YYYY-MM-DD HH:MM:SS[.uuuuuu]" to microseconds since epoch.
13+
int64_t parse_datetime(const char* s);
14+
15+
// Parse "HH:MM:SS[.uuuuuu]" to microseconds since midnight.
16+
int64_t parse_time(const char* s);
17+
18+
// Calendar math: compute days since 1970-01-01 for the given date.
19+
// Handles leap years correctly.
20+
int32_t days_since_epoch(int year, int month, int day);
21+
22+
} // namespace datetime_parse
23+
} // namespace sql_engine
24+
25+
#endif // SQL_ENGINE_DATETIME_PARSE_H

‎include/sql_engine/multi_remote_executor.h‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#ifndef SQL_ENGINE_MULTI_REMOTE_EXECUTOR_H
2+
#define SQL_ENGINE_MULTI_REMOTE_EXECUTOR_H
3+
4+
#include "sql_engine/remote_executor.h"
5+
#include "sql_engine/mysql_remote_executor.h"
6+
#include "sql_engine/pgsql_remote_executor.h"
7+
#include "sql_engine/backend_config.h"
8+
#include "sql_parser/arena.h"
9+
#include "sql_parser/common.h"
10+
11+
#include <unordered_map>
12+
#include <string>
13+
14+
namespace sql_engine {
15+
16+
class MultiRemoteExecutor : public RemoteExecutor {
17+
public:
18+
explicit MultiRemoteExecutor(sql_parser::Arena& arena);
19+
~MultiRemoteExecutor() override;
20+
21+
void add_backend(const BackendConfig& config);
22+
ResultSet execute(const char* backend_name, sql_parser::StringRef sql) override;
23+
DmlResult execute_dml(const char* backend_name, sql_parser::StringRef sql) override;
24+
void disconnect_all();
25+
26+
private:
27+
MySQLRemoteExecutor mysql_exec_;
28+
PgSQLRemoteExecutor pgsql_exec_;
29+
std::unordered_map<std::string, sql_parser::Dialect> backend_dialects_;
30+
};
31+
32+
} // namespace sql_engine
33+
34+
#endif // SQL_ENGINE_MULTI_REMOTE_EXECUTOR_H

‎include/sql_engine/mysql_remote_executor.h‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#ifndef SQL_ENGINE_MYSQL_REMOTE_EXECUTOR_H
2+
#define SQL_ENGINE_MYSQL_REMOTE_EXECUTOR_H
3+
4+
#include "sql_engine/remote_executor.h"
5+
#include "sql_engine/backend_config.h"
6+
#include "sql_engine/value.h"
7+
#include "sql_engine/row.h"
8+
#include "sql_parser/arena.h"
9+
10+
#include <mysql/mysql.h>
11+
#include <unordered_map>
12+
#include <string>
13+
14+
namespace sql_engine {
15+
16+
class MySQLRemoteExecutor : public RemoteExecutor {
17+
public:
18+
explicit MySQLRemoteExecutor(sql_parser::Arena& arena);
19+
~MySQLRemoteExecutor() override;
20+
21+
void add_backend(const BackendConfig& config);
22+
ResultSet execute(const char* backend_name, sql_parser::StringRef sql) override;
23+
DmlResult execute_dml(const char* backend_name, sql_parser::StringRef sql) override;
24+
void disconnect_all();
25+
26+
private:
27+
struct Connection {
28+
BackendConfig config;
29+
MYSQL* conn = nullptr;
30+
bool connected = false;
31+
};
32+
33+
std::unordered_map<std::string, Connection> backends_;
34+
sql_parser::Arena& arena_;
35+
36+
Connection& get_or_connect(const std::string& name);
37+
ResultSet mysql_result_to_resultset(MYSQL_RES* res);
38+
Value mysql_field_to_value(const char* data, unsigned long length,
39+
enum_field_types type, bool is_null);
40+
};
41+
42+
} // namespace sql_engine
43+
44+
#endif // SQL_ENGINE_MYSQL_REMOTE_EXECUTOR_H

‎include/sql_engine/pgsql_remote_executor.h‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#ifndef SQL_ENGINE_PGSQL_REMOTE_EXECUTOR_H
2+
#define SQL_ENGINE_PGSQL_REMOTE_EXECUTOR_H
3+
4+
#include "sql_engine/remote_executor.h"
5+
#include "sql_engine/backend_config.h"
6+
#include "sql_engine/value.h"
7+
#include "sql_engine/row.h"
8+
#include "sql_parser/arena.h"
9+
10+
#include <libpq-fe.h>
11+
#include <unordered_map>
12+
#include <string>
13+
14+
namespace sql_engine {
15+
16+
class PgSQLRemoteExecutor : public RemoteExecutor {
17+
public:
18+
explicit PgSQLRemoteExecutor(sql_parser::Arena& arena);
19+
~PgSQLRemoteExecutor() override;
20+
21+
void add_backend(const BackendConfig& config);
22+
ResultSet execute(const char* backend_name, sql_parser::StringRef sql) override;
23+
DmlResult execute_dml(const char* backend_name, sql_parser::StringRef sql) override;
24+
void disconnect_all();
25+
26+
private:
27+
struct Connection {
28+
BackendConfig config;
29+
PGconn* conn = nullptr;
30+
bool connected = false;
31+
};
32+
33+
std::unordered_map<std::string, Connection> backends_;
34+
sql_parser::Arena& arena_;
35+
36+
Connection& get_or_connect(const std::string& name);
37+
ResultSet pg_result_to_resultset(PGresult* res);
38+
Value pg_field_to_value(const char* data, int length, Oid type, bool is_null);
39+
};
40+
41+
} // namespace sql_engine
42+
43+
#endif // SQL_ENGINE_PGSQL_REMOTE_EXECUTOR_H

‎scripts/start_test_backends.sh‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/bin/bash
2+
set -e
3+
4+
echo "Starting test backends..."
5+
6+
# Remove any existing containers
7+
docker rm -f parsersql-test-mysql parsersql-test-pgsql 2>/dev/null || true
8+
9+
# Start MySQL 8
10+
docker run -d --name parsersql-test-mysql \
11+
-p 13306:3306 \
12+
-e MYSQL_ROOT_PASSWORD=test \
13+
-e MYSQL_DATABASE=testdb \
14+
mysql:8.0
15+
16+
# Start PostgreSQL 16
17+
docker run -d --name parsersql-test-pgsql \
18+
-p 15432:5432 \
19+
-e POSTGRES_PASSWORD=test \
20+
-e POSTGRES_DB=testdb \
21+
postgres:16
22+
23+
# Wait for MySQL to be ready
24+
echo "Waiting for MySQL..."
25+
for i in $(seq 1 60); do
26+
if docker exec parsersql-test-mysql mysql -uroot -ptest -e "SELECT 1" &>/dev/null 2>&1; then
27+
echo "MySQL ready after ${i}s"
28+
break
29+
fi
30+
if [ "$i" -eq 60 ]; then
31+
echo "MySQL failed to start"
32+
exit 1
33+
fi
34+
sleep 1
35+
done
36+
37+
# Wait for PostgreSQL to be ready
38+
echo "Waiting for PostgreSQL..."
39+
for i in $(seq 1 60); do
40+
if docker exec parsersql-test-pgsql pg_isready -Upostgres &>/dev/null 2>&1; then
41+
echo "PostgreSQL ready after ${i}s"
42+
break
43+
fi
44+
if [ "$i" -eq 60 ]; then
45+
echo "PostgreSQL failed to start"
46+
exit 1
47+
fi
48+
sleep 1
49+
done
50+
51+
# Load test data
52+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
53+
echo "Loading MySQL test data..."
54+
docker exec -i parsersql-test-mysql mysql -uroot -ptest testdb < "${SCRIPT_DIR}/test_data_mysql.sql"
55+
echo "Loading PostgreSQL test data..."
56+
docker exec -i parsersql-test-pgsql psql -Upostgres testdb < "${SCRIPT_DIR}/test_data_pgsql.sql"
57+
58+
echo "Backends ready!"

‎scripts/stop_test_backends.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
docker rm -f parsersql-test-mysql parsersql-test-pgsql 2>/dev/null || true
3+
echo "Test backends stopped."

‎scripts/test_data_mysql.sql‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
DROP TABLE IF EXISTS orders;
2+
DROP TABLE IF EXISTS users;
3+
4+
CREATE TABLE users (
5+
id INT PRIMARY KEY,
6+
name VARCHAR(255),
7+
age INT,
8+
dept VARCHAR(100)
9+
);
10+
11+
INSERT INTO users VALUES
12+
(1, 'Alice', 30, 'Engineering'),
13+
(2, 'Bob', 25, 'Sales'),
14+
(3, 'Carol', 35, 'Engineering'),
15+
(4, 'Dave', 28, 'Sales'),
16+
(5, 'Eve', 32, 'Engineering');
17+
18+
CREATE TABLE orders (
19+
id INT PRIMARY KEY,
20+
user_id INT,
21+
total DECIMAL(10,2),
22+
status VARCHAR(50)
23+
);
24+
25+
INSERT INTO orders VALUES
26+
(101, 1, 150.00, 'completed'),
27+
(102, 2, 75.50, 'pending'),
28+
(103, 1, 200.00, 'completed'),
29+
(104, 3, 50.00, 'cancelled');

‎scripts/test_data_pgsql.sql‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
DROP TABLE IF EXISTS orders;
2+
DROP TABLE IF EXISTS users;
3+
4+
CREATE TABLE users (
5+
id INT PRIMARY KEY,
6+
name VARCHAR(255),
7+
age INT,
8+
dept VARCHAR(100)
9+
);
10+
11+
INSERT INTO users VALUES
12+
(1, 'Alice', 30, 'Engineering'),
13+
(2, 'Bob', 25, 'Sales'),
14+
(3, 'Carol', 35, 'Engineering'),
15+
(4, 'Dave', 28, 'Sales'),
16+
(5, 'Eve', 32, 'Engineering');
17+
18+
CREATE TABLE orders (
19+
id INT PRIMARY KEY,
20+
user_id INT,
21+
total DECIMAL(10,2),
22+
status VARCHAR(50)
23+
);
24+
25+
INSERT INTO orders VALUES
26+
(101, 1, 150.00, 'completed'),
27+
(102, 2, 75.50, 'pending'),
28+
(103, 1, 200.00, 'completed'),
29+
(104, 3, 50.00, 'cancelled');

0 commit comments

Comments
 (0)