From bbee64ed73ea6eef8f1ab91b0647e17af73b3d23 Mon Sep 17 00:00:00 2001 From: Katze719 Date: Mon, 24 Aug 2026 21:54:40 +0200 Subject: [PATCH] refactor: remove FFI metadata export --- CMakeLists.txt | 107 ---------- README.md | 16 -- cmake/export_ast_json.cmake | 50 ----- tools/clang_ast_to_ffi_json.py | 345 --------------------------------- 4 files changed, 518 deletions(-) delete mode 100644 cmake/export_ast_json.cmake delete mode 100644 tools/clang_ast_to_ffi_json.py diff --git a/CMakeLists.txt b/CMakeLists.txt index e570faf..5a15526 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,26 +11,6 @@ set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_CXX_MODULE_STD 26) set(CMAKE_CXX_MODULE_EXTENSIONS OFF) -option(CPP_CORE_ENABLE_AST_EXPORT "Enable clang-based JSON AST export for the FFI headers" ON) -set( - CPP_CORE_AST_JSON_OUTPUT - "${CMAKE_BINARY_DIR}/ast/cpp_core_ffi_ast.json" - CACHE FILEPATH - "Output path for the generated FFI AST JSON file" -) -set( - CPP_CORE_AST_CLANGXX - "" - CACHE FILEPATH - "Path to the clang++ executable used for AST JSON export" -) -set( - CPP_CORE_AST_SLIM_JSON_OUTPUT - "${CMAKE_BINARY_DIR}/ast/cpp_core_ffi_api.json" - CACHE FILEPATH - "Output path for the reduced FFI API JSON metadata" -) - include(cmake/CPM.cmake) CPMAddPackage( @@ -93,85 +73,6 @@ if(BUILD_TESTING) target_link_libraries(cpp_core_compile_tests PRIVATE cpp_core_strict_warnings) endif() -if(CPP_CORE_ENABLE_AST_EXPORT) - find_package(Python3 COMPONENTS Interpreter REQUIRED) - - file( - GLOB _cpp_core_ast_interface_headers - CONFIGURE_DEPENDS - "${CMAKE_CURRENT_SOURCE_DIR}/include/cpp_core/interface/*.h" - ) - - set( - _cpp_core_ast_headers - "${CMAKE_CURRENT_SOURCE_DIR}/include/cpp_core/serial.h" - "${CMAKE_CURRENT_SOURCE_DIR}/include/cpp_core/error_callback.h" - "${CMAKE_CURRENT_SOURCE_DIR}/include/cpp_core/module_api.h" - "${CMAKE_CURRENT_SOURCE_DIR}/include/cpp_core/version.hpp" - ${_cpp_core_ast_interface_headers} - ) - - if(CPP_CORE_AST_CLANGXX) - set(_cpp_core_ast_clangxx "${CPP_CORE_AST_CLANGXX}") - else() - find_program(_cpp_core_ast_clangxx NAMES clang++) - endif() - - if(NOT _cpp_core_ast_clangxx) - message( - FATAL_ERROR - "CPP_CORE_ENABLE_AST_EXPORT=ON requires clang++. " - "Install clang++ or set CPP_CORE_AST_CLANGXX to an explicit path." - ) - endif() - - set(_cpp_core_ast_wrapper "${CMAKE_CURRENT_BINARY_DIR}/ast/cpp_core_ffi_ast.cpp") - get_filename_component(_cpp_core_ast_json_dir "${CPP_CORE_AST_JSON_OUTPUT}" DIRECTORY) - get_filename_component(_cpp_core_ast_slim_json_dir "${CPP_CORE_AST_SLIM_JSON_OUTPUT}" DIRECTORY) - - file( - GENERATE - OUTPUT "${_cpp_core_ast_wrapper}" - CONTENT "#include \n" - ) - - add_custom_command( - OUTPUT "${CPP_CORE_AST_JSON_OUTPUT}" - COMMAND ${CMAKE_COMMAND} -E make_directory "${_cpp_core_ast_json_dir}" - COMMAND - ${CMAKE_COMMAND} - -DCPP_CORE_AST_CLANGXX=${_cpp_core_ast_clangxx} - -DCPP_CORE_AST_INCLUDE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/include - -DCPP_CORE_AST_OUTPUT=${CPP_CORE_AST_JSON_OUTPUT} - -DCPP_CORE_AST_SOURCE=${_cpp_core_ast_wrapper} - -DCPP_CORE_AST_STANDARD=${CMAKE_CXX_STANDARD} - -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/export_ast_json.cmake - DEPENDS "${_cpp_core_ast_wrapper}" ${_cpp_core_ast_headers} - COMMENT "Exporting FFI header AST JSON with clang++" - VERBATIM - ) - - add_custom_command( - OUTPUT "${CPP_CORE_AST_SLIM_JSON_OUTPUT}" - COMMAND ${CMAKE_COMMAND} -E make_directory "${_cpp_core_ast_slim_json_dir}" - COMMAND - "${Python3_EXECUTABLE}" - "${CMAKE_CURRENT_SOURCE_DIR}/tools/clang_ast_to_ffi_json.py" - --input "${CPP_CORE_AST_JSON_OUTPUT}" - --output "${CPP_CORE_AST_SLIM_JSON_OUTPUT}" - --source-root "${CMAKE_CURRENT_SOURCE_DIR}" - --public-header "cpp_core/serial.h" - DEPENDS - "${CPP_CORE_AST_JSON_OUTPUT}" - "${CMAKE_CURRENT_SOURCE_DIR}/tools/clang_ast_to_ffi_json.py" - COMMENT "Reducing clang AST JSON to compact FFI API metadata" - VERBATIM - ) - - add_custom_target(cpp_core_ast_raw_json DEPENDS "${CPP_CORE_AST_JSON_OUTPUT}") - add_custom_target(cpp_core_ast_slim_json DEPENDS "${CPP_CORE_AST_SLIM_JSON_OUTPUT}") -endif() - # Install rules -------------------------------------------------------------- include(GNUInstallDirs) @@ -185,14 +86,6 @@ install( DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) -if(CPP_CORE_ENABLE_AST_EXPORT) - install( - FILES "${CPP_CORE_AST_SLIM_JSON_OUTPUT}" - DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/cpp_core - OPTIONAL - ) -endif() - # CMake package configuration ------------------------------------------------ include(CMakePackageConfigHelpers) diff --git a/README.md b/README.md index e6415f5..8dc5574 100644 --- a/README.md +++ b/README.md @@ -86,22 +86,6 @@ The CMake project exports the package target and also builds these relevant targ - `cpp_core::cpp_core`: header-only interface target - `cpp_core_compile_tests`: compile-time validation target when testing is enabled -Optional FFI AST export: - -```sh -cmake -S . -B build -G Ninja -DCMAKE_TOOLCHAIN_FILE=cmake/gcc-toolchain.cmake -DCPP_CORE_ENABLE_AST_EXPORT=ON -cmake --build build --target cpp_core_ast_slim_json -``` - -- The project still configures with GCC; the AST export itself invokes `clang++` separately -- The current FFI generation workflow is validated with `clang++ 22.1.4` (`Fedora 22.1.4-1.fc44`) -- `cpp_core_ast_raw_json` builds only the raw clang AST dump -- Requires `clang++` on `PATH` or `-DCPP_CORE_AST_CLANGXX=/path/to/clang++` -- Requires a Python 3 interpreter for the slim metadata reduction step -- Writes the combined FFI header AST JSON to `build/ast/cpp_core_ffi_ast.json` -- Writes compact, ship-friendly FFI metadata to `build/ast/cpp_core_ffi_api.json` -- Exports the `#include ` surface intended for downstream FFI adapter generation - ## ABI Surface The main aggregated interface lives in: diff --git a/cmake/export_ast_json.cmake b/cmake/export_ast_json.cmake deleted file mode 100644 index 0593d5a..0000000 --- a/cmake/export_ast_json.cmake +++ /dev/null @@ -1,50 +0,0 @@ -cmake_minimum_required(VERSION 3.30) - -foreach(required_var - CPP_CORE_AST_CLANGXX - CPP_CORE_AST_INCLUDE_DIR - CPP_CORE_AST_OUTPUT - CPP_CORE_AST_SOURCE - CPP_CORE_AST_STANDARD) - if(NOT DEFINED ${required_var} OR "${${required_var}}" STREQUAL "") - message(FATAL_ERROR "Missing required variable: ${required_var}") - endif() -endforeach() - -get_filename_component(_cpp_core_ast_output_dir "${CPP_CORE_AST_OUTPUT}" DIRECTORY) -file(MAKE_DIRECTORY "${_cpp_core_ast_output_dir}") - -execute_process( - COMMAND - "${CMAKE_COMMAND}" -E env - CCACHE_DISABLE=1 - "${CPP_CORE_AST_CLANGXX}" - "-std=c++${CPP_CORE_AST_STANDARD}" - "-I${CPP_CORE_AST_INCLUDE_DIR}" - -fsyntax-only - -Xclang - -ast-dump=json - "${CPP_CORE_AST_SOURCE}" - OUTPUT_FILE "${CPP_CORE_AST_OUTPUT}" - ERROR_VARIABLE _cpp_core_ast_stderr - RESULT_VARIABLE _cpp_core_ast_result -) - -if(NOT _cpp_core_ast_result EQUAL 0) - file(REMOVE "${CPP_CORE_AST_OUTPUT}") - string(STRIP "${_cpp_core_ast_stderr}" _cpp_core_ast_stderr) - message( - FATAL_ERROR - "clang AST export failed with exit code ${_cpp_core_ast_result}.\n${_cpp_core_ast_stderr}" - ) -endif() - -if(NOT EXISTS "${CPP_CORE_AST_OUTPUT}") - message(FATAL_ERROR "clang AST export did not produce ${CPP_CORE_AST_OUTPUT}") -endif() - -file(SIZE "${CPP_CORE_AST_OUTPUT}" _cpp_core_ast_size) -if(_cpp_core_ast_size EQUAL 0) - file(REMOVE "${CPP_CORE_AST_OUTPUT}") - message(FATAL_ERROR "clang AST export produced an empty file: ${CPP_CORE_AST_OUTPUT}") -endif() diff --git a/tools/clang_ast_to_ffi_json.py b/tools/clang_ast_to_ffi_json.py deleted file mode 100644 index 699e155..0000000 --- a/tools/clang_ast_to_ffi_json.py +++ /dev/null @@ -1,345 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import annotations - -import argparse -import json -from pathlib import Path -import re -from typing import Any - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Reduce a clang AST JSON dump to compact FFI API metadata.") - parser.add_argument("--input", required=True, help="Path to the full clang AST JSON file.") - parser.add_argument("--output", required=True, help="Path to the reduced output JSON file.") - parser.add_argument("--source-root", required=True, help="Repository root used for path normalization.") - parser.add_argument("--public-header", required=True, help="Stable public header exposed to consumers.") - return parser.parse_args() - - -def walk(node: Any): - stack = [node] - while stack: - current = stack.pop() - if isinstance(current, dict): - yield current - for value in reversed(list(current.values())): - if isinstance(value, (dict, list)): - stack.append(value) - elif isinstance(current, list): - for item in reversed(current): - if isinstance(item, (dict, list)): - stack.append(item) - - -def first_file(*candidates: Any) -> str | None: - for candidate in candidates: - if isinstance(candidate, dict): - file_value = candidate.get("file") - if file_value: - return file_value - return None - - -def declaration_file(node: dict[str, Any]) -> str | None: - loc = node.get("loc", {}) - range_begin = node.get("range", {}).get("begin", {}) - range_end = node.get("range", {}).get("end", {}) - return first_file( - range_begin.get("expansionLoc"), - range_begin.get("spellingLoc"), - loc, - range_begin, - range_end, - loc.get("includedFrom"), - range_begin.get("includedFrom"), - range_end.get("includedFrom"), - ) - - -def normalize_path(path: str | None, source_root: Path) -> str | None: - if not path: - return None - path_obj = Path(path).resolve() - try: - return path_obj.relative_to(source_root).as_posix() - except ValueError: - return path_obj.as_posix() - - -def normalize_header_path(path: str | None, source_root: Path) -> str | None: - normalized = normalize_path(path, source_root) - if normalized and normalized.startswith("include/"): - return normalized.removeprefix("include/") - return normalized - - -def has_default_visibility(node: dict[str, Any]) -> bool: - return any( - isinstance(child, dict) - and child.get("kind") == "VisibilityAttr" - and child.get("visibility") == "default" - for child in node.get("inner", []) - ) - - -def is_exported_function(node: dict[str, Any], source_root: Path) -> bool: - if node.get("kind") != "FunctionDecl" or not node.get("name"): - return False - if not has_default_visibility(node): - return False - - mangled_name = node.get("mangledName") - if mangled_name and mangled_name != node.get("name"): - return False - - path = normalize_path(declaration_file(node), source_root) - return bool(path and path.startswith("include/cpp_core/")) - - -def split_top_level(text: str) -> list[str]: - parts: list[str] = [] - current: list[str] = [] - angle_depth = 0 - paren_depth = 0 - square_depth = 0 - - for char in text: - if char == "<": - angle_depth += 1 - elif char == ">": - angle_depth = max(0, angle_depth - 1) - elif char == "(": - paren_depth += 1 - elif char == ")": - paren_depth = max(0, paren_depth - 1) - elif char == "[": - square_depth += 1 - elif char == "]": - square_depth = max(0, square_depth - 1) - - if char == "," and angle_depth == 0 and paren_depth == 0 and square_depth == 0: - part = "".join(current).strip() - if part: - parts.append(part) - current = [] - continue - - current.append(char) - - tail = "".join(current).strip() - if tail: - parts.append(tail) - return parts - - -def parse_function_pointer(type_name: str) -> dict[str, Any] | None: - signature_match = re.match(r"^(?P.+?)\s*\(\*\)\((?P.*)\)$", type_name) - if not signature_match: - return None - - params_text = signature_match.group("params").strip() - params = [] if not params_text or params_text == "void" else split_top_level(params_text) - return { - "returnType": signature_match.group("return_type").strip(), - "parameters": params, - } - - -def extract_default_expr(node: dict[str, Any] | None) -> Any: - if not node: - return None - - kind = node.get("kind") - if kind in {"ImplicitCastExpr", "ConstantExpr", "ExprWithCleanups"}: - for child in node.get("inner", []): - if isinstance(child, dict): - return extract_default_expr(child) - return None - if kind == "IntegerLiteral": - return node.get("value") - if kind == "FloatingLiteral": - return node.get("value") - if kind == "StringLiteral": - return node.get("value") - if kind == "CXXBoolLiteralExpr": - return node.get("value") - if kind == "CXXNullPtrLiteralExpr": - return "nullptr" - if kind == "DeclRefExpr": - referenced = node.get("referencedDecl", {}) - return referenced.get("name") or node.get("name") - if kind == "UnaryOperator": - inner = next((child for child in node.get("inner", []) if isinstance(child, dict)), None) - inner_value = extract_default_expr(inner) - if inner_value is None: - return None - return f"{node.get('opcode', '')}{inner_value}" - return None - - -def text_from_comment(node: dict[str, Any]) -> str: - kind = node.get("kind") - if kind == "TextComment": - return node.get("text", "") - if kind == "InlineCommandComment": - args = node.get("args", []) - if node.get("renderKind") == "monospaced": - return " ".join(f"`{arg}`" for arg in args) - return " ".join(args) - - chunks = [] - for child in node.get("inner", []): - if isinstance(child, dict): - text = text_from_comment(child) - if text: - chunks.append(text) - joined = " ".join(chunks) - joined = re.sub(r"\s+", " ", joined).strip() - joined = re.sub(r"`([^`]+?)([.,;:])`", r"`\1`\2", joined) - joined = re.sub(r"\s+([.,;])", r"\1", joined) - return joined - - -def extract_comment(node: dict[str, Any]) -> dict[str, Any]: - full_comment = next( - (child for child in node.get("inner", []) if isinstance(child, dict) and child.get("kind") == "FullComment"), - None, - ) - if not full_comment: - return {} - - doc: dict[str, Any] = {"params": {}} - details: list[str] = [] - - for child in full_comment.get("inner", []): - if not isinstance(child, dict): - continue - - kind = child.get("kind") - if kind == "BlockCommandComment" and child.get("name") == "brief": - brief = text_from_comment(child) - if brief: - doc["brief"] = brief - continue - - if kind == "BlockCommandComment" and child.get("name") in {"return", "returns"}: - returns = text_from_comment(child) - if returns: - doc["returns"] = returns - continue - - if kind == "ParamCommandComment": - name = child.get("param") - text = text_from_comment(child) - if name and text: - param_entry: dict[str, Any] = {"description": text} - direction = child.get("direction") - if direction: - param_entry["direction"] = direction - doc["params"][name] = param_entry - continue - - if kind == "ParagraphComment": - paragraph = text_from_comment(child) - if paragraph: - details.append(paragraph) - - if details: - doc["details"] = details - if not doc["params"]: - doc.pop("params") - return doc - - -def extract_return_type(function_type: str) -> str: - if "->" in function_type: - return function_type.rsplit("->", 1)[1].strip() - match = re.match(r"^(?P.+?)\s*\(", function_type) - return match.group("return_type").strip() if match else function_type - - -def build_parameter(node: dict[str, Any], doc_params: dict[str, Any]) -> dict[str, Any]: - type_info = node.get("type", {}) - public_type = type_info.get("qualType") - callback_source_type = type_info.get("desugaredQualType") or public_type - parameter: dict[str, Any] = { - "name": node.get("name", ""), - "type": public_type, - } - - default_expr = next((child for child in node.get("inner", []) if isinstance(child, dict)), None) - default_value = extract_default_expr(default_expr) - if default_value is not None: - parameter["default"] = default_value - - callback = parse_function_pointer(callback_source_type) - if callback: - parameter["callback"] = callback - - doc_entry = doc_params.get(parameter["name"]) - if doc_entry: - parameter["doc"] = doc_entry - - return parameter - - -def build_function(node: dict[str, Any], source_root: Path) -> dict[str, Any]: - type_name = node.get("type", {}).get("qualType", "") - doc = extract_comment(node) - doc_params = doc.pop("params", {}) - - function: dict[str, Any] = { - "name": node["name"], - "symbol": node.get("mangledName", node["name"]), - "declaredIn": normalize_header_path(declaration_file(node), source_root), - "returnType": extract_return_type(type_name), - "parameters": [ - build_parameter(child, doc_params) - for child in node.get("inner", []) - if isinstance(child, dict) and child.get("kind") == "ParmVarDecl" - ], - } - - if node.get("inline"): - function["inline"] = True - if doc: - function["doc"] = doc - - return function - - -def main() -> int: - args = parse_args() - source_root = Path(args.source_root).resolve() - input_path = Path(args.input) - output_path = Path(args.output) - - with input_path.open(encoding="utf-8") as input_file: - ast = json.load(input_file) - - functions = [ - build_function(node, source_root) - for node in walk(ast) - if is_exported_function(node, source_root) - ] - functions.sort(key=lambda item: item["name"]) - - metadata = { - "schema": "cpp_core_ffi_api", - "schemaVersion": 1, - "publicHeader": args.public_header, - "functions": functions, - } - - output_path.parent.mkdir(parents=True, exist_ok=True) - with output_path.open("w", encoding="utf-8") as output_file: - json.dump(metadata, output_file, indent=2) - output_file.write("\n") - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main())