diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml index 6fdbff835..ebacd5766 100644 --- a/.github/workflows/presubmit.yml +++ b/.github/workflows/presubmit.yml @@ -234,7 +234,7 @@ jobs: make -C backends/cuda tracer_cuda.c cuda_library.rb btx_cuda_model.yaml libcuda.la make -C backends/cuda tracer_cudart.c libcudart.la make -C backends/ze tracer_ze.c ze_library.rb btx_ze_model.yaml libze_loader.la - make -C backends/opencl libOpenCL.la + make -C backends/opencl opencl_model.yaml btx_cl_model.yaml libOpenCL.la make -C backends/cxi cxi_sampling.tp make -C backends/itt tracer_itt.c itt_library.rb btx_itt_model.yaml libittnotify.la make -C backends/opencl opencl_profiling.tp diff --git a/.gitignore b/.gitignore index fa728da51..362d83d4e 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ m4 aclocal.m4 configure Makefile.in +__pycache__ diff --git a/backends/cuda/Makefile.am b/backends/cuda/Makefile.am index 342b869e2..8188e0736 100644 --- a/backends/cuda/Makefile.am +++ b/backends/cuda/Makefile.am @@ -121,32 +121,35 @@ ML_CUDA_FILES = \ $(srcdir)/cuda_meta_parameters.yaml \ $(srcdir)/cuda_exports_meta_parameters.yaml -ML_CUDART_FILES = \ - $(srcdir)/cudart_meta_parameters.yaml - -EXTRA_DIST += $(ML_CUDA_FILES) $(ML_CUDART_FILES) +EXTRA_DIST += $(ML_CUDA_FILES) CUDA_MODEL = \ $(srcdir)/cuda_model.rb \ $(ML_CUDA_FILES) \ $(srcdir)/cuda_events.yaml \ $(CUDA_EXTRACTED) \ + $(top_srcdir)/utils/api_model.rb \ $(top_srcdir)/utils/yaml_ast.rb \ $(top_srcdir)/utils/yaml_ast_lttng.rb \ $(top_srcdir)/utils/meta_parameters.rb \ $(top_srcdir)/utils/LTTng.rb \ $(top_srcdir)/utils/command.rb \ + $(top_srcdir)/utils/command_index.rb \ + $(top_srcdir)/utils/meta_parameter_spec.rb \ $(srcdir)/cuda.h.include \ cuda_exports.h.include CUDART_MODEL = \ $(srcdir)/cudart_model.rb \ - $(ML_CUDART_FILES) \ $(CUDART_EXTRACTED) \ + $(top_srcdir)/utils/api_model.rb \ $(top_srcdir)/utils/yaml_ast.rb \ $(top_srcdir)/utils/yaml_ast_lttng.rb \ $(top_srcdir)/utils/meta_parameters.rb \ $(top_srcdir)/utils/LTTng.rb \ + $(top_srcdir)/utils/command.rb \ + $(top_srcdir)/utils/command_index.rb \ + $(top_srcdir)/utils/meta_parameter_spec.rb \ $(srcdir)/cudart.h.include btx_cuda_model.yaml: $(srcdir)/gen_babeltrace_cuda_model.rb $(CUDA_LIB_GEN) $(CUDA_MODEL) @@ -306,6 +309,8 @@ install-data-hook: CUDA_LIB_GEN = \ $(top_srcdir)/utils/gen_babeltrace_model_helper.rb \ + $(top_srcdir)/utils/api_model.rb \ + $(top_srcdir)/utils/type_registry.rb \ $(top_srcdir)/utils/gen_library_base.rb \ $(srcdir)/gen_cuda_library_base.rb \ $(top_srcdir)/utils/gen_probe_base.rb diff --git a/backends/cuda/cuda_model.rb b/backends/cuda/cuda_model.rb index a02a0b065..23d34e095 100644 --- a/backends/cuda/cuda_model.rb +++ b/backends/cuda/cuda_model.rb @@ -1,6 +1,7 @@ require 'yaml' require 'pp' require 'set' +require_relative '../../utils/api_model' require_relative '../../utils/yaml_ast_lttng' require_relative '../../utils/LTTng' require_relative '../../utils/command' @@ -8,29 +9,26 @@ SRC_DIR = ENV['SRC_DIR'] || '.' -RESULT_NAME = 'cuResult' +# CUdeviceptr is a device address carried in an integer, so it reads as hex +# rather than as a decimal that means nothing to anyone. +cuda_api = ApiModel.load_file('cuda_api.yaml', hex_ints: ['CUdeviceptr']) +cuda_exports_api = ApiModel.load_file('cuda_exports_api.yaml') -$cuda_api_versions_yaml = YAML.load_file('cuda_api_versions.yaml') -$cuda_api_yaml = YAML.load_file('cuda_api.yaml') -$cuda_exports_api_yaml = YAML.load_file('cuda_exports_api.yaml') +# The driver and its export tables are traced as one API: an export-table +# typedef can name a driver struct, so they have to be classified together. +API = cuda_api + cuda_exports_api -$cuda_api = YAMLCAst.from_yaml_ast($cuda_api_yaml) -$cuda_exports_api = YAMLCAst.from_yaml_ast($cuda_exports_api_yaml) +gen_ffi_type_map(API.types, API.type_classes) -cuda_funcs_e = $cuda_api['functions'] -cuda_exports_funcs_e = $cuda_exports_api['functions'] - -cuda_types_e = $cuda_api['typedefs'] -cuda_exports_type_e = $cuda_exports_api['typedefs'] - -typedefs = cuda_types_e + cuda_exports_type_e -structs = $cuda_api['structs'] + $cuda_exports_api['structs'] - -find_all_types(typedefs) -gen_struct_map(typedefs, structs) -gen_ffi_type_map(typedefs) - -HEX_INT_TYPES.push('CUdeviceptr') +CONTEXT = BackendContext.new( + result_name: 'cuResult', + # cuda's pointers start at a _uninit trampoline that calls _init_tracer(), so + # no function is singled out as the initializer and Command#init? is never + # asked. + init_functions: nil, + struct_map: API.struct_map, + type_classes: API.type_classes +) class TracepointParameter attr_reader :name, :type, :init, :after @@ -81,33 +79,19 @@ def initialize(command, name) end end -$cuda_meta_parameters = YAML.load_file(File.join(SRC_DIR, 'cuda_meta_parameters.yaml')) -$cuda_meta_parameters['meta_parameters'].each do |func, list| - list.each do |type, *args| - register_meta_parameter func, Kernel.const_get(type), *args - end -end -$cuda_exports_meta_parameters = YAML.load_file(File.join(SRC_DIR, 'cuda_exports_meta_parameters.yaml')) -$cuda_exports_meta_parameters['meta_parameters'].each do |func, list| - list.each do |type, *args| - register_meta_parameter func, Kernel.const_get(type), *args - end -end - -$cuda_commands = cuda_funcs_e.collect do |func| - Command.new(func) -end +meta_parameters = load_meta_parameters('cuda_meta_parameters.yaml', 'cuda_exports_meta_parameters.yaml') -$cuda_exports_commands = cuda_exports_funcs_e.collect do |func| - Command.new(func) -end +# The driver and its export tables are one API but two LTTng providers, so the +# commands are grouped by the provider that will carry them. +COMMANDS = CommandIndex.new( + { lttng_ust_cuda: cuda_api.functions, lttng_ust_cuda_exports: cuda_exports_api.functions }.transform_values do |funcs| + funcs.collect { |func| Command.new(func, context: CONTEXT, meta_parameters: meta_parameters[func.name]) } + end +) -def upper_snake_case(str) - str.gsub(/([A-Z][A-Z0-9]*)/, '_\1').upcase -end +check_meta_parameters(meta_parameters, COMMANDS) -CUDA_POINTER_NAMES = ($cuda_commands + - $cuda_exports_commands).collect do |c| +CUDA_POINTER_NAMES = COMMANDS.collect do |c| [c, upper_snake_case(c.pointer_name)] end.to_h @@ -119,7 +103,7 @@ def upper_snake_case(str) cuLaunchKernel_ptsz cuLaunchKernelEx cuLaunchKernelEx_ptsz].each do |m| - register_prologue m, dump_args + COMMANDS.add_prologue m, dump_args end dump_args = <func, nodeParams->kernelParams, nodeParams->extra); } @@ -192,7 +176,7 @@ def upper_snake_case(str) stream_commands = [] no_stream_commands = [] -mem_commands = $cuda_commands.select { |c| c.name.match(/cuMemcpy|cuMemset/) } +mem_commands = COMMANDS.groups[:lttng_ust_cuda].select { |c| c.name.match(/cuMemcpy|cuMemset/) } mem_stream_commands = mem_commands.select { |c| c.name.match(/Async/) } mem_no_stream_commands = mem_commands - mem_stream_commands stream_commands += mem_stream_commands.collect(&:name) @@ -221,31 +205,31 @@ def upper_snake_case(str) ] stream_commands.each do |m| - register_prologue m, profiling_start_stream - register_epilogue m, profiling_stop_stream + COMMANDS.add_prologue m, profiling_start_stream + COMMANDS.add_epilogue m, profiling_stop_stream end no_stream_commands.each do |m| - register_prologue m, profiling_start_no_stream - register_epilogue m, profiling_stop_no_stream + COMMANDS.add_prologue m, profiling_start_no_stream + COMMANDS.add_epilogue m, profiling_stop_no_stream end config_commands.each do |m| - register_prologue m, profiling_start_config - register_epilogue m, profiling_stop_config + COMMANDS.add_prologue m, profiling_start_config + COMMANDS.add_epilogue m, profiling_stop_config end # if a context is to be destroyed we must attempt to get profiling event results %w[cuCtxDestroy cuCtxDestroy_v2].each do |m| - register_prologue m, < suffix -> versions map, used only here. +api_versions = YAML.load_file('cuda_api_versions.yaml') +command_names = COMMANDS.groups[:lttng_ust_cuda].collect(&:name).to_set pt_condition = '((flags & CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM) && !(flags & CU_GET_PROC_ADDRESS_LEGACY_STREAM))' normal_condition = '((flags & CU_GET_PROC_ADDRESS_LEGACY_STREAM) || !(flags & CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM))' @@ -296,7 +282,7 @@ def upper_snake_case(str) fprintf(stderr, "THAPI: CUDA version %d is unsupported, could not wrap %s symbol\\n", cudaVersion, symbol); } else if (_retval == CUDA_SUCCESS && pfn && *pfn) { EOF - str << $cuda_api_versions_yaml.map { |name, suffixes| + str << api_versions.map { |name, suffixes| suffixes.map { |suffix, versions| versions.map.with_index { |version, i| fullname = "#{name}#{"_v#{versions.size - i}" if versions.size - i > 1}#{suffix}" @@ -313,20 +299,11 @@ def upper_snake_case(str) }.flatten.join(< -# EOF -($cuda_commands + $cuda_exports_commands).each do |c| +COMMANDS.each do |c| puts "#define #{CUDA_POINTER_NAMES[c]} #{c.pointer_name}" end -$cuda_commands.each do |c| +COMMANDS.groups[:lttng_ust_cuda].each do |c| puts <<~EOF static #{YAMLCAst::Declaration.new(name: c.name + '_unsupp', type: c.function.type)} { @@ -52,7 +49,7 @@ EOF end -$cuda_exports_commands.each do |c| +COMMANDS.groups[:lttng_ust_cuda_exports].each do |c| puts <<~EOF #{c.decl_pointer(c.pointer_type_name)}; @@ -60,14 +57,11 @@ EOF end -$cuda_commands.each do |c| +COMMANDS.groups[:lttng_ust_cuda].each do |c| puts <<~EOF #{c.decl_hidden_alias}; static void wrap_#{c.name}(void **pfn); EOF - # puts <c_ptr; - # } else { - # closure = (struct cuda_closure *)malloc(sizeof(struct cuda_closure) + #{c.parameters.size} * sizeof(ffi_type *)); - # if (closure != NULL) { - # closure->types = (ffi_type **)((intptr_t)closure + sizeof(struct cuda_closure)); - # closure->closure = ffi_closure_alloc(sizeof(ffi_closure), &(closure->c_ptr)); - # if (closure->closure != NULL) { - # closure->ptr = *pfn; - # EOF - # c.parameters.each_with_index { |a, i| - # if a.type.kind_of?(YAMLCAst::Pointer) - # ffi_type = "ffi_type_pointer" - # elsif FFI_TYPE_MAP["#{a.type}"] - # ffi_type = FFI_TYPE_MAP["#{a.type}"] - # else - # raise "Unsupported type: #{a.type}" - # end - # str << <types[#{i}] = &#{ffi_type}; - # EOF - # } - # if c.type.kind_of?(YAMLCAst::Void) - # ffi_ret_type = "ffi_type_void" - # elsif c.type.kind_of?(YAMLCAst::Pointer) - # ffi_ret_type = "ffi_type_pointer" - # elsif FFI_TYPE_MAP["#{c.type}"] - # ffi_ret_type = FFI_TYPE_MAP["#{c.type}"] - # else - # raise "Unsupported type: #{c.type}" - # end - # str << <cif), FFI_DEFAULT_ABI, #{c.parameters.size}, &#{ffi_ret_type}, closure->types) == FFI_OK) { - # if (ffi_prep_closure_loc(closure->closure, &(closure->cif), (void (*)(ffi_cif*, void *, void **, void *))#{c.ffi_name}, *pfn, closure->c_ptr) == FFI_OK) { - # pthread_mutex_lock(&cuda_closures_mutex); - # HASH_ADD_PTR(cuda_closures, ptr, closure); - # pthread_mutex_unlock(&cuda_closures_mutex); - # *pfn = closure->c_ptr; - # } else { - # ffi_closure_free(closure->closure); - # free(closure); - # } - # } else { - # ffi_closure_free(closure->closure); - # free(closure); - # } - # } else { - # free(closure); - # } - # } - # } - # EOF - # rescue => e - # str = < LTTNG_USABLE_PARAMS - $tracepoint_lambda.call(provider, c, :start) - $tracepoint_lambda.call(provider, c, :stop) + print_tracepoint(provider, c, :start) + print_tracepoint(provider, c, :stop) end diff --git a/backends/cuda/gen_cuda_library.rb b/backends/cuda/gen_cuda_library.rb index a9d1ecc20..3ac330e66 100644 --- a/backends/cuda/gen_cuda_library.rb +++ b/backends/cuda/gen_cuda_library.rb @@ -1,13 +1,5 @@ require_relative 'gen_cuda_library_base' -def print_enum(name, enum) - print_enum_with_namespace(:CUDA, name, enum) -end - -def print_cuda_object(object) - print_object(object) -end - print_ffi_module(:CUDA) puts <<~EOF @@ -19,53 +11,10 @@ module CUDA CUDA_IPC_HANDLE_SIZE = 64 extend FFI::Library - module Handle - def to_s - s = '{ reserved: "' - s << self[:reserved].to_a.collect { |v| "\\\\x%02x" % ((v + 256)%256) }.join - s << '" }' - end - end - - module UUID - def to_s - a = self[:bytes].to_a.collect { |v| v < 0 ? 0x100 + v : v } - s = "{ id: " - s << "%02x" % a[0] - s << "%02x" % a[1] - s << "%02x" % a[2] - s << "%02x" % a[3] - s << "-" - s << "%02x" % a[4] - s << "%02x" % a[5] - s << "-" - s << "%02x" % a[6] - s << "%02x" % a[7] - s << "-" - s << "%02x" % a[8] - s << "%02x" % a[9] - s << "-" - s << "%02x" % a[10] - s << "%02x" % a[11] - s << "%02x" % a[12] - s << "%02x" % a[13] - s << "%02x" % a[14] - s << "%02x" % a[15] - s << " }" - end - end - EOF -def print_union(name, union) - print_union_with_namespace(:CUDA, name, union) -end - -def print_struct(name, struct) - prepends = [] - prepends << 'UUID' if to_class_name(name).match('UUID') - print_struct_with_namespace(:CUDA, name, struct, prepends: prepends) -end +print_handle_uuid_modules +puts puts < EOF -$cuda_commands.each do |c| +COMMANDS.groups[provider].each do |c| next if c.parameters && c.parameters.length > LTTNG_USABLE_PARAMS - $tracepoint_lambda.call(provider, c, :start) - $tracepoint_lambda.call(provider, c, :stop) + print_tracepoint(provider, c, :start) + print_tracepoint(provider, c, :stop) end diff --git a/backends/cuda/gen_cudart.rb b/backends/cuda/gen_cudart.rb index 12f010e06..355a62d90 100644 --- a/backends/cuda/gen_cudart.rb +++ b/backends/cuda/gen_cudart.rb @@ -9,11 +9,11 @@ #include "cudart_tracepoints.h" EOF -$cudart_commands.each do |c| +COMMANDS.each do |c| puts "#define #{CUDART_POINTER_NAMES[c]} #{c.pointer_name}" end -$cudart_commands.each do |c| +COMMANDS.each do |c| puts <<~EOF #{c.decl_pointer(c.pointer_type_name)}; @@ -26,7 +26,7 @@ static void find_cudart_symbols(void * handle, int verbose) { EOF -$cudart_commands.each do |c| +COMMANDS.each do |c| puts < EOF -$cudart_commands.each do |c| +COMMANDS.groups[provider].each do |c| next if c.parameters && c.parameters.length > LTTNG_USABLE_PARAMS - $tracepoint_lambda.call(provider, c, :start) - $tracepoint_lambda.call(provider, c, :stop) + print_tracepoint(provider, c, :start) + print_tracepoint(provider, c, :stop) end diff --git a/backends/hip/Makefile.am b/backends/hip/Makefile.am index 3084db704..385b5954d 100644 --- a/backends/hip/Makefile.am +++ b/backends/hip/Makefile.am @@ -208,10 +208,14 @@ HIP_MODEL = \ $(ML_HIP_FILES) \ $(srcdir)/hip_events.yaml \ $(HIP_EXTRACTED) \ + $(top_srcdir)/utils/api_model.rb \ $(top_srcdir)/utils/yaml_ast.rb \ $(top_srcdir)/utils/yaml_ast_lttng.rb \ $(top_srcdir)/utils/meta_parameters.rb \ $(top_srcdir)/utils/LTTng.rb \ + $(top_srcdir)/utils/command.rb \ + $(top_srcdir)/utils/command_index.rb \ + $(top_srcdir)/utils/meta_parameter_spec.rb \ hip.h.include btx_hip_model.yaml: $(srcdir)/gen_babeltrace_hip_model.rb $(HIP_LIB_GEN) $(HIP_MODEL) @@ -296,6 +300,8 @@ install-data-hook: HIP_LIB_GEN = \ $(top_srcdir)/utils/gen_babeltrace_model_helper.rb \ + $(top_srcdir)/utils/api_model.rb \ + $(top_srcdir)/utils/type_registry.rb \ $(top_srcdir)/utils/gen_library_base.rb \ $(srcdir)/gen_hip_library_base.rb \ $(top_srcdir)/utils/gen_probe_base.rb diff --git a/backends/hip/gen_babeltrace_hip_model.rb b/backends/hip/gen_babeltrace_hip_model.rb index a4576c1a2..8c484af78 100644 --- a/backends/hip/gen_babeltrace_hip_model.rb +++ b/backends/hip/gen_babeltrace_hip_model.rb @@ -1,19 +1,9 @@ require_relative 'gen_hip_library_base' require_relative '../../utils/gen_babeltrace_model_helper' -event_classes = - [[:lttng_ust_hip, $hip_commands]].collect do |provider, commands| - commands.collect do |c| - [gen_event_bt_model(provider, c, :start), - gen_event_bt_model(provider, c, :stop)] - end - end.flatten(2) +registry = build_ast_registry('hip', expect_bitfields: false) -hip_events = YAML.load_file(File.join(SRC_DIR, 'hip_events.yaml')) -event_classes += hip_events.collect do |provider, es| - es['events'].collect do |event| - gen_extra_event_bt_model(provider, event) - end -end.flatten +event_classes = gen_command_events_bt_model(registry, COMMANDS.groups) +event_classes += gen_extra_events_bt_model(registry, 'hip_events.yaml') puts YAML.dump(gen_yaml(event_classes, 'hip')) diff --git a/backends/hip/gen_hip.rb b/backends/hip/gen_hip.rb index de6338195..4660504cf 100644 --- a/backends/hip/gen_hip.rb +++ b/backends/hip/gen_hip.rb @@ -7,11 +7,11 @@ #include "hip_tracepoints.h" EOF -$hip_commands.each do |c| +COMMANDS.each do |c| puts "#define #{HIP_POINTER_NAMES[c]} #{c.pointer_name}" end -$hip_commands.each do |c| +COMMANDS.each do |c| puts <<~EOF #{c.decl_pointer(c.pointer_type_name)}; @@ -24,7 +24,7 @@ static void find_hip_symbols(void * handle, int verbose) { EOF -$hip_commands.each do |c| +COMMANDS.each do |c| puts < EOF -$hip_commands.each do |c| +COMMANDS.groups[provider].each do |c| next if c.parameters && c.parameters.length > LTTNG_USABLE_PARAMS - $tracepoint_lambda.call(provider, c, :start) - $tracepoint_lambda.call(provider, c, :stop) + print_tracepoint(provider, c, :start) + print_tracepoint(provider, c, :stop) end diff --git a/backends/hip/hip_model.rb b/backends/hip/hip_model.rb index 60c69b52a..42bf0afe7 100644 --- a/backends/hip/hip_model.rb +++ b/backends/hip/hip_model.rb @@ -1,5 +1,6 @@ require 'yaml' require 'pp' +require_relative '../../utils/api_model' require_relative '../../utils/yaml_ast_lttng' require_relative '../../utils/LTTng' require_relative '../../utils/command' @@ -7,38 +8,25 @@ SRC_DIR = ENV['SRC_DIR'] || '.' -RESULT_NAME = 'hipResult' +API = ApiModel.load_file('hip_api.yaml') -$hip_api_yaml = YAML.load_file('hip_api.yaml') -$hip_api = YAMLCAst.from_yaml_ast($hip_api_yaml) +gen_ffi_type_map(API.types, API.type_classes) -funcs = $hip_api['functions'] -typedefs = $hip_api['typedefs'] -structs = $hip_api['structs'] +CONTEXT = BackendContext.new( + result_name: 'hipResult', + init_functions: /.*/, + struct_map: API.struct_map, + type_classes: API.type_classes +) -find_all_types(typedefs) -OBJECT_TYPES.push('hipGraphicsResource_t') -POINTER_TYPES.delete('hipGraphicsResource_t') -gen_struct_map(typedefs, structs) -gen_ffi_type_map(typedefs) +meta_parameters = load_meta_parameters('hip_meta_parameters.yaml') -INIT_FUNCTIONS = /.*/ +COMMANDS = CommandIndex.new(lttng_ust_hip: API.functions.collect do |func| + Command.new(func, context: CONTEXT, meta_parameters: meta_parameters[func.name]) +end) -$hip_meta_parameters = YAML.load_file(File.join(SRC_DIR, 'hip_meta_parameters.yaml')) -$hip_meta_parameters['meta_parameters'].each do |func, list| - list.each do |type, *args| - register_meta_parameter func, Kernel.const_get(type), *args - end -end +check_meta_parameters(meta_parameters, COMMANDS) -$hip_commands = funcs.collect do |func| - Command.new(func) -end - -def upper_snake_case(str) - str.gsub(/([A-Z][A-Z0-9]*)/, '_\1').upcase -end - -HIP_POINTER_NAMES = $hip_commands.collect do |c| +HIP_POINTER_NAMES = COMMANDS.collect do |c| [c, upper_snake_case(c.pointer_name)] end.to_h diff --git a/backends/itt/Makefile.am b/backends/itt/Makefile.am index 90fadcbd3..cdf46d2fb 100644 --- a/backends/itt/Makefile.am +++ b/backends/itt/Makefile.am @@ -44,10 +44,14 @@ ITT_MODEL = \ $(srcdir)/itt_model.rb \ $(srcdir)/itt_meta_parameters.yaml \ $(top_srcdir)/utils/gen_probe_base.rb \ + $(top_srcdir)/utils/api_model.rb \ $(top_srcdir)/utils/yaml_ast.rb \ $(top_srcdir)/utils/yaml_ast_lttng.rb \ $(top_srcdir)/utils/meta_parameters.rb \ $(top_srcdir)/utils/LTTng.rb \ + $(top_srcdir)/utils/command.rb \ + $(top_srcdir)/utils/command_index.rb \ + $(top_srcdir)/utils/meta_parameter_spec.rb \ $(ITT_EXTRACTED) # Generate the Probe (tracepoint) @@ -129,6 +133,8 @@ bin_SCRIPTS = \ ITT_LIB_GEN = \ $(top_srcdir)/utils/gen_babeltrace_model_helper.rb \ + $(top_srcdir)/utils/api_model.rb \ + $(top_srcdir)/utils/type_registry.rb \ $(top_srcdir)/utils/gen_library_base.rb \ $(srcdir)/gen_itt_library_base.rb \ $(top_srcdir)/utils/gen_probe_base.rb diff --git a/backends/itt/gen_babeltrace_itt_model.rb b/backends/itt/gen_babeltrace_itt_model.rb index bc4922964..b132f320a 100644 --- a/backends/itt/gen_babeltrace_itt_model.rb +++ b/backends/itt/gen_babeltrace_itt_model.rb @@ -1,18 +1,9 @@ require_relative 'gen_itt_library_base' require_relative '../../utils/gen_babeltrace_model_helper' -event_classes = - [[:lttng_ust_itt, $itt_commands]].collect do |provider, commands| - commands.collect do |c| - [gen_event_bt_model(provider, c)] - end - end.flatten(2) +registry = build_ast_registry('itt', expect_bitfields: false) -itt_events = YAML.load_file(File.join(SRC_DIR, 'itt_events.yaml')) -event_classes += itt_events.collect do |provider, es| - es['events'].collect do |event| - gen_extra_event_bt_model(provider, event) - end -end.flatten +event_classes = gen_command_events_bt_model(registry, COMMANDS.groups, phased: false) +event_classes += gen_extra_events_bt_model(registry, 'itt_events.yaml') puts YAML.dump(gen_yaml(event_classes, 'itt')) diff --git a/backends/itt/gen_itt.rb b/backends/itt/gen_itt.rb index de130ff70..30d30188b 100644 --- a/backends/itt/gen_itt.rb +++ b/backends/itt/gen_itt.rb @@ -2,31 +2,30 @@ # Customization of codegen -$itt_commands.each do |c| +COMMANDS.each do |c| next unless c.has_return_type? - register_prologue(c.name, - if c.type.is_a?(YAMLCAst::Pointer) - "#{c.type} _retval = calloc(1, sizeof(*_retval));" - else - # `= {}` is C23, so use `= {0}` which also works for scalars - # pre-C23. Can be modernized once we require C23. - "#{c.type} _retval = {0};" - end) + c.add_prologue(if c.type.is_a?(YAMLCAst::Pointer) + "#{c.type} _retval = calloc(1, sizeof(*_retval));" + else + # `= {}` is C23, so use `= {0}` which also works for scalars + # pre-C23. Can be modernized once we require C23. + "#{c.type} _retval = {0};" + end) - register_epilogue(c.name, 'return _retval;') + c.add_epilogue('return _retval;') end # Sometime, but not always, those function are called by ittstatic # But we never use them in btx -register_prologue('__itt_event_create', '_retval = atomic_fetch_add(&event_counter, 1);') -register_prologue('__itt_domain_create', '_retval->flags = 1; _retval->nameA=name;') -register_prologue('__itt_string_handle_create', '_retval->strA=name;') -register_prologue('__itt_task_begin', 'if (domain->flags == 0) return;') -register_prologue('__itt_task_end', 'if (domain->flags == 0) return;') +COMMANDS.add_prologue('__itt_event_create', '_retval = atomic_fetch_add(&event_counter, 1);') +COMMANDS.add_prologue('__itt_domain_create', '_retval->flags = 1; _retval->nameA=name;') +COMMANDS.add_prologue('__itt_string_handle_create', '_retval->strA=name;') +COMMANDS.add_prologue('__itt_task_begin', 'if (domain->flags == 0) return;') +COMMANDS.add_prologue('__itt_task_end', 'if (domain->flags == 0) return;') -register_prologue('__itt_metadata_add', - 'tracepoint(lttng_ust_itt_metadata, metadata, type, count, count * __itt_metadata_type_size(type), data);') +COMMANDS.add_prologue('__itt_metadata_add', + 'tracepoint(lttng_ust_itt_metadata, metadata, type, count, count * __itt_metadata_type_size(type), data);') # Printing @@ -85,7 +84,7 @@ EOF provider = :lttng_ust_itt -puts $itt_commands.filter_map { |c| +puts COMMANDS.filter_map { |c| next if c.function.inline l = ["#{c.decl} {"] diff --git a/backends/itt/gen_itt_library.rb b/backends/itt/gen_itt_library.rb index 1e78ce2cd..0c990f941 100644 --- a/backends/itt/gen_itt_library.rb +++ b/backends/itt/gen_itt_library.rb @@ -1,55 +1,6 @@ require_relative 'gen_itt_library_base' require 'set' -def unwrap_typedef_to_concrete(t) - # unwrap chains like Typedef -> Typedef -> Enum - seen = Set.new.compare_by_identity - t = t.type while t.respond_to?(:type) && seen.add?(t) - t -end - -def enum_with_members_or_nil(e) - e if e && e.respond_to?(:members) && e.members && !e.members.empty? -end - -def find_enum_by_name(name, api) - base = name.to_s - base_stripped = base.sub(/_t\z/, '') - - enums = Array(api['enums']) - typedefs = Array(api['typedefs']) - - # direct match (array or hash) - if api['enums'].is_a?(Hash) - e = api['enums'][base] || api['enums'][base_stripped] - return e if enum_with_members_or_nil(e) - else - e = enums.find { |x| x.respond_to?(:name) && [base, base_stripped].include?(x.name) } - return e if enum_with_members_or_nil(e) - end - - # typedef: name or stripped name - td = typedefs.find { |t| t.respond_to?(:name) && [base, base_stripped].include?(t.name) } - if td - e = unwrap_typedef_to_concrete(td) - return e if enum_with_members_or_nil(e) - end - - # forward-declared enum object; look for another enum with same name but members set - if e && e.respond_to?(:name) - alt = enums.find { |x| x != e && x.respond_to?(:name) && x.name == e.name && enum_with_members_or_nil(x) } - return alt if alt - end - - # look for any anonymous enum that has members - if td - anon = unwrap_typedef_to_concrete(td) - return anon if enum_with_members_or_nil(anon) - end - - nil -end - def print_enum(name, enum) by_name = {} current = -1 @@ -98,10 +49,6 @@ def print_enum(name, enum) RUBY end -def print_itt_object(object) - print_object(object) -end - print_ffi_module(:ITT) puts <<~EOF @@ -110,24 +57,16 @@ module ITT EOF -def print_union(name, union) - print_union_with_namespace(:ITT, name, union) -end - -def print_struct(name, struct) - members = struct.to_ffi - +def print_struct(name, struct, fnptr_syms) # Replace function pointer field types with :pointer. # This avoids unresolved type errors when callbacks are defined later. - if defined?($fnptr_syms) && $fnptr_syms - members = members.map do |m| - t = m[1] - if t.is_a?(Array) - elem_t = t[0] - [m[0], [($fnptr_syms.include?(elem_t.to_s) ? :pointer : elem_t), t[1]]] - else - [m[0], ($fnptr_syms.include?(t.to_s) ? :pointer : t)] - end + members = struct.to_ffi.map do |m| + t = m[1] + if t.is_a?(Array) + elem_t = t[0] + [m[0], [(fnptr_syms.include?(elem_t.to_s) ? :pointer : elem_t), t[1]]] + else + [m[0], (fnptr_syms.include?(t.to_s) ? :pointer : t)] end end @@ -162,8 +101,8 @@ class #{to_class_name(name)} < FFI::ITTStruct # Build a set of all function pointer typedef symbols to detect struct fields # that should be converted to :pointer when generating layouts -$fnptr_syms = Set.new( - $all_types.select do |t| +fnptr_syms = Set.new( + API.types.select do |t| t.type.is_a?(YAMLCAst::Pointer) && t.type.type.is_a?(YAMLCAst::Function) end.map { |t| to_ffi_name(t.name).to_s } ) @@ -171,21 +110,17 @@ class #{to_class_name(name)} < FFI::ITTStruct # Collect callbacks to print after all other types callbacks = [] -$all_types.each do |t| +API.types.each do |t| if t.type.is_a? YAMLCAst::Enum - print_enum(t.name, find_enum_by_name(t.name, $itt_api)) - elsif $objects.include?(t.name) - print_itt_object(t.name) + print_enum(t.name, API.enum(t.type)) + elsif API.object?(t.name) + print_object(t.name) elsif t.type.is_a? YAMLCAst::Struct - struct = $all_structs.find { |s| t.type.name == s.name } - next unless struct - - print_struct(t.name, struct) + struct = API.struct(t.type) + print_struct(t.name, struct, fnptr_syms) elsif t.type.is_a? YAMLCAst::Union - union = $all_unions.find { |s| t.type.name == s.name } - next unless union - - print_union(t.name, union) + union = API.union(t.type) + print_union_with_namespace(:ITT, t.name, union) elsif t.type.is_a?(YAMLCAst::Pointer) && t.type.type.is_a?(YAMLCAst::Function) # Defer callbacks until the end so all referenced types are defined callbacks << [t.name, t.type.type] diff --git a/backends/itt/gen_itt_library_base.rb b/backends/itt/gen_itt_library_base.rb index 4457f0305..8bbe1ea1d 100644 --- a/backends/itt/gen_itt_library_base.rb +++ b/backends/itt/gen_itt_library_base.rb @@ -2,35 +2,6 @@ require_relative '../../utils/gen_probe_base' require_relative '../../utils/gen_library_base' -$all_types = $itt_api['typedefs'] -$all_structs = $itt_api['structs'] -$all_unions = $itt_api['unions'] -$all_enums = $itt_api['enums'] -$all_funcs = $itt_api['functions'] - -$all_enum_names = [] -$all_bitfield_names = [] -$all_struct_names = [] -$all_union_names = [] - -$objects = $all_types.select do |t| - t.type.is_a?(YAMLCAst::Pointer) && - t.type.type.is_a?(YAMLCAst::Struct) -end.collect { |t| t.name } - -$all_types.each do |t| - $objects.push t.name if t.type.is_a?(YAMLCAst::CustomType) && OBJECT_TYPES.include?(t.type.name) -end - -$int_scalars = {} -$all_types.each do |t| - $int_scalars[t.name] = t.type.name if t.type.is_a?(YAMLCAst::CustomType) && INT_TYPES.include?(t.type.name) -end - -def to_snake_case(str) - str.gsub(/([A-Z][A-Z0-9]*)/, '_\1').downcase -end - # Convert C / ITT names (e.g. "__itt_domain_t") to Ruby CamelCase class names def to_class_name(name) # Derive namespace (e.g. "ITT::" or "") @@ -64,28 +35,8 @@ def to_scoped_class_name(name) end def to_name_space(name) - name.match(/\A(__itt[dt]?)_/)[1].upcase -end - -$all_types.each do |t| - if t.type.is_a? YAMLCAst::Enum - enum = $all_enums.find { |e| t.type.name == e.name } - # Handle anonymous enum, and typedef enum - if enum&.name&.end_with?('flag_t') - $all_bitfield_names.push t.name - else - $all_enum_names.push t.name - end - elsif t.type.is_a? YAMLCAst::Struct - $all_struct_names.push t.name - elsif t.type.is_a? YAMLCAst::Union - $all_union_names.push t.name - end + match_name_space(name, /\A(__itt[dt]?)_/, strict: true).upcase end -$all_bitfield_names += $all_bitfield_names.select do |n| - n.end_with?('_flag_t') -end.map { |n| n.gsub('_flag_t', '_flags_t') } - FFI_STRUCT = 'FFI::ITTStruct' FFI_UNION = 'FFI::ITTUnion' diff --git a/backends/itt/gen_itt_tracepoints.rb b/backends/itt/gen_itt_tracepoints.rb index 1c1c35c90..65b965b0d 100644 --- a/backends/itt/gen_itt_tracepoints.rb +++ b/backends/itt/gen_itt_tracepoints.rb @@ -8,8 +8,8 @@ #include "ittnotify.h" EOF -$itt_commands.each do |c| +COMMANDS.groups[provider].each do |c| next if c.parameters && c.parameters.length > LTTNG_USABLE_PARAMS - $tracepoint_lambda.call(provider, c) + print_tracepoint(provider, c) end diff --git a/backends/itt/itt_model.rb b/backends/itt/itt_model.rb index 4f3c9dd41..ca9cb1fc4 100644 --- a/backends/itt/itt_model.rb +++ b/backends/itt/itt_model.rb @@ -1,5 +1,6 @@ require 'yaml' require 'pp' +require_relative '../../utils/api_model' require_relative '../../utils/yaml_ast_lttng' require_relative '../../utils/LTTng' require_relative '../../utils/command' @@ -7,26 +8,18 @@ SRC_DIR = ENV['SRC_DIR'] || '.' -RESULT_NAME = 'ittResult' +API = ApiModel.load_file('itt_api.yaml') -$itt_api_yaml = YAML.load_file('itt_api.yaml') -$itt_api = YAMLCAst.from_yaml_ast($itt_api_yaml) +gen_ffi_type_map(API.types, API.type_classes) -typedefs = $itt_api['typedefs'] -structs = $itt_api['structs'] +CONTEXT = BackendContext.new( + result_name: 'ittResult', + init_functions: /None/, + struct_map: API.struct_map, + type_classes: API.type_classes +) -find_all_types(typedefs) -gen_struct_map(typedefs, structs) -gen_ffi_type_map(typedefs) - -INIT_FUNCTIONS = /None/ - -$itt_meta_parameters = YAML.load_file(File.join(SRC_DIR, 'itt_meta_parameters.yaml')) -$itt_meta_parameters['meta_parameters'].each do |func, list| - list.each do |type, *args| - register_meta_parameter func, Kernel.const_get(type), *args - end -end +meta_parameters = load_meta_parameters('itt_meta_parameters.yaml') # Function we care whitelisted_functions = %w[ @@ -40,16 +33,10 @@ __itt_metadata_add ] -$itt_commands = $itt_api['functions'].filter_map do |func| +COMMANDS = CommandIndex.new(lttng_ust_itt: API.functions.filter_map do |func| next unless whitelisted_functions.include?(func.name) - Command.new(func) -end - -def upper_snake_case(str) - str.gsub(/([A-Z][A-Z0-9]*)/, '_\1').upcase -end + Command.new(func, context: CONTEXT, meta_parameters: meta_parameters[func.name]) +end) -ITT_POINTER_NAMES = $itt_commands.collect do |c| - [c, upper_snake_case(c.pointer_name)] -end.to_h +check_meta_parameters(meta_parameters, COMMANDS) diff --git a/backends/mpi/Makefile.am b/backends/mpi/Makefile.am index f3d0ea28c..80b78284f 100644 --- a/backends/mpi/Makefile.am +++ b/backends/mpi/Makefile.am @@ -41,10 +41,14 @@ MPI_MODEL = \ $(srcdir)/mpi_model.rb \ $(srcdir)/mpi_meta_parameters.yaml \ $(srcdir)/mpi_events.yaml \ + $(top_srcdir)/utils/api_model.rb \ $(top_srcdir)/utils/yaml_ast.rb \ $(top_srcdir)/utils/yaml_ast_lttng.rb \ $(top_srcdir)/utils/meta_parameters.rb \ $(top_srcdir)/utils/LTTng.rb \ + $(top_srcdir)/utils/command.rb \ + $(top_srcdir)/utils/command_index.rb \ + $(top_srcdir)/utils/meta_parameter_spec.rb \ $(MPI_EXTRACTED) btx_mpi_model.yaml: $(srcdir)/gen_babeltrace_mpi_model.rb $(MPI_LIB_GEN) $(MPI_MODEL) @@ -144,6 +148,8 @@ install-data-hook: MPI_LIB_GEN = \ $(top_srcdir)/utils/gen_babeltrace_model_helper.rb \ + $(top_srcdir)/utils/api_model.rb \ + $(top_srcdir)/utils/type_registry.rb \ $(top_srcdir)/utils/gen_library_base.rb \ $(srcdir)/gen_mpi_library_base.rb \ $(top_srcdir)/utils/gen_probe_base.rb diff --git a/backends/mpi/gen_babeltrace_mpi_model.rb b/backends/mpi/gen_babeltrace_mpi_model.rb index de6c42ba1..7615b1f95 100644 --- a/backends/mpi/gen_babeltrace_mpi_model.rb +++ b/backends/mpi/gen_babeltrace_mpi_model.rb @@ -1,19 +1,9 @@ require_relative 'gen_mpi_library_base' require_relative '../../utils/gen_babeltrace_model_helper' -event_classes = - [[:lttng_ust_mpi, $mpi_commands]].collect do |provider, commands| - commands.collect do |c| - [gen_event_bt_model(provider, c, :start), - gen_event_bt_model(provider, c, :stop)] - end - end.flatten(2) +registry = build_ast_registry('mpi', expect_bitfields: false) -mpi_events = YAML.load_file(File.join(SRC_DIR, 'mpi_events.yaml')) -event_classes += mpi_events.collect do |provider, es| - es['events'].collect do |event| - gen_extra_event_bt_model(provider, event) - end -end.flatten +event_classes = gen_command_events_bt_model(registry, COMMANDS.groups) +event_classes += gen_extra_events_bt_model(registry, 'mpi_events.yaml') puts YAML.dump(gen_yaml(event_classes, 'mpi')) diff --git a/backends/mpi/gen_mpi.rb b/backends/mpi/gen_mpi.rb index bd939a9bb..969ecea46 100644 --- a/backends/mpi/gen_mpi.rb +++ b/backends/mpi/gen_mpi.rb @@ -50,7 +50,7 @@ def normal_wrapper(c, provider) end def define_and_find_mpi_symbols - $mpi_commands.each do |c| + COMMANDS.each do |c| puts <<~EOF #define #{MPI_POINTER_NAMES[c]} #{c.pointer_name} #{c.decl_pointer(c.pointer_type_name)}; @@ -60,7 +60,7 @@ def define_and_find_mpi_symbols end puts 'static void find_mpi_symbols(void * handle, int verbose) {' - $mpi_commands.each do |c| + COMMANDS.each do |c| puts < EOF -$mpi_commands.each do |c| +COMMANDS.groups[provider].each do |c| next if c.parameters && c.parameters.length > LTTNG_USABLE_PARAMS - $tracepoint_lambda.call(provider, c, :start) - $tracepoint_lambda.call(provider, c, :stop) + print_tracepoint(provider, c, :start) + print_tracepoint(provider, c, :stop) end diff --git a/backends/mpi/mpi_meta_parameters.yaml b/backends/mpi/mpi_meta_parameters.yaml index b89f5db36..6687ca471 100644 --- a/backends/mpi/mpi_meta_parameters.yaml +++ b/backends/mpi/mpi_meta_parameters.yaml @@ -63,10 +63,6 @@ meta_parameters: MPI_Buffer_iflush: - - OutScalar - request - MPI_COMM_DUP_FN: [] - MPI_COMM_NULL_COPY_FN: [] - MPI_COMM_NULL_DELETE_FN: [] - MPI_CONVERSION_FN_NULL: [] MPI_Cancel: - - InScalar - request @@ -114,7 +110,6 @@ meta_parameters: - port_name - - OutScalar - newcomm - MPI_Comm_copy_attr_function: [] MPI_Comm_create: - - OutScalar - newcomm @@ -139,7 +134,6 @@ meta_parameters: - - OutScalar - comm_keyval MPI_Comm_delete_attr: [] - MPI_Comm_delete_attr_function: [] MPI_Comm_disconnect: - - InScalar - comm @@ -149,7 +143,6 @@ meta_parameters: MPI_Comm_dup_with_info: - - OutScalar - newcomm - MPI_Comm_errhandler_function: [] MPI_Comm_f2c: [] MPI_Comm_flush_buffer: [] MPI_Comm_free: @@ -230,11 +223,6 @@ meta_parameters: - - OutScalar - flag MPI_Compare_and_swap: [] - MPI_Copy_function: [] - MPI_DUP_FN: [] - MPI_Datarep_conversion_function: [] - MPI_Datarep_extent_function: [] - MPI_Delete_function: [] MPI_Dims_create: [] MPI_Dist_graph_create: - - InScalar @@ -267,7 +255,6 @@ meta_parameters: MPI_Exscan_init: - - OutScalar - request - MPI_F_sync_reg: [] MPI_Fetch_and_op: [] MPI_File_c2f: [] MPI_File_call_errhandler: [] @@ -282,7 +269,6 @@ meta_parameters: MPI_File_delete: - - InString - filename - MPI_File_errhandler_function: [] MPI_File_f2c: [] MPI_File_get_amode: - - OutScalar @@ -496,10 +482,7 @@ meta_parameters: - nnodes - - InScalar - nedges - MPI_Grequest_cancel_function: [] MPI_Grequest_complete: [] - MPI_Grequest_free_function: [] - MPI_Grequest_query_function: [] MPI_Grequest_start: - - InScalar - query_fn @@ -757,8 +740,6 @@ meta_parameters: - message - - OutScalar - status - MPI_NULL_COPY_FN: [] - MPI_NULL_DELETE_FN: [] MPI_Neighbor_allgather: [] MPI_Neighbor_allgather_init: - - OutScalar @@ -937,7 +918,6 @@ meta_parameters: - session_errhandler_fn - - OutScalar - errhandler - MPI_Session_errhandler_function: [] MPI_Session_f2c: [] MPI_Session_finalize: - - InScalar @@ -969,7 +949,6 @@ meta_parameters: - - InScalar - session MPI_Session_set_errhandler: [] - MPI_Sizeof: [] MPI_Ssend: [] MPI_Ssend_init: - - OutScalar @@ -1041,9 +1020,6 @@ meta_parameters: MPI_Status_set_elements_x: - - InScalar - status - MPI_TYPE_DUP_FN: [] - MPI_TYPE_NULL_COPY_FN: [] - MPI_TYPE_NULL_DELETE_FN: [] MPI_T_category_changed: - - OutScalar - update_number @@ -1290,7 +1266,6 @@ meta_parameters: MPI_Type_contiguous: - - OutScalar - newtype - MPI_Type_copy_attr_function: [] MPI_Type_create_darray: - - OutScalar - newtype @@ -1332,7 +1307,6 @@ meta_parameters: - - OutScalar - newtype MPI_Type_delete_attr: [] - MPI_Type_delete_attr_function: [] MPI_Type_dup: - - OutScalar - newtype @@ -1414,10 +1388,6 @@ meta_parameters: - service_name - - InString - port_name - MPI_User_function: [] - MPI_WIN_DUP_FN: [] - MPI_WIN_NULL_COPY_FN: [] - MPI_WIN_NULL_DELETE_FN: [] MPI_Wait: - - InScalar - request @@ -1440,7 +1410,6 @@ meta_parameters: MPI_Win_c2f: [] MPI_Win_call_errhandler: [] MPI_Win_complete: [] - MPI_Win_copy_attr_function: [] MPI_Win_create: - - OutScalar - win @@ -1460,9 +1429,7 @@ meta_parameters: - - OutScalar - win_keyval MPI_Win_delete_attr: [] - MPI_Win_delete_attr_function: [] MPI_Win_detach: [] - MPI_Win_errhandler_function: [] MPI_Win_f2c: [] MPI_Win_fence: [] MPI_Win_flush: [] diff --git a/backends/mpi/mpi_model.rb b/backends/mpi/mpi_model.rb index c8f3cc56c..54a6ba3cb 100644 --- a/backends/mpi/mpi_model.rb +++ b/backends/mpi/mpi_model.rb @@ -1,5 +1,6 @@ require 'yaml' require 'pp' +require_relative '../../utils/api_model' require_relative '../../utils/yaml_ast_lttng' require_relative '../../utils/LTTng' require_relative '../../utils/command' @@ -7,21 +8,11 @@ SRC_DIR = ENV['SRC_DIR'] || '.' -RESULT_NAME = 'mpiResult' +API = ApiModel.load_file('mpi_api.yaml') -$mpi_api_yaml = YAML.load_file('mpi_api.yaml') -$mpi_api = YAMLCAst.from_yaml_ast($mpi_api_yaml) +gen_ffi_type_map(API.types, API.type_classes) -typedefs = $mpi_api.fetch('typedefs', []) -structs = $mpi_api.fetch('structs', []) - -find_all_types(typedefs) -gen_struct_map(typedefs, structs) -gen_ffi_type_map(typedefs) - -mpi_funcs_e = $mpi_api['functions'] - -INIT_FUNCTIONS = / +init_functions = / \b(?:P?MPI_Init| P?MPI_Init_thread| P?MPI_Initialized| @@ -55,35 +46,29 @@ P?MPI_T_init_thread)\b /ix -$mpi_meta_parameters = YAML.load_file(File.join(SRC_DIR, 'mpi_meta_parameters.yaml')) -$mpi_meta_parameters.fetch('meta_parameters', []).each do |func, list| - list.each do |type, *args| - register_meta_parameter func, Kernel.const_get(type), *args - end -end +CONTEXT = BackendContext.new( + result_name: 'mpiResult', + init_functions: init_functions, + struct_map: API.struct_map, + type_classes: API.type_classes +) -$mpi_commands = mpi_funcs_e.collect do |func| - Command.new(func) -end +meta_parameters = load_meta_parameters('mpi_meta_parameters.yaml') -# https://api.rubyonrails.org/classes/ActiveSupport/Inflector.html#method-i-underscore -# As a rule of thumb you can think of underscore as the inverse of camelize, -def underscore(camel_cased_word) - return camel_cased_word.to_s.dup unless /[A-Z-]|::/.match?(camel_cased_word) +COMMANDS = CommandIndex.new(lttng_ust_mpi: API.functions.collect do |func| + Command.new(func, context: CONTEXT, meta_parameters: meta_parameters[func.name]) +end) - word = camel_cased_word.to_s.gsub('::', '/') - word.gsub!(/(?=a)b/) { "#{Regexp.last_match(1) && '_'}#{Regexp.last_match(2).downcase}" } - word.gsub!(/(?<=[A-Z])(?=[A-Z][a-z])|(?<=[a-z\d])(?=[A-Z])/, '_') - word.tr!('-', '_') - word.downcase! - word -end +check_meta_parameters(meta_parameters, COMMANDS) -MPI_POINTER_NAMES = $mpi_commands.collect do |c| - [c, underscore(c.pointer_name).upcase] +# MPI spells its functions MPI_Comm_rank, already snake_case, so the macro name +# is the function name upcased. The other backends snake-case a camelCase name +# with upper_snake_case; here that would give _MPI__COMM_RANK_PTR. +MPI_POINTER_NAMES = COMMANDS.collect do |c| + [c, c.pointer_name.upcase] end.to_h -register_epilogue 'MPI_Type_commit', < EOF -$ompt_commands.each do |c| +COMMANDS.groups[provider].each do |c| next if c.parameters && c.parameters.length > LTTNG_USABLE_PARAMS - $tracepoint_lambda.call(provider, c) + print_tracepoint(provider, c) end diff --git a/backends/omp/ompt_model.rb b/backends/omp/ompt_model.rb index e3e9ef240..723c1ed4b 100644 --- a/backends/omp/ompt_model.rb +++ b/backends/omp/ompt_model.rb @@ -1,5 +1,6 @@ require 'yaml' require 'pp' +require_relative '../../utils/api_model' require_relative '../../utils/yaml_ast_lttng' require_relative '../../utils/LTTng' require_relative '../../utils/command' @@ -7,19 +8,18 @@ SRC_DIR = ENV['SRC_DIR'] || '.' -RESULT_NAME = 'ompResult' +API = ApiModel.load_file('ompt_api.yaml') -$ompt_api_yaml = YAML.load_file('ompt_api.yaml') -$ompt_api = YAMLCAst.from_yaml_ast($ompt_api_yaml) +gen_ffi_type_map(API.types, API.type_classes) -typedefs = $ompt_api['typedefs'] -structs = $ompt_api['structs'] +CONTEXT = BackendContext.new( + result_name: 'ompResult', + init_functions: /None/, + struct_map: API.struct_map, + type_classes: API.type_classes +) -find_all_types(typedefs) -gen_struct_map(typedefs, structs) -gen_ffi_type_map(typedefs) - -OMPT_CALLBACKS = typedefs.select do |t| +OMPT_CALLBACKS = API.types.select do |t| t.type.is_a?(YAMLCAst::Pointer) && t.type.type.is_a?(YAMLCAst::Function) && t.name.match(/ompt_callback_.*_t/) end.reject do |t| %w[ompt_callback_buffer_complete_t ompt_callback_buffer_request_t].include?(t.name) @@ -27,23 +27,10 @@ YAMLCAst::Declaration.new(name: t.name.gsub(/_t\z/, '') + '_func', type: t.type.type) end -INIT_FUNCTIONS = /None/ - -$ompt_meta_parameters = YAML.load_file(File.join(SRC_DIR, 'ompt_meta_parameters.yaml')) -$ompt_meta_parameters['meta_parameters'].each do |func, list| - list.each do |type, *args| - register_meta_parameter func, Kernel.const_get(type), *args - end -end - -$ompt_commands = OMPT_CALLBACKS.collect do |func| - Command.new(func) -end +meta_parameters = load_meta_parameters('ompt_meta_parameters.yaml') -def upper_snake_case(str) - str.gsub(/([A-Z][A-Z0-9]*)/, '_\1').upcase -end +COMMANDS = CommandIndex.new(lttng_ust_ompt: OMPT_CALLBACKS.collect do |func| + Command.new(func, context: CONTEXT, meta_parameters: meta_parameters[func.name]) +end) -OMPT_POINTER_NAMES = $ompt_commands.collect do |c| - [c, upper_snake_case(c.pointer_name)] -end.to_h +check_meta_parameters(meta_parameters, COMMANDS) diff --git a/backends/opencl/Makefile.am b/backends/opencl/Makefile.am index fad629f50..10ca388df 100644 --- a/backends/opencl/Makefile.am +++ b/backends/opencl/Makefile.am @@ -39,6 +39,9 @@ TRACEPOINT_GEN = \ OPENCL_MODEL = \ $(TRACEPOINT_GEN) \ $(ML_FILES) \ + $(top_srcdir)/utils/LTTng.rb \ + $(top_srcdir)/utils/command_index.rb \ + $(top_srcdir)/utils/meta_parameter_spec.rb \ cl.xml.patched BTX_OPENCL_GENERATED = \ diff --git a/backends/opencl/gen_babeltrace_cl_model.rb b/backends/opencl/gen_babeltrace_cl_model.rb index f6aa1dc6b..66d6fddfb 100644 --- a/backends/opencl/gen_babeltrace_cl_model.rb +++ b/backends/opencl/gen_babeltrace_cl_model.rb @@ -1,10 +1,5 @@ require 'yaml' -INT_SIZE_MAP = {} -INT_SIGN_MAP = {} -$all_enums = {} -$all_types = {} - require_relative '../../utils/gen_babeltrace_model_helper' OPENCL_MODEL = YAML.load_file('opencl_model.yaml') @@ -45,6 +40,10 @@ def cl_to_class(type) 'CL::' + type.sub(/\Acl_/, '').split('_').collect(&:capitalize).join end +# Not the AST backends' gen_bt_field_model, because the two disagree on what a +# field is: opencl emits real CTF enumerations, which that function has no case +# for, and keeps pointer fields signed, while its `*`-in-the-type-string rule +# forces every pointer to unsigned 64. def parse_field(field) d = {} d[:field_class] = {} diff --git a/backends/opencl/gen_opencl.rb b/backends/opencl/gen_opencl.rb index b2f63027c..a5bbc79d5 100644 --- a/backends/opencl/gen_opencl.rb +++ b/backends/opencl/gen_opencl.rb @@ -41,11 +41,11 @@ tracepoint(provider, name, __VA_ARGS__) EOF -$opencl_commands.each do |c| +OPENCL_COMMANDS.groups[:core].each do |c| puts "#define #{OPENCL_POINTER_NAMES[c]} #{c.prototype.pointer_name}" end -$opencl_commands.each do |c| +OPENCL_COMMANDS.groups[:core].each do |c| puts <<~EOF typedef #{c.decl_pointer(type: true)}; @@ -53,7 +53,7 @@ EOF end -$opencl_extension_commands.each do |c| +OPENCL_COMMANDS.groups[:extension].each do |c| puts <<~EOF typedef #{c.decl_pointer(type: true)}; @@ -66,7 +66,7 @@ static void find_opencl_symbols(void * handle, int verbose) { EOF -$opencl_commands.each do |c| +OPENCL_COMMANDS.groups[:core].each do |c| next if c.extension? && c.prototype.name.match(/KHR$|EXT$/) puts < vals } r['trace_name'] = e['trace_name'] if e['trace_name'] @@ -66,7 +66,7 @@ c.parameters.select { |p| p.lttng_in_type }.each do |p| field = {} lttng = p.lttng_in_type - fname = LTTng.name(*lttng) + fname = LTTngFieldTuple.name(*lttng) field.merge!(params[fname]) field['lttng'] = lttng[0] fields[fname] = field @@ -74,21 +74,21 @@ c.meta_parameters.select { |p| p.lttng_in_type }.each do |p| meta_field = {} lttng = p.lttng_in_type - fname = LTTng.name(*lttng) + fname = LTTngFieldTuple.name(*lttng) if fname == 'errcode_ret_val' meta_field['type'] = 'cl_errcode' elsif fname.match(/_val\z/) pname = fname.gsub(/_val\z/, '') meta_field['type'] = params[pname]['type'] else - meta_field['type'] = params[LTTng.expression(*lttng)]['type'] + meta_field['type'] = params[LTTngFieldTuple.expression(*lttng)]['type'] end if meta_field['type'].match(/\*\z/) meta_field['type'] = meta_field['type'].sub(/\*\z/, '') meta_field['pointer'] = true end - meta_field['array'] = true if LTTng.array?(*lttng) - meta_field['string'] = true if LTTng.string?(*lttng) + meta_field['array'] = true if LTTngFieldTuple.array?(*lttng) + meta_field['string'] = true if LTTngFieldTuple.string?(*lttng) meta_field['lttng'] = lttng[0] meta_field['length'] = lttng[4] if meta_field['lttng'].match('ctf_array') if meta_field['array'] && @@ -105,14 +105,16 @@ field['type'] = c.prototype.return_type lttng = c.prototype.lttng_return_type field['lttng'] = lttng[0] - fname = LTTng.name(*lttng) + fname = LTTngFieldTuple.name(*lttng) field['type'] = 'cl_errcode' if fname == 'errcode_ret_val' fields[fname] = field end - c.meta_parameters.select { |p| p.lttng_out_type && LTTng.name(*p.lttng_out_type) != '_param_name' }.each do |p| + c.meta_parameters.select do |p| + p.lttng_out_type && LTTngFieldTuple.name(*p.lttng_out_type) != '_param_name' + end.each do |p| meta_field = {} lttng = p.lttng_out_type - fname = LTTng.name(*lttng) + fname = LTTngFieldTuple.name(*lttng) if fname == 'errcode_ret_val' meta_field['type'] = 'cl_errcode' elsif fname.match(/_val\z/) @@ -120,7 +122,7 @@ meta_field['type'] = params[pname]['type'] else begin - meta_field['type'] = params[LTTng.expression(*lttng)]['type'] + meta_field['type'] = params[LTTngFieldTuple.expression(*lttng)]['type'] rescue StandardError warn name, lttng.inspect end @@ -129,8 +131,8 @@ meta_field['type'] = meta_field['type'].gsub(/\*\z/, '') meta_field['pointer'] = true end - meta_field['array'] = true if LTTng.array?(*lttng) - meta_field['string'] = true if LTTng.string?(*lttng) + meta_field['array'] = true if LTTngFieldTuple.array?(*lttng) + meta_field['string'] = true if LTTngFieldTuple.string?(*lttng) meta_field['lttng'] = lttng[0] meta_field['length'] = lttng[4] if meta_field['lttng'].match('ctf_array') if meta_field['array'] && @@ -145,7 +147,7 @@ [name, fields] } -($opencl_commands + $opencl_extension_commands).each do |c| +(OPENCL_COMMANDS.groups[:core] + OPENCL_COMMANDS.groups[:extension]).each do |c| %w[start stop].each do |dir| name, val = event_lambda.call(c, dir) events[name] = val diff --git a/backends/opencl/gen_opencl_probes.rb b/backends/opencl/gen_opencl_probes.rb index 4fdd608ef..a6bd8caff 100644 --- a/backends/opencl/gen_opencl_probes.rb +++ b/backends/opencl/gen_opencl_probes.rb @@ -13,36 +13,6 @@ EOF -if GENERATE_ENUMS_TRACEPOINTS - puts <<~EOF - TRACEPOINT_ENUM( - lttng_ust_opencl, - cl_bool, - TP_ENUM_VALUES( - ctf_enum_value("CL_FALSE", 0) - ctf_enum_value("CL_TRUE", 1) - ) - ) - - EOF - - ENUMS.each do |name, e| - name = e['type_name'] if e['type_name'] - puts <<~EOF - TRACEPOINT_ENUM( - lttng_ust_opencl, - #{name}, - TP_ENUM_VALUES( - #{e['values'].collect do |k, v| # {' '} - "ctf_enum_value(\"#{k}\", #{"(#{name})" unless e['type_name']}#{v})" - end.join("\n ")} - ) - ) - - EOF - end -end - tracepoint_lambda = lambda { |c, dir| event = {} event['name'] = c.prototype.name @@ -82,17 +52,17 @@ end event[dir] = fields - print_tracepoint('lttng_ust_opencl', event, dir) + LTTng.print_tracepoint('lttng_ust_opencl', event, dir) } -$opencl_commands.each do |c| +OPENCL_COMMANDS.groups[:core].each do |c| next if c.parameters.length > LTTNG_USABLE_PARAMS tracepoint_lambda.call(c, 'start') tracepoint_lambda.call(c, 'stop') end -$opencl_extension_commands.each do |c| +OPENCL_COMMANDS.groups[:extension].each do |c| next if c.parameters.length > LTTNG_USABLE_PARAMS tracepoint_lambda.call(c, 'start') @@ -105,6 +75,6 @@ callbacks = YAML.load_file(File.join(SRC_DIR, 'opencl_wrapper_events.yaml'))[namespace] callbacks['events'].each do |e| %w[start stop].each do |dir| - print_tracepoint(namespace, e, dir) + LTTng.print_tracepoint(namespace, e, dir) end end diff --git a/backends/opencl/opencl_model.rb b/backends/opencl/opencl_model.rb index 0a6574f4b..2be53c79d 100644 --- a/backends/opencl/opencl_model.rb +++ b/backends/opencl/opencl_model.rb @@ -1,5 +1,8 @@ require 'nokogiri' require 'yaml' +require_relative '../../utils/LTTng' +require_relative '../../utils/command_index' +require_relative '../../utils/meta_parameter_spec' SRC_DIR = ENV['SRC_DIR'] || '.' @@ -7,10 +10,6 @@ STOP = 'exit' SUFFIXES = { 'start' => START, 'stop' => STOP } -MEMBER_SEPARATOR = '__' - -GENERATE_ENUMS_TRACEPOINTS = false - HOST_PROFILE = true WINDOWS = /D3D|DX9/ @@ -28,10 +27,6 @@ LTTNG_AVAILABLE_PARAMS = 25 LTTNG_USABLE_PARAMS = LTTNG_AVAILABLE_PARAMS - 1 -ENUMS = {} -ENUM_PARAM_NAME_MAP = {} -ENUM_TYPES = [] - # map = Hash::new { |h, k| h[k] = [] } doc = Nokogiri::XML(open('cl.xml.patched')) @@ -53,7 +48,7 @@ l['category'] == 'struct' end.collect -$constants = doc.xpath('//enums/enum').collect do |n| +CL_CONSTANTS = doc.xpath('//enums/enum').collect do |n| if n['value'] [n['name'], n['value']] elsif n['bitpos'] @@ -174,28 +169,10 @@ def enums end end -$requires = (doc.xpath('//feature/require').to_a + doc.xpath('//extensions/extension/require').to_a).collect do |r| +CL_REQUIRES = (doc.xpath('//feature/require').to_a + doc.xpath('//extensions/extension/require').to_a).collect do |r| Require.new(r) end -if GENERATE_ENUMS_TRACEPOINTS - enums = YAML.load_file(File.join(SRC_DIR, 'supported_enums.yaml')) - - enums.each do |e| - vals = $requires.select do |r| - r.comment && r.comment.match(/#{e['name']}(\z| )/) - end.collect do |r| - r.enums - end.reduce(:+).collect do |v| - [v, $constants[v]] - end.to_h - ENUMS[e['name']] = { 'values' => vals, 'trace_name' => e['trace_name'], 'type_name' => e['type_name'] } - ENUM_PARAM_NAME_MAP[e['trace_name']] = e['type_name'] - ENUM_TYPES.push(e['type_name'] || e['name']) - end - ENUM_TYPES.push 'cl_bool' -end - class Declaration < CLXML attr_reader :type, :name @@ -292,7 +269,6 @@ def lttng_in_type t = @type t = CL_TYPE_MAP[@type] if CL_TYPE_MAP[@type] - return ['ctf_enum', 'lttng_ust_opencl', @type, t, @name, @name] if ENUM_TYPES.include? @type case t when *CL_OBJECTS, *CL_EXT_OBJECTS @@ -375,8 +351,6 @@ def lttng_return_type case @return_type when 'cl_int' - return %w[ctf_enum lttng_ust_opencl cl_errcode cl_int errcode_ret_val _retval] if GENERATE_ENUMS_TRACEPOINTS - return %w[ctf_integer cl_int errcode_ret_val _retval] when *CL_OBJECTS @@ -447,24 +421,19 @@ def initialize(command, name, nocheck: false) type = command[name].type.gsub('*', '') type = CL_TYPE_MAP[type] if CL_TYPE_MAP[type] - if ENUM_PARAM_NAME_MAP[name] - @lttng_in_type = ['ctf_enum', 'lttng_ust_opencl', ENUM_PARAM_NAME_MAP[name], type, name + '_val', - nocheck ? "*#{name}" : "#{name} == NULL ? 0 : *#{name}"] + case type + when *CL_OBJECTS, *CL_EXT_OBJECTS, 'void' + @lttng_in_type = ['ctf_integer_hex', 'uintptr_t', name + '_val', + nocheck ? "(uintptr_t)(*#{name})" : "(uintptr_t)(#{name} == NULL ? 0 : *#{name})"] + when *CL_INT_SCALARS + @lttng_in_type = ['ctf_integer', type, name + '_val', nocheck ? "*#{name}" : "#{name} == NULL ? 0 : *#{name}"] + when *CL_FLOAT_SCALARS + @lttng_in_type = ['ctf_float', type, name + '_val', nocheck ? "*#{name}" : "#{name} == NULL ? 0 : *#{name}"] + when *CL_STRUCTS + @lttng_in_type = ['ctf_sequence_text', 'uint8_t', name + '_val', "(uint8_t *)#{name}", 'size_t', + "#{name} == NULL ? 0 : sizeof(#{type})"] else - case type - when *CL_OBJECTS, *CL_EXT_OBJECTS, 'void' - @lttng_in_type = ['ctf_integer_hex', 'uintptr_t', name + '_val', - nocheck ? "(uintptr_t)(*#{name})" : "(uintptr_t)(#{name} == NULL ? 0 : *#{name})"] - when *CL_INT_SCALARS - @lttng_in_type = ['ctf_integer', type, name + '_val', nocheck ? "*#{name}" : "#{name} == NULL ? 0 : *#{name}"] - when *CL_FLOAT_SCALARS - @lttng_in_type = ['ctf_float', type, name + '_val', nocheck ? "*#{name}" : "#{name} == NULL ? 0 : *#{name}"] - when *CL_STRUCTS - @lttng_in_type = ['ctf_sequence_text', 'uint8_t', name + '_val', "(uint8_t *)#{name}", 'size_t', - "#{name} == NULL ? 0 : sizeof(#{type})"] - else - raise "Unknown Type: #{type.inspect}!" - end + raise "Unknown Type: #{type.inspect}!" end end end @@ -476,21 +445,16 @@ def initialize(command, name, nocheck: false) type = command[name].type.gsub('*', '') type = CL_TYPE_MAP[type] if CL_TYPE_MAP[type] - if ENUM_PARAM_NAME_MAP[name] - @lttng_out_type = ['ctf_enum', 'lttng_ust_opencl', ENUM_PARAM_NAME_MAP[name], type, name + '_val', - nocheck ? "*#{name}" : "#{name} == NULL ? 0 : *#{name}"] + case type + when *CL_OBJECTS, *CL_EXT_OBJECTS, 'void' + @lttng_out_type = ['ctf_integer_hex', 'uintptr_t', name + '_val', + nocheck ? "(uintptr_t)(*#{name})" : "(uintptr_t)(#{name} == NULL ? 0 : *#{name})"] + when *CL_INT_SCALARS + @lttng_out_type = ['ctf_integer', type, name + '_val', nocheck ? "*#{name}" : "#{name} == NULL ? 0 : *#{name}"] + when *CL_FLOAT_SCALARS + @lttng_out_type = ['ctf_float', type, name + '_val', nocheck ? "*#{name}" : "#{name} == NULL ? 0 : *#{name}"] else - case type - when *CL_OBJECTS, *CL_EXT_OBJECTS, 'void' - @lttng_out_type = ['ctf_integer_hex', 'uintptr_t', name + '_val', - nocheck ? "(uintptr_t)(*#{name})" : "(uintptr_t)(#{name} == NULL ? 0 : *#{name})"] - when *CL_INT_SCALARS - @lttng_out_type = ['ctf_integer', type, name + '_val', nocheck ? "*#{name}" : "#{name} == NULL ? 0 : *#{name}"] - when *CL_FLOAT_SCALARS - @lttng_out_type = ['ctf_float', type, name + '_val', nocheck ? "*#{name}" : "#{name} == NULL ? 0 : *#{name}"] - else - raise "Unknown Type: #{type.inspect}!" - end + raise "Unknown Type: #{type.inspect}!" end end end @@ -643,51 +607,38 @@ def initialize(name, type, init) Event = AutoOutScalar.create('event') -def register_meta_parameter(method, type, *args) - unless OPENCL_COMMAND_NAMES.include?(method) || OPENCL_EXTENSION_COMMAND_NAMES.include?(method) - raise "Unknown method: #{method}!" - end - - META_PARAMETERS[method].push [type, args] -end - -def register_prologue(method, code) - unless OPENCL_COMMAND_NAMES.include?(method) || OPENCL_EXTENSION_COMMAND_NAMES.include?(method) - raise "Unknown method: #{method}!" - end - - PROLOGUES[method].push(code) -end - -def register_epilogue(method, code) - unless OPENCL_COMMAND_NAMES.include?(method) || OPENCL_EXTENSION_COMMAND_NAMES.include?(method) - raise "Unknown method: #{method}!" - end - - EPILOGUES[method].push(code) -end - AUTO_META_PARAMETERS = [EventWaitList, ErrCodeRet, ParamValueSizeRet, ParamValue, Event] -META_PARAMETERS = Hash.new { |h, k| h[k] = [] } -PROLOGUES = Hash.new { |h, k| h[k] = [] } -EPILOGUES = Hash.new { |h, k| h[k] = [] } class Command < CLXML attr_reader :prototype, :parameters, :tracepoint_parameters, :meta_parameters, :prologues, :epilogues - def initialize(command) - super + # `meta_parameters` is this function's rows from a meta-parameter spec, as + # returned by load_meta_parameters: a list of [MetaParameter subclass, args]. + def initialize(command, meta_parameters: []) + super(command) @prototype = Prototype.new(command.search('proto')) @parameters = command.search('param').collect { |p| Parameter.new(p) } @tracepoint_parameters = [] @meta_parameters = AUTO_META_PARAMETERS.collect { |klass| klass.create_if_match(self) }.compact - @meta_parameters += META_PARAMETERS[@prototype.name].collect do |type, args| + @meta_parameters += meta_parameters.collect do |type, args| type.new(self, *args) end @extension = @prototype.name.match(EXTENSION_FUNCTIONS) @init = @prototype.name.match(INIT_FUNCTIONS) - @prologues = PROLOGUES[@prototype.name] - @epilogues = EPILOGUES[@prototype.name] + @prologues = [] + @epilogues = [] + end + + def name + @prototype.name + end + + def add_prologue(code) + @prologues.push(code) + end + + def add_epilogue(code) + @epilogues.push(code) end def [](name) @@ -730,76 +681,60 @@ def void_parameters? end end -OPENCL_COMMAND_NAMES = funcs_e.collect { |c| Prototype.new(c.search('proto')) }.collect { |p| p.name } -OPENCL_EXTENSION_COMMAND_NAMES = ext_funcs_e.collect { |c| Prototype.new(c.search('proto')) }.collect { |p| p.name } +meta_parameters = load_meta_parameters('opencl_meta_parameters.yaml') -$meta_parameters = YAML.load_file(File.join(SRC_DIR, 'opencl_meta_parameters.yaml')) -$meta_parameters['meta_parameters'].each do |func, list| - list.each do |type, *args| - register_meta_parameter func, Kernel.const_get(type), *args +# Both groups go to the one lttng_ust_opencl provider, so they are grouped by +# what actually separates them: an extension is reached through +# clGetExtensionFunctionAddress rather than dlsym, which several of the +# generators below need to tell apart. +OPENCL_COMMANDS = CommandIndex.new( + { core: funcs_e, extension: ext_funcs_e }.transform_values do |funcs| + funcs.collect { |func| Command.new(func, meta_parameters: meta_parameters[func.search('proto/name').text]) } end -end - -$opencl_commands = funcs_e.collect do |func| - Command.new(func) -end - -$opencl_extension_commands = ext_funcs_e.collect do |func| - Command.new(func) -end +) -$opencl_commands.each do |c| - eval "$#{c.prototype.name} = c" -end - -$opencl_extension_commands.each do |c| - eval "$#{c.prototype.name} = c" -end - -def upper_snake_case(str) - str.gsub(/([A-Z][A-Z0-9]*)/, '_\1').upcase -end +check_meta_parameters(meta_parameters, OPENCL_COMMANDS) -OPENCL_POINTER_NAMES = ($opencl_commands.collect do |c| +OPENCL_POINTER_NAMES = (OPENCL_COMMANDS.groups[:core].collect do |c| [c, upper_snake_case(c.prototype.pointer_name)] -end + $opencl_extension_commands.collect do |c| +end + OPENCL_COMMANDS.groups[:extension].collect do |c| [c, c.prototype.pointer_name] end).to_h -($opencl_commands + $opencl_extension_commands).select do |c| +OPENCL_COMMANDS.select do |c| c.parameters.find { |p| p.name == 'errcode_ret' && p.pointer? } end.each do |c| - c.prologues.push <= dump_start && _enqueue_counter <= dump_end) { cl_command_queue_properties properties; - #{OPENCL_POINTER_NAMES[$clGetCommandQueueInfo]}(command_queue, CL_QUEUE_PROPERTIES, sizeof(cl_command_queue_properties), &properties, NULL); + #{OPENCL_POINTER_NAMES[OPENCL_COMMANDS['clGetCommandQueueInfo']]}(command_queue, CL_QUEUE_PROPERTIES, sizeof(cl_command_queue_properties), &properties, NULL); _dump_release_events = dump_kernel_args(command_queue, kernel, _enqueue_counter, properties, &num_events_in_wait_list, (cl_event **)&event_wait_list); if ((properties & CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE) && event == NULL) { event = &extra_event; @@ -872,40 +807,40 @@ def lttng_out_type } } EOF -register_prologue 'clEnqueueNDRangeKernel', str -register_prologue 'clEnqueueNDRangeKernelINTEL', str +OPENCL_COMMANDS.add_prologue 'clEnqueueNDRangeKernel', str +OPENCL_COMMANDS.add_prologue 'clEnqueueNDRangeKernelINTEL', str str = < 0) { struct opencl_version version = {1, 0}; @@ -1055,12 +990,12 @@ def lttng_out_type if (_free_options) free((char *)options); EOF -register_epilogue 'clBuildProgram', str -register_epilogue 'clCompileProgram', str -register_epilogue 'clLinkProgram', str +OPENCL_COMMANDS.add_epilogue 'clBuildProgram', str +OPENCL_COMMANDS.add_epilogue 'clCompileProgram', str +OPENCL_COMMANDS.add_epilogue 'clLinkProgram', str l = lambda { |func, name: 'pfn_notify', extra_conditions: nil| - register_prologue func, < $@ @@ -211,6 +215,8 @@ install-data-hook: ZE_LIB_GEN = \ $(top_srcdir)/utils/gen_babeltrace_model_helper.rb \ + $(top_srcdir)/utils/api_model.rb \ + $(top_srcdir)/utils/type_registry.rb \ $(top_srcdir)/utils/gen_library_base.rb \ $(srcdir)/gen_ze_library_base.rb \ $(top_srcdir)/utils/gen_probe_base.rb diff --git a/backends/ze/gen_babeltrace_ze_model.rb b/backends/ze/gen_babeltrace_ze_model.rb index 58393acbe..b6fccb462 100644 --- a/backends/ze/gen_babeltrace_ze_model.rb +++ b/backends/ze/gen_babeltrace_ze_model.rb @@ -1,36 +1,10 @@ require_relative 'gen_ze_library_base' require_relative '../../utils/gen_babeltrace_model_helper' -require 'set' -event_classes = - [[:lttng_ust_ze, $ze_commands], - [:lttng_ust_zet, $zet_commands], - [:lttng_ust_zes, $zes_commands], - [:lttng_ust_zel, $zel_commands], - [:lttng_ust_zer, $zer_commands], - [:lttng_ust_zex, $zex_commands]].collect do |provider, commands| - commands.collect do |c| - [gen_event_bt_model(provider, c, :start), - gen_event_bt_model(provider, c, :stop)] - end - end.flatten(2) +registry = build_ast_registry('ze', expect_bitfields: true) -ze_events = YAML.load_file(File.join(SRC_DIR, 'ze_events.yaml')) -event_classes += ze_events.collect do |provider, es| - es['events'].collect do |event| - gen_extra_event_bt_model(provider, event) - end -end.flatten - -def get_structs_types(namespace, types, structs) - types.select do |t| - t.type.is_a?(YAMLCAst::Struct) && (struct = structs.find do |s| - t.type.name == s.name - end) && struct.members.first.name == 'stype' - end.map(&:name).reject do |n| - n.start_with?("#{namespace}_base_") - end.to_set -end +event_classes = gen_command_events_bt_model(registry, COMMANDS.groups) +event_classes += gen_extra_events_bt_model(registry, 'ze_events.yaml') def gen_struct_event_bt_model(provider, struct) { @@ -72,17 +46,10 @@ def gen_struct_event_bt_model(provider, struct) } end -event_classes += - [[:lttng_ust_ze_structs, get_structs_types(:ze, $ze_api['typedefs'], $ze_api['structs'])], - [:lttng_ust_zet_structs, get_structs_types(:zet, $zet_api['typedefs'], $zet_api['structs'])], - [:lttng_ust_zes_structs, get_structs_types(:zes, $zes_api['typedefs'], $zes_api['structs'])], - [:lttng_ust_zel_structs, get_structs_types(:zel, $zel_api['typedefs'], $zel_api['structs'])], - [:lttng_ust_zer_structs, get_structs_types(:zer, $zer_api['typedefs'], $zer_api['structs'])], - [:lttng_ust_zex_structs, - get_structs_types(:zex, $zex_api['typedefs'], $zex_api['structs'])]].collect do |provider, structs| - structs.collect do |struct| - gen_struct_event_bt_model(provider, struct) - end - end.flatten +event_classes += APIS.collect do |ns, api| + concrete_stype_structs(ns, api).collect do |struct| + gen_struct_event_bt_model(:"lttng_ust_#{ns}_structs", struct) + end +end.flatten puts YAML.dump(gen_yaml(event_classes, 'ze')) diff --git a/backends/ze/gen_ze.rb b/backends/ze/gen_ze.rb index cff13fdeb..66a18505e 100644 --- a/backends/ze/gen_ze.rb +++ b/backends/ze/gen_ze.rb @@ -40,16 +40,6 @@ EOF -def get_structs_types(namespace, types, structs) - types.select do |t| - t.type.is_a?(YAMLCAst::Struct) && (struct = structs.find do |s| - t.type.name == s.name - end) && struct.members.first.name == 'stype' - end.reject do |t| - t.name.start_with?("#{namespace}_base_") - end.map(&:name).to_set -end - def gen_struct_printer(namespace, types) puts <<~EOF static @@ -57,9 +47,9 @@ def gen_struct_printer(namespace, types) #{namespace}_structure_type_t stype = (#{namespace}_structure_type_t)((ze_base_desc_t *)p)->stype; switch (stype) { EOF - types.reject { |t| $struct_type_reject.include?(t.to_s) }.each do |t| + types.reject { |t| STRUCT_TYPE_REJECT.include?(t.to_s) }.each do |t| ename = "#{namespace.to_s.upcase}_STRUCTURE_TYPE_#{t.delete_prefix(namespace.to_s + '_').delete_suffix('_t').upcase}" - ename = $struct_type_conversion_table[ename] if $struct_type_conversion_table[ename] + ename = STRUCT_TYPE_CONVERSION_TABLE[ename] if STRUCT_TYPE_CONVERSION_TABLE[ename] puts <_structure_type_t, which zer and zex do not +# declare: zer has no api.yaml at all, and zex names no structure types. -all_commands = $ze_commands + $zet_commands + $zes_commands + $zel_commands + $zer_commands +# zex is excluded: it is reached through libffi closures, not dlsym'd symbols. +zex_commands = COMMANDS.groups[:lttng_ust_zex] +all_commands = COMMANDS.to_a - zex_commands all_commands.each do |c| puts "#define #{ZE_POINTER_NAMES[c]} #{c.pointer_name}" end @@ -124,7 +111,7 @@ def gen_struct_printer(namespace, types) EOF end -$zex_commands.each do |c| +zex_commands.each do |c| puts <<~EOF #{c.decl_pointer(c.pointer_type_name)}; @@ -250,56 +237,33 @@ def gen_struct_printer(namespace, types) EOF } -$ze_commands.each do |c| - next if c.name.match(/zeGet.*ProcAddrTable|^zeLoaderInit|^zeLoaderGetTracingHandle/) - - puts <<~EOF - #{c.decl_hidden_alias}; - - EOF -end -$zet_commands.each do |c| - puts <<~EOF unless c.name.match(/zetGet.*ProcAddrTable/) - #{c.decl_hidden_alias}; - - EOF -end -$zes_commands.each do |c| - puts <<~EOF unless c.name.match(/zesGet.*ProcAddrTable/) - #{c.decl_hidden_alias}; - - EOF -end -$zel_commands.each do |c| - puts <<~EOF if c.name.match(/^zelTracer/) && !c.name.match(/RegisterCallback$|ResetAllCallbacks$/) - #{c.decl_hidden_alias}; +# Which of a namespace's entry points get a hidden alias. zel is the exception: +# only its tracer API is aliased, so it opts in rather than out. +aliased = { + ze: ->(n) { !n.match(/zeGet.*ProcAddrTable|^zeLoaderInit|^zeLoaderGetTracingHandle/) }, + zet: ->(n) { !n.match(/zetGet.*ProcAddrTable/) }, + zes: ->(n) { !n.match(/zesGet.*ProcAddrTable/) }, + zel: ->(n) { n.match(/^zelTracer/) && !n.match(/RegisterCallback$|ResetAllCallbacks$/) }, + zer: ->(n) { !n.match(/zerGet.*ProcAddrTable/) }, +} - EOF -end -$zer_commands.each do |c| - puts <<~EOF unless c.name.match(/zerGet.*ProcAddrTable/) - #{c.decl_hidden_alias}; +aliased.each do |ns, alias_wanted| + COMMANDS.groups[:"lttng_ust_#{ns}"].each do |c| + puts <<~EOF if alias_wanted.call(c.name) + #{c.decl_hidden_alias}; - EOF + EOF + end end -$ze_commands.each do |c| - normal_wrapper.call(c, :lttng_ust_ze, ze_struct_types) -end -$zet_commands.each do |c| - normal_wrapper.call(c, :lttng_ust_zet, zet_struct_types) -end -$zes_commands.each do |c| - normal_wrapper.call(c, :lttng_ust_zes, zes_struct_types) -end -$zel_commands.each do |c| - normal_wrapper.call(c, :lttng_ust_zel, zel_struct_types) -end -$zer_commands.each do |c| - normal_wrapper.call(c, :lttng_ust_zer, zer_struct_types) +%i[ze zet zes zel zer].each do |ns| + provider = :"lttng_ust_#{ns}" + COMMANDS.groups[provider].each do |c| + normal_wrapper.call(c, provider, struct_types[ns]) + end end -$zex_commands.each do |c| +zex_commands.each do |c| puts <<~EOF static #{c.decl_ffi_wrapper} { (void)cif; @@ -309,7 +273,7 @@ def gen_struct_printer(namespace, types) #{p} = *(#{p.type} *)args[#{i}]; EOF end - common_block.call(c, :lttng_ust_zex, zex_struct_types) + common_block.call(c, :lttng_ust_zex, struct_types[:zex]) if c.has_return_type? puts < LTTNG_USABLE_PARAMS - $tracepoint_lambda.call(provider, c, :start) - $tracepoint_lambda.call(provider, c, :stop) + print_tracepoint(provider, c, :start) + print_tracepoint(provider, c, :stop) end diff --git a/backends/ze/gen_zel_structs_tracepoints.rb b/backends/ze/gen_zel_structs_tracepoints.rb index 5a7d07b50..c6298c092 100644 --- a/backends/ze/gen_zel_structs_tracepoints.rb +++ b/backends/ze/gen_zel_structs_tracepoints.rb @@ -8,10 +8,6 @@ #include "ze.h.include" EOF -$zel_api['typedefs'].select do |t| - t.type.is_a?(YAMLCAst::Struct) && (struct = $zel_api['structs'].find do |s| - t.type.name == s.name - end) && struct.members.first.name == 'stype' -end.each do |t| - $struct_tracepoint_lambda.call(provider, t.name) +stype_structs(APIS[:zel]).each do |t| + print_struct_tracepoint(provider, t) end diff --git a/backends/ze/gen_zel_tracepoints.rb b/backends/ze/gen_zel_tracepoints.rb index 093593e02..f47e90fe8 100644 --- a/backends/ze/gen_zel_tracepoints.rb +++ b/backends/ze/gen_zel_tracepoints.rb @@ -8,9 +8,9 @@ #include "ze.h.include" EOF -$zel_commands.each do |c| +COMMANDS.groups[provider].each do |c| next if c.parameters && c.parameters.length > LTTNG_USABLE_PARAMS - $tracepoint_lambda.call(provider, c, :start) - $tracepoint_lambda.call(provider, c, :stop) + print_tracepoint(provider, c, :start) + print_tracepoint(provider, c, :stop) end diff --git a/backends/ze/gen_zer_structs_tracepoints.rb b/backends/ze/gen_zer_structs_tracepoints.rb index a27481a9e..402ddcab8 100644 --- a/backends/ze/gen_zer_structs_tracepoints.rb +++ b/backends/ze/gen_zer_structs_tracepoints.rb @@ -8,10 +8,6 @@ #include "ze.h.include" EOF -$zer_api['typedefs'].select do |t| - t.type.is_a?(YAMLCAst::Struct) && (struct = $zel_api['structs'].find do |s| - t.type.name == s.name - end) && struct.members.first.name == 'stype' -end.each do |t| - $struct_tracepoint_lambda.call(provider, t.name) +stype_structs(APIS[:zer]).each do |t| + print_struct_tracepoint(provider, t) end diff --git a/backends/ze/gen_zer_tracepoints.rb b/backends/ze/gen_zer_tracepoints.rb index 347a4a346..178d66761 100644 --- a/backends/ze/gen_zer_tracepoints.rb +++ b/backends/ze/gen_zer_tracepoints.rb @@ -8,9 +8,9 @@ #include "ze.h.include" EOF -$zer_commands.each do |c| +COMMANDS.groups[provider].each do |c| next if c.parameters && c.parameters.length > LTTNG_USABLE_PARAMS - $tracepoint_lambda.call(provider, c, :start) - $tracepoint_lambda.call(provider, c, :stop) + print_tracepoint(provider, c, :start) + print_tracepoint(provider, c, :stop) end diff --git a/backends/ze/gen_zes_structs_tracepoints.rb b/backends/ze/gen_zes_structs_tracepoints.rb index 39b4ecc7d..fbc8a3580 100644 --- a/backends/ze/gen_zes_structs_tracepoints.rb +++ b/backends/ze/gen_zes_structs_tracepoints.rb @@ -8,10 +8,6 @@ #include "ze.h.include" EOF -$zes_api['typedefs'].select do |t| - t.type.is_a?(YAMLCAst::Struct) && (struct = $zes_api['structs'].find do |s| - t.type.name == s.name - end) && struct.members.first.name == 'stype' -end.each do |t| - $struct_tracepoint_lambda.call(provider, t.name) +stype_structs(APIS[:zes]).each do |t| + print_struct_tracepoint(provider, t) end diff --git a/backends/ze/gen_zes_tracepoints.rb b/backends/ze/gen_zes_tracepoints.rb index 02696db4d..58e440527 100644 --- a/backends/ze/gen_zes_tracepoints.rb +++ b/backends/ze/gen_zes_tracepoints.rb @@ -8,9 +8,9 @@ #include "ze.h.include" EOF -$zes_commands.each do |c| +COMMANDS.groups[provider].each do |c| next if c.parameters && c.parameters.length > LTTNG_USABLE_PARAMS - $tracepoint_lambda.call(provider, c, :start) - $tracepoint_lambda.call(provider, c, :stop) + print_tracepoint(provider, c, :start) + print_tracepoint(provider, c, :stop) end diff --git a/backends/ze/gen_zet_structs_tracepoints.rb b/backends/ze/gen_zet_structs_tracepoints.rb index 22ccfd9a5..cb44f2e1f 100644 --- a/backends/ze/gen_zet_structs_tracepoints.rb +++ b/backends/ze/gen_zet_structs_tracepoints.rb @@ -8,10 +8,6 @@ #include "ze.h.include" EOF -$zet_api['typedefs'].select do |t| - t.type.is_a?(YAMLCAst::Struct) && (struct = $zet_api['structs'].find do |s| - t.type.name == s.name - end) && struct.members.first.name == 'stype' -end.each do |t| - $struct_tracepoint_lambda.call(provider, t.name) +stype_structs(APIS[:zet]).each do |t| + print_struct_tracepoint(provider, t) end diff --git a/backends/ze/gen_zet_tracepoints.rb b/backends/ze/gen_zet_tracepoints.rb index 727e4d017..d8795a4c7 100644 --- a/backends/ze/gen_zet_tracepoints.rb +++ b/backends/ze/gen_zet_tracepoints.rb @@ -8,9 +8,9 @@ #include "ze.h.include" EOF -$zet_commands.each do |c| +COMMANDS.groups[provider].each do |c| next if c.parameters && c.parameters.length > LTTNG_USABLE_PARAMS - $tracepoint_lambda.call(provider, c, :start) - $tracepoint_lambda.call(provider, c, :stop) + print_tracepoint(provider, c, :start) + print_tracepoint(provider, c, :stop) end diff --git a/backends/ze/gen_zex_structs_tracepoints.rb b/backends/ze/gen_zex_structs_tracepoints.rb index cf946fa72..f8997d3d1 100644 --- a/backends/ze/gen_zex_structs_tracepoints.rb +++ b/backends/ze/gen_zex_structs_tracepoints.rb @@ -8,10 +8,6 @@ #include "ze.h.include" EOF -$zex_api['typedefs'].select do |t| - t.type.is_a?(YAMLCAst::Struct) && (struct = $zex_api['structs'].find do |s| - t.type.name == s.name - end) && struct.members.first.name == 'stype' -end.each do |t| - $struct_tracepoint_lambda.call(provider, t.name) +stype_structs(APIS[:zex]).each do |t| + print_struct_tracepoint(provider, t) end diff --git a/backends/ze/gen_zex_tracepoints.rb b/backends/ze/gen_zex_tracepoints.rb index ff244f63f..c087e5acb 100644 --- a/backends/ze/gen_zex_tracepoints.rb +++ b/backends/ze/gen_zex_tracepoints.rb @@ -8,9 +8,9 @@ #include "ze.h.include" EOF -$zex_commands.each do |c| +COMMANDS.groups[provider].each do |c| next if c.parameters && c.parameters.length > LTTNG_USABLE_PARAMS - $tracepoint_lambda.call(provider, c, :start) - $tracepoint_lambda.call(provider, c, :stop) + print_tracepoint(provider, c, :start) + print_tracepoint(provider, c, :stop) end diff --git a/backends/ze/ze_model.rb b/backends/ze/ze_model.rb index b9fd4b8e7..8a799f323 100644 --- a/backends/ze/ze_model.rb +++ b/backends/ze/ze_model.rb @@ -1,5 +1,6 @@ require 'yaml' require 'pp' +require_relative '../../utils/api_model' require_relative '../../utils/yaml_ast_lttng' require_relative '../../utils/LTTng' require_relative '../../utils/command' @@ -8,44 +9,55 @@ SRC_DIR = ENV['SRC_DIR'] || '.' -RESULT_NAME = 'zeResult' - -$ze_api_yaml = YAML.load_file('ze_api.yaml') -$zet_api_yaml = YAML.load_file('zet_api.yaml') -$zes_api_yaml = YAML.load_file('zes_api.yaml') -$zel_api_yaml = YAML.load_file('zel_api.yaml') -# $zer_api_yaml = YAML.load_file('zer_api.yaml') -$zer_api_yaml = { 'typedefs' => [], 'structs' => [], 'functions' => [] } -$zex_api_yaml = YAML.load_file('zex_api.yaml') - -$ze_api = YAMLCAst.from_yaml_ast($ze_api_yaml) -$zet_api = YAMLCAst.from_yaml_ast($zet_api_yaml) -$zes_api = YAMLCAst.from_yaml_ast($zes_api_yaml) -$zel_api = YAMLCAst.from_yaml_ast($zel_api_yaml) -$zer_api = YAMLCAst.from_yaml_ast($zer_api_yaml) -$zex_api = YAMLCAst.from_yaml_ast($zex_api_yaml) - -ze_funcs_e = $ze_api['functions'] -zet_funcs_e = $zet_api['functions'] -zes_funcs_e = $zes_api['functions'] -zel_funcs_e = $zel_api['functions'] -zer_funcs_e = $zer_api['functions'] -zex_funcs_e = $zex_api['functions'] - -typedefs = $ze_api['typedefs'] + $zet_api['typedefs'] + $zes_api['typedefs'] + $zel_api['typedefs'] + $zer_api['typedefs'] + $zex_api['typedefs'] -structs = $ze_api['structs'] + $zet_api['structs'] + $zes_api['structs'] + $zel_api['structs'] + $zer_api['structs'] + $zex_api['structs'] - -find_all_types(typedefs) -gen_struct_map(typedefs, structs) -gen_ffi_type_map(typedefs) - -# zesInit is included here so that a pure-Sysman program (one that only calls -# zesInit, never zeInit) still triggers the tracer initialization. -# Ideally we would split this into INIT_ZE_FUNCTIONS / INIT_ZES_FUNCTIONS -# so each namespace initializes its own symbols. -INIT_FUNCTIONS = /zeInit|zeLoaderInit|zeInitDrivers|zesInit/ - -$struct_type_conversion_table = { +# The namespaces THAPI traces, each declared by its own header. zer has no +# generated api.yaml yet, so it is an empty model rather than a missing key: +# every namespace answers, and the loops below stay uniform. +APIS = { + ze: ApiModel.load_file('ze_api.yaml'), + zet: ApiModel.load_file('zet_api.yaml'), + zes: ApiModel.load_file('zes_api.yaml'), + zel: ApiModel.load_file('zel_api.yaml'), + zer: ApiModel.new, + zex: ApiModel.load_file('zex_api.yaml'), +}.freeze + +# The extensible structs of one namespace. Level Zero starts each of them with +# an `stype` tag naming its own type, which is how the tracer knows what a +# `void *` points at. +def stype_structs(api) + api.types.select do |t| + t.type.is_a?(YAMLCAst::Struct) && + (struct = api.structs.find { |s| t.type.name == s.name }) && + struct.members.first.name == 'stype' + end.map(&:name) +end + +# Those a caller can be handed. The `_base_` types are the tag's own base +# classes: a tracepoint exists for each, but no API call ever passes one, so +# nothing dispatches on them. +def concrete_stype_structs(namespace, api) + stype_structs(api).reject { |n| n.start_with?("#{namespace}_base_") }.to_set +end + +# Every namespace as one API, the same thing `API` names in every other +# backend. The derivations have to see all of them at once: a zet typedef +# routinely names a ze struct. +API = APIS.values.inject(:+) + +gen_ffi_type_map(API.types, API.type_classes) + +CONTEXT = BackendContext.new( + result_name: 'zeResult', + # zesInit is included here so that a pure-Sysman program (one that only calls + # zesInit, never zeInit) still triggers the tracer initialization. + # Ideally we would split this into INIT_ZE_FUNCTIONS / INIT_ZES_FUNCTIONS + # so each namespace initializes its own symbols. + init_functions: /zeInit|zeLoaderInit|zeInitDrivers|zesInit/, + struct_map: API.struct_map, + type_classes: API.type_classes +) + +STRUCT_TYPE_CONVERSION_TABLE = { 'ZE_STRUCTURE_TYPE_IMAGE_MEMORY_PROPERTIES_EXP' => 'ZE_STRUCTURE_TYPE_IMAGE_MEMORY_EXP_PROPERTIES', 'ZE_STRUCTURE_TYPE_IMAGE_PITCHED_EXP_DESC' => 'ZE_STRUCTURE_TYPE_PITCHED_IMAGE_EXP_DESC', 'ZE_STRUCTURE_TYPE_IMAGE_BINDLESS_EXP_DESC' => 'ZE_STRUCTURE_TYPE_BINDLESS_IMAGE_EXP_DESC', @@ -62,86 +74,34 @@ 'ZES_STRUCTURE_TYPE_MEM_PAGE_OFFLINE_STATE_EXP' => 'ZES_STRUCTURE_TYPE_MEMORY_PAGE_OFFLINE_STATE_EXP', } -$struct_type_reject = Set.new(['zet_metric_source_id_exp_t']) +STRUCT_TYPE_REJECT = Set.new(['zet_metric_source_id_exp_t']) -$ze_meta_parameters = YAML.load_file(File.join(SRC_DIR, 'ze_meta_parameters.yaml')) -$ze_meta_parameters['meta_parameters'].each do |func, list| - list.each do |type, *args| - register_meta_parameter func, Kernel.const_get(type), *args - end -end -$zet_meta_parameters = YAML.load_file(File.join(SRC_DIR, 'zet_meta_parameters.yaml')) -$zet_meta_parameters['meta_parameters'].each do |func, list| - list.each do |type, *args| - register_meta_parameter func, Kernel.const_get(type), *args - end -end +# Each namespace declares its meta-parameters in its own file, so the list +# follows APIS rather than restating it. zer's file exists but stays out until +# zer has a generated api.yaml to match it against. +meta_parameters = load_meta_parameters(*(APIS.keys - [:zer]).collect { |ns| "#{ns}_meta_parameters.yaml" }) -$zes_meta_parameters = YAML.load_file(File.join(SRC_DIR, 'zes_meta_parameters.yaml')) -$zes_meta_parameters['meta_parameters'].each do |func, list| - list.each do |type, *args| - register_meta_parameter func, Kernel.const_get(type), *args - end -end +# One group per namespace, because each namespace has its own LTTng provider. +COMMANDS = CommandIndex.new(APIS.to_h do |ns, api| + [:"lttng_ust_#{ns}", api.functions.collect do |func| + Command.new(func, context: CONTEXT, meta_parameters: meta_parameters[func.name]) + end] +end) -$zel_meta_parameters = YAML.load_file(File.join(SRC_DIR, 'zel_meta_parameters.yaml')) -$zel_meta_parameters['meta_parameters'].each do |func, list| - list.each do |type, *args| - register_meta_parameter func, Kernel.const_get(type), *args - end -end - -# $zer_meta_parameters = YAML.load_file(File.join(SRC_DIR, 'zer_meta_parameters.yaml')) -# $zer_meta_parameters['meta_parameters'].each do |func, list| -# list.each do |type, *args| -# register_meta_parameter func, Kernel.const_get(type), *args -# end -# end - -$zex_meta_parameters = YAML.load_file(File.join(SRC_DIR, 'zex_meta_parameters.yaml')) -$zex_meta_parameters['meta_parameters'].each do |func, list| - list.each do |type, *args| - register_meta_parameter func, Kernel.const_get(type), *args - end -end - -$ze_commands = ze_funcs_e.collect do |func| - Command.new(func) -end - -$zet_commands = zet_funcs_e.collect do |func| - Command.new(func) -end - -$zes_commands = zes_funcs_e.collect do |func| - Command.new(func) -end - -$zel_commands = zel_funcs_e.collect do |func| - Command.new(func) -end - -$zer_commands = zer_funcs_e.collect do |func| - Command.new(func) -end - -$zex_commands = zex_funcs_e.collect do |func| - Command.new(func) -end - -def upper_snake_case(str) - str.gsub(/([A-Z][A-Z0-9]*)/, '_\1').upcase -end +check_meta_parameters(meta_parameters, COMMANDS) -ze_pointer_names = ($ze_commands + $zet_commands + $zes_commands + $zel_commands + $zer_commands).collect do |c| +# zex is called through libffi rather than dlsym, so its pointer keeps the name +# from the header instead of the upper-snake macro the other namespaces get. +zex_commands = COMMANDS.groups[:lttng_ust_zex] +ze_pointer_names = (COMMANDS.to_a - zex_commands).collect do |c| [c, upper_snake_case(c.pointer_name)] end -ze_pointer_names += $zex_commands.collect do |c| +ze_pointer_names += zex_commands.collect do |c| [c, c.pointer_name] end ZE_POINTER_NAMES = ze_pointer_names.to_h -register_epilogue 'zeCommandListCreate', <flags & ZE_COMMAND_LIST_FLAG_IN_ORDER) != 0; @@ -150,7 +110,7 @@ def upper_snake_case(str) } EOF -register_epilogue 'zeCommandListCreateImmediate', <flags & ZE_COMMAND_QUEUE_FLAG_IN_ORDER) != 0; @@ -167,7 +127,7 @@ def upper_snake_case(str) # Reset wipes that body, so we reclaim the slots/slabs/events now. Without it # the stale slots are re-published on the next Execute (over-count) and slabs # leak. The cl stays registered, empty for reuse. -register_epilogue 'zeCommandListReset', < 0 && phCommandLists) _on_execute_command_lists_prologue(numCommandLists, phCommandLists, hCommandQueue, hFence); EOF # Sync hooks: walk dependency edges from the synced anchor and drain # everything reachable. Each sync API has a different anchor. -register_epilogue 'zeCommandQueueSynchronize', <') if path.length == 1 @@ -81,56 +113,10 @@ def [](name) return nil unless res path.each do |n| - res = STRUCT_MAP[res.type.type.name].find { |m| m.name == n } + res = @context.struct_map[res.type.type.name].find { |m| m.name == n } return nil unless res end res end end end - -class Member - def initialize(_command, member, prefix, dir = :start) - @member = member - @dir = dir - @prefix = prefix - name = "#{prefix}#{MEMBER_SEPARATOR}#{member.name}" - expr = "#{prefix} ? #{prefix}->#{member.name} : 0" - @lttng_type = member.type.lttng_type - @lttng_type.name = name - @lttng_type.expr = expr - end - - def lttng_in_type - @dir == :start ? @lttng_type : nil - end - - def lttng_out_type - @dir == :start ? nil : @lttng_type - end -end - -def register_meta_parameter(method, type, *args) - META_PARAMETERS[method].push [type, args] -end - -def register_meta_struct(method, name, type) - raise "Unknown struct: #{type}!" unless STRUCT_TYPES.include?(type) - - STRUCT_MAP[type].each do |m| - META_PARAMETERS[method].push [Member, [m, name]] - end -end - -def register_prologue(method, code) - PROLOGUES[method].push(code) -end - -def register_epilogue(method, code) - EPILOGUES[method].push(code) -end - -AUTO_META_PARAMETERS = [] -META_PARAMETERS = Hash.new { |h, k| h[k] = [] } -PROLOGUES = Hash.new { |h, k| h[k] = [] } -EPILOGUES = Hash.new { |h, k| h[k] = [] } diff --git a/utils/command_index.rb b/utils/command_index.rb new file mode 100644 index 000000000..d9580deff --- /dev/null +++ b/utils/command_index.rb @@ -0,0 +1,43 @@ +# A backend's commands, indexed by name and kept in the groups they were built +# in -- one per LTTng provider for most backends, core versus extension for +# opencl -- because generators want both shapes. +# +# Looking up a name that matches no command raises: it is always a typo or a +# renamed function, which would otherwise attach its code to nothing at all. +# +# This lives apart from utils/command.rb because opencl builds its commands +# from the Khronos XML with a Command class of its own, and so cannot require +# that file, but indexes them just the same. +class CommandIndex + include Enumerable + + attr_reader :groups + + def initialize(groups) + @groups = groups.freeze + @by_name = {} + @groups.each_value do |commands| + commands.each do |c| + raise "#{c.name} appears in two command lists" if @by_name.key?(c.name) + + @by_name[c.name] = c + end + end + end + + def each(&block) + @by_name.each_value(&block) + end + + def [](name) + @by_name.fetch(name) { raise "Unknown method: #{name}!" } + end + + def add_prologue(name, code) + self[name].add_prologue(code) + end + + def add_epilogue(name, code) + self[name].add_epilogue(code) + end +end diff --git a/utils/gen_babeltrace_lib_helper.rb b/utils/gen_babeltrace_lib_helper.rb index c058ea3d2..a147ca380 100644 --- a/utils/gen_babeltrace_lib_helper.rb +++ b/utils/gen_babeltrace_lib_helper.rb @@ -7,51 +7,52 @@ def add_babeltrace_event_callbacks(file) fields = m.reject { |f| /^_.*_length$/ =~ f[:name] } .map do |f| - name = f[:name] - fc = f[:field_class] - be_class = f[:metadata]&.[](:be_class) + name = f[:name] + fc = f[:field_class] + be_class = f[:metadata]&.[](:be_class) - default_command = %(s << "#{name}: \#{defi["#{name}"]}") + default_command = %(s << "#{name}: \#{defi["#{name}"]}") - case fc[:type] - when 'integer_signed', 'integer_unsigned' - if be_class - if $all_bitfield_names.include?(fc[:cast_type]) - %{s << "#{name}: [ \#{#{be_class}.from_native(defi["#{name}"], nil).join(", ")} ]"} - else - %{s << "#{name}: \#{#{be_class}.from_native(defi["#{name}"], nil)}"} - end - elsif fc[:preferred_display_base] == 16 - %(s << "#{name}: \#{"0x%016x" % defi["#{name}"]}") - else - default_command - end - when 'double', 'single' - default_command - when 'string' - if be_class - if name.end_with?('_vals') - %{s << "#{name}: [ \#{p = FFI::MemoryPointer.from_string(defi["#{name}"]); sz = #{be_class}.size; n = p.size / sz; (0...n).collect { |i| #{be_class}.new(p + i*sz).to_s}.join(", ")} ]"} - else - %{s << "#{name}: \#{defi["#{name}"].size > 0 ? #{be_class}.new(FFI::MemoryPointer.from_string(defi["#{name}"])) : nil}"} - end - else - %(s << "#{name}: \#{defi["#{name}"].inspect}") - end - when 'array_dynamic', 'array_static' - case fc[:element_field_class][:type] - when 'integer_signed', 'integer_unsigned' - if fc[:element_field_class][:preferred_display_base] == 16 - %{s << "#{name}: [ \#{defi["#{name}"].collect { |v| "0x%016x" % v }.join(", ")} ]"} - else - default_command - end - else - raise "Unsupported field type for array: #{fc[:element_field_class][:type]}" - end - else - raise "Unsupported field type: #{fc[:type]}" - end + case fc[:type] + when 'integer_signed', 'integer_unsigned' + if be_class + # API is in scope: callers require their gen__library_base first. + if API.bitfield_names.include?(fc[:cast_type]) + %{s << "#{name}: [ \#{#{be_class}.from_native(defi["#{name}"], nil).join(", ")} ]"} + else + %{s << "#{name}: \#{#{be_class}.from_native(defi["#{name}"], nil)}"} + end + elsif fc[:preferred_display_base] == 16 + %(s << "#{name}: \#{"0x%016x" % defi["#{name}"]}") + else + default_command + end + when 'double', 'single' + default_command + when 'string' + if be_class + if name.end_with?('_vals') + %{s << "#{name}: [ \#{p = FFI::MemoryPointer.from_string(defi["#{name}"]); sz = #{be_class}.size; n = p.size / sz; (0...n).collect { |i| #{be_class}.new(p + i*sz).to_s}.join(", ")} ]"} + else + %{s << "#{name}: \#{defi["#{name}"].size > 0 ? #{be_class}.new(FFI::MemoryPointer.from_string(defi["#{name}"])) : nil}"} + end + else + %(s << "#{name}: \#{defi["#{name}"].inspect}") + end + when 'array_dynamic', 'array_static' + case fc[:element_field_class][:type] + when 'integer_signed', 'integer_unsigned' + if fc[:element_field_class][:preferred_display_base] == 16 + %{s << "#{name}: [ \#{defi["#{name}"].collect { |v| "0x%016x" % v }.join(", ")} ]"} + else + default_command + end + else + raise "Unsupported field type for array: #{fc[:element_field_class][:type]}" + end + else + raise "Unsupported field type: #{fc[:type]}" + end end.join("\n s << ', '\n ") # Now just print the full strings to pretty printf the struct diff --git a/utils/gen_babeltrace_model_helper.rb b/utils/gen_babeltrace_model_helper.rb index 7dda338ff..591c7f257 100644 --- a/utils/gen_babeltrace_model_helper.rb +++ b/utils/gen_babeltrace_model_helper.rb @@ -1,38 +1,24 @@ -# Include global variable INT_SIGN_MAP, ScalarMetaParameter, etc require_relative 'yaml_ast' - -$integer_sizes = INT_SIZE_MAP.transform_values { |v| v * 8 } -$integer_signed = INT_SIGN_MAP - -$all_enums.each do |t| - $integer_sizes["enum #{t.name}"] = 32 -end - -$all_enums.each do |t| - $integer_signed["enum #{t.name}"] = true -end - -def integer_size(t) - return 64 if t.match(/\*/) - return 64 if t.match(/\[.*\]/) - - r = $integer_sizes[t] - raise "unknown integer type #{t}" if r.nil? - - r -end - -def integer_signed?(t) - return false if t.match(/\*/) - return false if t.match(/\[.*\]/) - - r = $integer_signed[t] - raise "unknown integer type #{t}" if r.nil? - - r +require_relative 'type_registry' + +# Build the babeltrace TypeRegistry from the backend's API model. Whether an +# API has bitfield types is a fact about its headers, so each backend states +# what it expects: a backend that silently stopped classifying them would +# otherwise emit plain integers where enum metadata belongs. +def build_ast_registry(backend, expect_bitfields:) + registry = TypeRegistry.new( + all_types: API.types, all_enums: API.enums, + enum_names: API.enum_names, bitfield_names: API.bitfield_names, struct_names: API.struct_names, + class_namer: method(:to_scoped_class_name) + ) + if expect_bitfields + raise "#{backend}: expected bitfield types" if registry.bitfield_names.empty? + else + raise "#{backend}: expected no bitfield types" unless registry.bitfield_names.empty? + end + registry end -# End of global variable use def meta_parameter_types_name(m, dir = nil) lttng = if dir == :start m.lttng_in_type @@ -86,13 +72,12 @@ def get_extra_fields_types_name(event) end.flatten(1) end -$types_by_name = $all_types.map { |ty| [ty.name, ty] }.to_h - -def gen_bt_field_model(lttng_name, type, name, lttng) +def gen_bt_field_model(registry, lttng_name, type, name, lttng) + types_by_name = registry.types_by_name member = { name: name } field = { cast_type: type.gsub(/\[.*\]/, '*') } - if $types_by_name[type].is_a?(YAMLCAst::Declaration) && $types_by_name[type].type.is_a?(YAMLCAst::Function) + if types_by_name[type].is_a?(YAMLCAst::Declaration) && types_by_name[type].type.is_a?(YAMLCAst::Function) field[:cast_type] = "#{type} *" end @@ -101,18 +86,18 @@ def gen_bt_field_model(lttng_name, type, name, lttng) when 'ctf_float' field[:type] = type == 'float' ? 'single' : type when 'ctf_integer', 'ctf_integer_hex' - field[:type] = integer_signed?(type) ? 'integer_signed' : 'integer_unsigned' - field[:field_value_range] = integer_size(type) + field[:type] = registry.integer_signed?(type) ? 'integer_signed' : 'integer_unsigned' + field[:field_value_range] = registry.integer_size(type) field[:preferred_display_base] = 16 if lttng_name.end_with?('_hex') - if $all_enum_names.include?(type) || $all_bitfield_names.include?(type) - member[:metadata] = { be_class: to_scoped_class_name(type) } + if registry.enum_names.include?(type) || registry.bitfield_names.include?(type) + member[:metadata] = { be_class: registry.class_namer.call(type) } end when 'ctf_sequence', 'ctf_sequence_hex' array_type = lttng.type.to_s field[:type] = 'array_dynamic' field[:element_field_class] = - { type: integer_signed?(array_type) ? 'integer_signed' : 'integer_unsigned', - field_value_range: integer_size(array_type) } + { type: registry.integer_signed?(array_type) ? 'integer_signed' : 'integer_unsigned', + field_value_range: registry.integer_size(array_type) } field[:element_field_class][:preferred_display_base] = 16 if lttng_name.end_with?('_hex') @@ -124,8 +109,8 @@ def gen_bt_field_model(lttng_name, type, name, lttng) array_type = lttng.type.to_s field[:type] = 'array_static' field[:element_field_class] = - { type: integer_signed?(array_type) ? 'integer_signed' : 'integer_unsigned', - field_value_range: integer_size(array_type) } + { type: registry.integer_signed?(array_type) ? 'integer_signed' : 'integer_unsigned', + field_value_range: registry.integer_size(array_type) } field[:element_field_class][:preferred_display_base] = 16 if lttng_name.end_with?('_hex') field[:length] = lttng.length when 'ctf_string' @@ -133,13 +118,11 @@ def gen_bt_field_model(lttng_name, type, name, lttng) when 'ctf_sequence_text', 'ctf_array_text' field[:type] = 'string' t = type.sub(' *', '') - while $types_by_name.include?(t) && $types_by_name[t].type.is_a?(YAMLCAst::CustomType) - t = $types_by_name[t].type.name - end - member[:metadata] = { be_class: to_scoped_class_name(t) } if $all_struct_names.include?(t) + t = types_by_name[t].type.name while types_by_name.include?(t) && types_by_name[t].type.is_a?(YAMLCAst::CustomType) + member[:metadata] = { be_class: registry.class_namer.call(t) } if registry.struct_names.include?(t) # Too complicated, not sure why `all_struct_names` is not enough - if !field[:cast_type].end_with?('*') && ($all_struct_names.include?(t) || $types_by_name[t]&.type.is_a?(YAMLCAst::Union) || type.start_with?('struct')) + if !field[:cast_type].end_with?('*') && (registry.struct_names.include?(t) || types_by_name[t]&.type.is_a?(YAMLCAst::Union) || type.start_with?('struct')) field[:cast_type_is_struct] = true end else @@ -152,12 +135,13 @@ def gen_bt_field_model(lttng_name, type, name, lttng) def get_fields_types_name(c, dir) fields = [] - r = c.type.lttng_type - fields.push([r.macro.to_s, c.type.to_s, "#{RESULT_NAME}", r]) if dir != :start && r + r = c.type.lttng_type(c.type_classes) + fields.push([r.macro.to_s, c.type.to_s, c.result_name, r]) if dir != :start && r if dir != :stop fields += c.parameters.to_a.collect do |p| - [p.lttng_type.macro.to_s, p.type.to_s, p.name.to_s, p.lttng_type] + lttng = p.lttng_type(c.type_classes) + [lttng.macro.to_s, p.type.to_s, p.name.to_s, lttng] end end @@ -171,21 +155,21 @@ def get_fields_types_name(c, dir) end.flatten(1) end -def gen_event_fields_bt_model(c, dir) +def gen_event_fields_bt_model(registry, c, dir) types_name = get_fields_types_name(c, dir) types_name.collect do |lttng_name, type, name, lttng| - gen_bt_field_model(lttng_name, type.sub(/\Aconst /, ''), name, lttng) + gen_bt_field_model(registry, lttng_name, type.sub(/\Aconst /, ''), name, lttng) end end -def gen_extra_event_fields_bt_model(event) +def gen_extra_event_fields_bt_model(registry, event) types_name = get_extra_fields_types_name(event) types_name.collect do |lttng_name, type, name, lttng| - gen_bt_field_model(lttng_name, type.sub(/\Aconst /, ''), name, lttng) + gen_bt_field_model(registry, lttng_name, type.sub(/\Aconst /, ''), name, lttng) end end -def gen_event_bt_model(provider, c, dir = nil) +def gen_event_bt_model(registry, provider, c, dir = nil) d = if dir { name: "#{provider}:#{c.name}_#{SUFFIXES[dir]}" } # OMP backend @@ -193,7 +177,7 @@ def gen_event_bt_model(provider, c, dir = nil) { name: "#{provider}:#{c.name.gsub(/_func\z/, '')}" } end - m = gen_event_fields_bt_model(c, dir) + m = gen_event_fields_bt_model(registry, c, dir) unless m.empty? d[:payload_field_class] = @@ -205,9 +189,9 @@ def gen_event_bt_model(provider, c, dir = nil) d end -def gen_extra_event_bt_model(provider, event) +def gen_extra_event_bt_model(registry, provider, event) d = { name: "#{provider}:#{event['name']}" } - m = gen_extra_event_fields_bt_model(event) + m = gen_extra_event_fields_bt_model(registry, event) unless m.empty? d[:payload_field_class] = @@ -219,6 +203,28 @@ def gen_extra_event_bt_model(provider, event) d end +# itt and omp trace a single event per command; everyone else a start/stop pair. +def gen_command_events_bt_model(registry, provider_commands, phased: true) + provider_commands.collect do |provider, commands| + commands.collect do |c| + if phased + [gen_event_bt_model(registry, provider, c, :start), + gen_event_bt_model(registry, provider, c, :stop)] + else + [gen_event_bt_model(registry, provider, c)] + end + end + end.flatten(2) +end + +def gen_extra_events_bt_model(registry, filename) + YAML.load_file(File.join(SRC_DIR, filename)).collect do |provider, es| + es['events'].collect do |event| + gen_extra_event_bt_model(registry, provider, event) + end + end.flatten +end + def gen_yaml(event_classes, backend) { environment: { entries: [ diff --git a/utils/gen_library_base.rb b/utils/gen_library_base.rb index 7c5b26b6b..e7042d7c2 100644 --- a/utils/gen_library_base.rb +++ b/utils/gen_library_base.rb @@ -1,7 +1,37 @@ -require_relative 'yaml_ast' +require_relative 'api_model' + +# Class name for an API that already spells its types in the target case, so +# the only work is the namespace prefix. hip and mpi each write theirs two ways +# -- hipDeviceProp_t and HIP_ARRAY_DESCRIPTOR -- and only the lowercase +# spelling is title-cased, leaving HipDeviceProp_t and HIP_ARRAY_DESCRIPTOR. +# The rest of the name is left exactly as the header spells it. +# +# The backends whose headers are camelCase (cuda, ze, omp, itt) do more than +# this -- they split on '_' and recase every word -- so they keep their own. +def prefixed_class_name(name, namespace) + namespace = namespace.to_s + rest = name.sub(/\A#{namespace}/, '') + prefix = namespace.match?(/[[:lower:]]/) ? namespace.capitalize : namespace + res = prefix + rest + res[0] = res[0].upcase if res[0]&.match(/[[:lower:]]/) + res +end -def has_typedef?(name) - $all_types.any? { |t| t.type.respond_to?(:name) && t.type.name == name } +# The namespace prefix `name` starts with, as the pattern captured it, or nil. +# +# Whether nil is reachable is a property of the API, not a style choice, so it +# is opted into rather than left to whichever expression happened to be +# written. ze/omp/itt/mpi headers declare only their own types, so every name +# matches: they pass `strict: true`, and a future unprefixed type raises here +# by name instead of surfacing as a NoMethodError on nil deep inside a +# generator. cuda and hip vendor foreign types -- GLuint, dim3, VdpDevice, the +# OpenCL interop typedefs -- that belong to no namespace, so nil is a real +# answer their callers already handle. +def match_name_space(name, pattern, strict: false) + m = name.match(pattern) + raise "#{name} does not start with a known namespace" if m.nil? && strict + + m && m[1] end def to_ffi_name(name, default = true) @@ -126,6 +156,48 @@ def print_object(object) EOF end +# Shared by cuda/hip/mpi. ze inlines its own -- :data/:id fields, and a UUID +# printed back to front. +def print_handle_uuid_modules + puts <<'EOF' + module Handle + def to_s + s = '{ reserved: "' + s << self[:reserved].to_a.collect { |v| "\\x%02x" % ((v + 256)%256) }.join + s << '" }' + end + end + + module UUID + def to_s + a = self[:bytes].to_a.collect { |v| v < 0 ? 0x100 + v : v } + s = "{ id: " + s << "%02x" % a[0] + s << "%02x" % a[1] + s << "%02x" % a[2] + s << "%02x" % a[3] + s << "-" + s << "%02x" % a[4] + s << "%02x" % a[5] + s << "-" + s << "%02x" % a[6] + s << "%02x" % a[7] + s << "-" + s << "%02x" % a[8] + s << "%02x" % a[9] + s << "-" + s << "%02x" % a[10] + s << "%02x" % a[11] + s << "%02x" % a[12] + s << "%02x" % a[13] + s << "%02x" % a[14] + s << "%02x" % a[15] + s << " }" + end + end +EOF +end + def print_ffi_module(namespace, struct: true, union: true, enum: true, bitmask: true, inline_array: true, enclosing_module: true) puts <<~EOF @@ -259,9 +331,7 @@ def to_s end def close_type(name) - $all_types.select do |t| - t.type.is_a?(YAMLCAst::CustomType) && t.type.name == name - end.each do |t| + API.aliases_of(name).each do |t| puts < 1}: #{unknown.join(', ')}!" unless unknown.empty? +end diff --git a/utils/meta_parameters.rb b/utils/meta_parameters.rb index 13dfdc4bb..d38d50950 100644 --- a/utils/meta_parameters.rb +++ b/utils/meta_parameters.rb @@ -124,8 +124,8 @@ def initialize(command) ev = LTTng::TracepointField.new ev.macro = :ctf_string - ev.name = "#{RESULT_NAME}_val" - ev.expression = "#{RESULT_NAME}" + ev.name = "#{command.result_name}_val" + ev.expression = command.result_name @lttng_out_type = ev end end @@ -166,7 +166,7 @@ def initialize(command, name, type = nil) else t.type end - lttngt = st.lttng_type + lttngt = st.lttng_type(command.type_classes) lttngt.name = name + '_val' if lttngt.macro == :ctf_array_text lttngt.macro = :ctf_sequence_text @@ -246,7 +246,7 @@ def initialize(command, name, size) t.type end y = YAMLCAst::Array.new(type: tt) - lttngt = y.lttng_type(length: sz, length_type: st) + lttngt = y.lttng_type(command.type_classes, length: sz, length_type: st) lttngt.name = name + '_vals' lttngt.expression = sanitize_expression("#{name}") @lttng_type = lttngt @@ -290,7 +290,7 @@ def initialize(command, name, size) t.type end y = YAMLCAst::Array.new(type: tt) - lttngt = y.lttng_type(length: size, length_type: nil) + lttngt = y.lttng_type(command.type_classes, length: size, length_type: nil) lttngt.name = name + '_vals' lttngt.expression = sanitize_expression("#{name}") @lttng_type = lttngt @@ -346,7 +346,7 @@ def initialize(command, name, size) t.type.type end y = YAMLCAst::Array.new(type: tt) - lttngt = y.lttng_type(length: sz, length_type: st) + lttngt = y.lttng_type(command.type_classes, length: sz, length_type: st) lttngt.name = name + '_val_vals' lttngt.expression = sanitize_expression("*#{name}") @lttng_type = lttngt diff --git a/utils/test_compare_generated_file.py b/utils/test_compare_generated_file.py index 02b31f671..4ffd26e80 100644 --- a/utils/test_compare_generated_file.py +++ b/utils/test_compare_generated_file.py @@ -69,6 +69,15 @@ ] filenames += [ + "backends/opencl/tracer_opencl.c", + "backends/opencl/btx_cl_model.yaml", + "backends/opencl/opencl_model.yaml", + "backends/opencl/opencl_arguments.tp", + "backends/opencl/opencl_build.tp", + "backends/opencl/opencl_devices.tp", + "backends/opencl/opencl_dump.tp", + "backends/opencl/opencl_source.tp", + "backends/opencl/opencl_tracepoints.tp", "backends/opencl/opencl_profiling.tp", ] diff --git a/utils/thapi_log_to_bt_source_component.rb b/utils/thapi_log_to_bt_source_component.rb index f27fe4120..e17cfd714 100755 --- a/utils/thapi_log_to_bt_source_component.rb +++ b/utils/thapi_log_to_bt_source_component.rb @@ -15,7 +15,7 @@ require 'time' require 'set' -$empty_array_name = 'EMPTY_ARRAY' +EMPTY_ARRAY_NAME = 'EMPTY_ARRAY' SOURCE_TEMPLATE = <<~TEXT.freeze /* Code generated by #{__FILE__} */ @@ -23,7 +23,7 @@ #include #include - static const void** #{$empty_array_name}; + static const void** #{EMPTY_ARRAY_NAME}; void btx_push_usr_messages(void *btx_handle, void *usr_data, btx_source_status_t *status) { <%- if not data.empty? and not data.first[:hostname].nil? -%> @@ -58,7 +58,7 @@ def get_values(name, value) if value.empty? count = 0 # use static const empty array defined in callback template - value = $empty_array_name + value = EMPTY_ARRAY_NAME else values = value.split(/, */) count = 1 diff --git a/utils/type_registry.rb b/utils/type_registry.rb new file mode 100644 index 000000000..9ce8fd9ee --- /dev/null +++ b/utils/type_registry.rb @@ -0,0 +1,37 @@ +# The type facts the babeltrace-model generator needs to classify a field. +require_relative 'yaml_ast' + +class TypeRegistry + attr_reader :types_by_name, :enum_names, :bitfield_names, :struct_names, :class_namer + + # The name lists come from ApiModel#classified; this only derives the + # integer-size/-sign lookups (the C scalar widths, plus a 32-bit signed entry + # per enum) and the by-name type index. + def initialize(all_types:, all_enums:, enum_names:, bitfield_names:, struct_names:, class_namer:) + @types_by_name = all_types.to_h { |t| [t.name, t] } + @enum_names = enum_names + @bitfield_names = bitfield_names + @struct_names = struct_names + @class_namer = class_namer + + @integer_sizes = INT_SIZE_MAP.transform_values { |v| v * 8 } + @integer_signed = INT_SIGN_MAP.dup + all_enums.each do |e| + @integer_sizes["enum #{e.name}"] = 32 + @integer_signed["enum #{e.name}"] = true + end + end + + # A pointer or an array is an address, whatever it points at. + def integer_size(type) + return 64 if type.match(/\*|\[.*\]/) + + @integer_sizes.fetch(type) { raise "unknown integer type #{type}" } + end + + def integer_signed?(type) + return false if type.match(/\*|\[.*\]/) + + @integer_signed.fetch(type) { raise "unknown integer type #{type}" } + end +end diff --git a/utils/yaml_ast.rb b/utils/yaml_ast.rb index fdb313148..c8f22e1a3 100644 --- a/utils/yaml_ast.rb +++ b/utils/yaml_ast.rb @@ -1,3 +1,6 @@ +require 'yaml' +require 'set' + module YAMLCAst class Type attr_reader :const, :restrict, :volatile @@ -324,6 +327,14 @@ def to_s(name = nil, no_types = false) 'declaration' => Declaration, } + # Parse an api.yaml into its five lists of AST nodes, keyed as the file keys + # them. A header that declares no unions simply has no 'unions' key, so a + # list the file omits is absent rather than empty -- ApiModel, which is what + # callers actually want, supplies the defaults. + def self.load_file(path) + from_yaml_ast(YAML.load_file(path)) + end + def self.from_yaml_ast(ast) res = {} ast.each do |k, v| @@ -431,61 +442,109 @@ def find_types_map(types, cast_type, map) FFI_INT_TYPE_MAP = INT_TYPE_MAP.map { |k, v| [k, v[2]] }.to_h INT_TYPES = INT_TYPE_MAP.keys +# Integer types the tracer logs in hex rather than decimal. An API can name +# more of its own -- see ApiModel's hex_ints. HEX_INT_TYPES = %w[ intptr_t uintptr_t -] +].freeze FFI_FLOAT_TYPE_MAP = { 'float' => 'ffi_type_float', 'double' => 'ffi_type_double', } -FLOAT_TYPES = FFI_FLOAT_TYPE_MAP.keys FFI_TYPE_MAP = {} -OBJECT_TYPES = [] -ENUM_TYPES = [] -STRUCT_TYPES = [] -UNION_TYPES = [] -POINTER_TYPES = [] - -def find_all_types(types) - objs = types.filter_map do |t| - t.name if t.type.is_a?(YAMLCAst::Pointer) && t.type.type.is_a?(YAMLCAst::Struct) +# A typedef of pointer-to-struct is an opaque handle -- an "object" -- rather +# than a pointer the tracer should dereference. +# +# The struct is not always named directly: headers also spell it in two steps, +# typedef struct _Foo Foo; typedef Foo *Foo_t; +# which puts a CustomType between the pointer and the struct. Resolve those +# aliases through `types` so both spellings classify the same way; otherwise +# Foo_t classifies as a plain pointer. +def object_typedef?(t, types) + return false unless t.type.is_a?(YAMLCAst::Pointer) + + target = t.type.type + seen = Set.new + while target.is_a?(YAMLCAst::CustomType) && seen.add?(target.name) + aliased = types.find { |x| x.name == target.name } + break unless aliased + + target = aliased.type end - OBJECT_TYPES.concat objs - transitive_closure(types, OBJECT_TYPES) + target.is_a?(YAMLCAst::Struct) +end - find_types(types, YAMLCAst::Int, INT_TYPES) - find_types(types, YAMLCAst::Char, INT_TYPES) - find_types(types, YAMLCAst::Float, FLOAT_TYPES) - find_types(types, YAMLCAst::Enum, ENUM_TYPES) - find_types(types, YAMLCAst::Struct, STRUCT_TYPES) - find_types(types, YAMLCAst::Union, UNION_TYPES) - ptrs = types.filter_map do |t| - if (t.type.is_a?(YAMLCAst::Pointer) && !t.type.type.is_a?(YAMLCAst::Struct)) || t.type.is_a?(YAMLCAst::Function) - t.name +# How one API's typedef names sort into the categories the tracer generators +# care about. +# +# `integers` starts from the fixed C scalar names rather than being purely the +# API's own: a typedef chain bottoms out in `int` or `uint32_t`, so the +# category has to contain both to answer "is this an integer?" in one lookup. +TypeClasses = Struct.new(:objects, :integers, :hex_ints, :enums, :structs, :unions, :pointers, + keyword_init: true) do + # The one category a typedef name falls into, or nil when this API never + # names it. The order is the answer: an object typedef is a pointer under the + # hood and a hex int is an integer, so the more specific category has to win. + # The categories are otherwise disjoint (verified by hand across all seven + # backends), so nothing below the first match can also apply. + def category_of(name) + case name + when *objects, *pointers then :address + when *hex_ints then :hex_int + when *integers then :integer + when *enums then :enum + when *structs, *unions then :aggregate end end - POINTER_TYPES.concat ptrs + + def aggregate?(name) + category_of(name) == :aggregate + end end -STRUCT_MAP = {} +# `hex_ints` are the API's own integer types to log in hex, on top of the ones +# every API shares. +def find_all_types(types, hex_ints: []) + objects = transitive_closure(types, types.filter_map { |t| t.name if object_typedef?(t, types) }) + # Int and Char share one category, and the char pass closes over the list the + # int pass produced, so a typedef aliasing either resolves the same way. + integers = find_types(types, YAMLCAst::Int, INT_TYPES.dup) + integers = find_types(types, YAMLCAst::Char, integers) + pointers = types.filter_map do |t| + t.name if (t.type.is_a?(YAMLCAst::Pointer) && !object_typedef?(t, types)) || t.type.is_a?(YAMLCAst::Function) + end + + TypeClasses.new( + objects: objects, integers: integers, pointers: pointers, + hex_ints: HEX_INT_TYPES + hex_ints, + enums: find_types(types, YAMLCAst::Enum), + structs: find_types(types, YAMLCAst::Struct), + unions: find_types(types, YAMLCAst::Union) + ).each_pair { |_, v| v.freeze }.freeze +end -def gen_struct_map(types, structs) +# Each struct typedef mapped to its member list, so a meta-parameter naming +# `a->b` can be resolved to b's declaration. The layout is either inline on the +# typedef or carried by a separately declared struct of the same name. +def find_struct_map(types, structs) + struct_map = {} types.select { |t| t.type.is_a? YAMLCAst::Struct }.each do |t| if t.type.members - STRUCT_MAP[t.name] = t.type.members + struct_map[t.name] = t.type.members else mapped = structs.find { |str| str.name == t.type.name } - STRUCT_MAP[t.name] = mapped.members if mapped + struct_map[t.name] = mapped.members if mapped end end - transitive_closure_map(types, STRUCT_MAP) + transitive_closure_map(types, struct_map) + struct_map end -def gen_ffi_type_map(types) +def gen_ffi_type_map(types, type_classes) find_types_map(types, YAMLCAst::Int, INT_SIGN_MAP) find_types_map(types, YAMLCAst::Int, INT_SIZE_MAP) find_types_map(types, YAMLCAst::Int, FFI_INT_TYPE_MAP) @@ -496,18 +555,18 @@ def gen_ffi_type_map(types) find_types_map(types, YAMLCAst::Float, FFI_FLOAT_TYPE_MAP) FFI_TYPE_MAP.merge!(FFI_INT_TYPE_MAP, FFI_FLOAT_TYPE_MAP) - OBJECT_TYPES.each do |o| + type_classes.objects.each do |o| FFI_TYPE_MAP[o] = 'ffi_type_pointer' INT_SIZE_MAP[o] = 8 INT_SIGN_MAP[o] = false end # Debatable - ENUM_TYPES.each do |e| + type_classes.enums.each do |e| FFI_TYPE_MAP[e] = 'ffi_type_sint32' INT_SIZE_MAP[e] = 4 INT_SIGN_MAP[e] = true end - POINTER_TYPES.each do |p| + type_classes.pointers.each do |p| FFI_TYPE_MAP[p] = 'ffi_type_pointer' INT_SIZE_MAP[p] = 8 INT_SIGN_MAP[p] = false diff --git a/utils/yaml_ast_lttng.rb b/utils/yaml_ast_lttng.rb index 0578a10a3..461632864 100644 --- a/utils/yaml_ast_lttng.rb +++ b/utils/yaml_ast_lttng.rb @@ -5,19 +5,19 @@ module YAMLCAst class Type - def lttng_type + def lttng_type(_type_classes) raise "Unsupported type #{self}!" end end class Void - def lttng_type + def lttng_type(_type_classes) nil end end class Int - def lttng_type + def lttng_type(_type_classes) ev = LTTng::TracepointField.new ev.macro = :ctf_integer ev.type = name @@ -26,7 +26,7 @@ def lttng_type end class Float - def lttng_type + def lttng_type(_type_classes) ev = LTTng::TracepointField.new ev.macro = :ctf_float ev.type = name @@ -35,7 +35,7 @@ def lttng_type end class Char - def lttng_type + def lttng_type(_type_classes) ev = LTTng::TracepointField.new ev.macro = :ctf_integer ev.type = name @@ -44,7 +44,7 @@ def lttng_type end class Bool - def lttng_type + def lttng_type(_type_classes) ev = LTTng::TracepointField.new ev.macro = :ctf_integer ev.type = name @@ -53,7 +53,7 @@ def lttng_type end class Struct - def lttng_type + def lttng_type(_type_classes) ev = LTTng::TracepointField.new ev.macro = :ctf_array_text ev.type = :uint8_t @@ -67,7 +67,7 @@ def [](name) end class Union - def lttng_type + def lttng_type(_type_classes) ev = LTTng::TracepointField.new ev.macro = :ctf_array_text ev.type = :uint8_t @@ -77,7 +77,7 @@ def lttng_type end class Enum - def lttng_type + def lttng_type(_type_classes) ev = LTTng::TracepointField.new ev.macro = :ctf_integer ev.type = "enum #{name}" @@ -86,7 +86,7 @@ def lttng_type end class Pointer - def lttng_type + def lttng_type(_type_classes) ev = LTTng::TracepointField.new ev.macro = :ctf_integer_hex ev.type = :uintptr_t @@ -96,19 +96,14 @@ def lttng_type end class Declaration - def lttng_type - r = type.lttng_type + def lttng_type(type_classes) + r = type.lttng_type(type_classes) r.name = name r.expression = case type when Struct, Union "&#{name}" when CustomType - case type.name - when *STRUCT_TYPES, *UNION_TYPES - "&#{name}" - else - name - end + type_classes.aggregate?(type.name) ? "&#{name}" : name else name end @@ -117,23 +112,23 @@ def lttng_type end class CustomType - def lttng_type + def lttng_type(type_classes) ev = LTTng::TracepointField.new - case name - when *OBJECT_TYPES, *POINTER_TYPES + case type_classes.category_of(name) + when :address ev.macro = :ctf_integer_hex ev.type = :uintptr_t ev.cast = 'uintptr_t' - when *HEX_INT_TYPES + when :hex_int ev.macro = :ctf_integer_hex ev.type = name - when *INT_TYPES + when :integer ev.macro = :ctf_integer ev.type = name - when *ENUM_TYPES + when :enum ev.macro = :ctf_integer ev.type = :int32_t - when *STRUCT_TYPES, *UNION_TYPES + when :aggregate ev.macro = :ctf_array_text ev.type = :uint8_t ev.length = "sizeof(#{name})" @@ -145,7 +140,7 @@ def lttng_type end class Array - def lttng_type(length: nil, length_type: nil) + def lttng_type(type_classes, length: nil, length_type: nil) ev = LTTng::TracepointField.new if length ev.length = length @@ -177,28 +172,23 @@ def lttng_type(length: nil, length_type: nil) ev.macro = :"ctf_#{lttng_arr_type}_text" ev.type = type.name when YAMLCAst::CustomType - case type.name - # Usually binary data or text - when 'uint8_t' - ev.macro = :"ctf_#{lttng_arr_type}_text" - ev.type = :uint8_t - if ev.length - ev.length = "(#{ev.length}) * sizeof(uint8_t)" - ev.length_type = 'size_t' - end - when *OBJECT_TYPES, *POINTER_TYPES + # A uint8_t array is binary data or text rather than a run of numbers, + # so it gets an aggregate's treatment -- bytes, sized in bytes -- even + # though the name classifies as an integer. + case type.name == 'uint8_t' ? :aggregate : type_classes.category_of(type.name) + when :address ev.macro = :"ctf_#{lttng_arr_type}_hex" ev.type = :uintptr_t - when *HEX_INT_TYPES + when :hex_int ev.macro = :"ctf_#{lttng_arr_type}_hex" ev.type = type.name - when *INT_TYPES + when :integer ev.macro = :"ctf_#{lttng_arr_type}" ev.type = type.name - when *ENUM_TYPES + when :enum ev.macro = :"ctf_#{lttng_arr_type}" ev.type = :int32_t - when *STRUCT_TYPES, *UNION_TYPES + when :aggregate ev.macro = :"ctf_#{lttng_arr_type}_text" ev.type = :uint8_t if ev.length @@ -206,10 +196,10 @@ def lttng_type(length: nil, length_type: nil) ev.length_type = 'size_t' end else - super + super(type_classes) end else - super + super(type_classes) end ev end