Skip to content

vkconfig: Add layer manifest validation multithreading #2720

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
herlience wants to merge 3 commits into from
Closed

vkconfig: Add layer manifest validation multithreading #2720

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bcb9446
vkconfig: Remove Windows ARM release notification workaround
herlience Aug 19, 2026
3f6665f
vkconfig: Add layer manifest validation multithreading (#2659)
herlience Sep 2, 2026
67dc0f5
Delete vkconfig_gui/tab_preferences.cpp
herlience Sep 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
201 changes: 119 additions & 82 deletions vkconfig_core/layer_manager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@
#include <QDesktopServices>
#include <QFileDialog>

#include <future>
#include <thread>
#include <atomic>

static std::vector<Path> GetEnvVariablePaths(const char *variable_name, LayerType type) {
std::vector<Path> result;

Expand Down Expand Up @@ -457,47 +461,77 @@ void LayerManager::ApplyLayerDescriptor() {
void LayerManager::LoadAllInstalledLayers(ConfiguratorMode configurator_mode) {
std::array<std::vector<Path>, LAYERS_PATHS_COUNT> paths;

// Search new layers
paths[LAYERS_PATHS_IMPLICIT_SYSTEM] = GetImplicitLayerPaths();

// LAYERS_PATHS_IMPLICIT_ENV_SET: VK_IMPLICIT_LAYER_PATH env variables
paths[LAYERS_PATHS_IMPLICIT_ENV_SET] = GetEnvVariablePaths("VK_IMPLICIT_LAYER_PATH", LAYER_TYPE_IMPLICIT);

// LAYERS_PATHS_IMPLICIT_ENV_ADD: VK_ADD_IMPLICIT_LAYER_PATH env variables
paths[LAYERS_PATHS_IMPLICIT_ENV_ADD] = GetEnvVariablePaths("VK_ADD_IMPLICIT_LAYER_PATH", LAYER_TYPE_IMPLICIT);

// LAYERS_PATHS_EXPLICIT_SYSTEM
paths[LAYERS_PATHS_EXPLICIT_SYSTEM] = GetExplicitLayerPaths();

// LAYERS_PATHS_EXPLICIT_ENV_SET: VK_LAYER_PATH env variables
paths[LAYERS_PATHS_EXPLICIT_ENV_SET] = GetEnvVariablePaths("VK_LAYER_PATH", LAYER_TYPE_EXPLICIT);

// LAYERS_PATHS_EXPLICIT_ENV_ADD: VK_ADD_LAYER_PATH env variables
paths[LAYERS_PATHS_EXPLICIT_ENV_ADD] = GetEnvVariablePaths("VK_ADD_LAYER_PATH", LAYER_TYPE_EXPLICIT);

// LAYERS_PATHS_SDK
paths[LAYERS_PATHS_SDK].push_back(Path(Path::SDK_EXPLICIT_LAYERS));

// LAYERS_PATHS_GUI
std::vector<Path> added_paths = this->BuildLayerPaths();
paths[LAYERS_PATHS_GUI].insert(paths[LAYERS_PATHS_GUI].begin(), added_paths.begin(), added_paths.end());


struct PendingFile {
Path path;
LayerType type;
};
std::vector<PendingFile> pending_files;

for (std::size_t group_index = 0, group_count = paths.size(); group_index < group_count; ++group_index) {
const LayersPaths layers_path = static_cast<LayersPaths>(group_index);

const std::vector<Path> &paths_group = paths[group_index];

for (std::size_t i = 0, n = paths_group.size(); i < n; ++i) {
const std::vector<Path> &layers_paths = ::CollectLayersPaths(paths_group[i]);

for (std::size_t p = 0, o = layers_paths.size(); p < o; ++p) {
this->LoadLayers(layers_paths[p], ::GetLayerType(layers_path), configurator_mode);
pending_files.push_back({layers_paths[p], ::GetLayerType(layers_path)});
}
}
}

const std::size_t file_count = pending_files.size();
std::vector<LayerParseResult> parse_results(file_count);

const unsigned int hw_threads = std::max(1u, std::thread::hardware_concurrency());
const std::size_t thread_count = std::min<std::size_t>(hw_threads, file_count);

if (thread_count <= 1 || file_count == 0) {
for (std::size_t i = 0; i < file_count; ++i) {
parse_results[i] = ParseLayerFile(pending_files[i].path, pending_files[i].type);
}
} else {
std::vector<std::future<void>> futures;
std::atomic<std::size_t> next_index{0};

auto worker = [&]() {
for (;;) {
const std::size_t index = next_index.fetch_add(1);
if (index >= file_count) break;
parse_results[index] = ParseLayerFile(pending_files[index].path, pending_files[index].type);
}
};

for (std::size_t t = 0; t < thread_count; ++t) {
futures.push_back(std::async(std::launch::async, worker));
}
for (auto &f : futures) {
f.wait();
}
}

// this->ApplyLayerDescriptor();
for (std::size_t i = 0; i < file_count; ++i) {
const LayerParseResult &parsed = parse_results[i];
if (parsed.status == LAYER_LOAD_INVALID) {
continue;
}
this->ApplyParsedLayer(parsed, configurator_mode);
}
}


LayerValidated LayerManager::Validate(const Path &layer_path, QString json_text, ConfiguratorMode configurator_mode) const {
JsonValidator validator;
bool result = validator.Check(json_text);
Expand Down Expand Up @@ -559,115 +593,118 @@ LayerDescriptor LayerManager::GetDescriptor(const Path &layer_path, const std::s
return LayerDescriptor();
}

LayerLoadStatus LayerManager::LoadLayers(const Path &layer_path, LayerType type, ConfiguratorMode configurator_mode) {
QFile file(layer_path.AbsolutePath().c_str());
if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
assert(0);
return LAYER_LOAD_INVALID;
}

QString json_text = file.readAll();
file.close();
LayerLoadStatus LayerManager::LoadLayer(const QJsonObject &json_layer_object, const Path &layer_path, LayerType type,
Version file_format_version, LayerDescriptor descriptor) {
Layer layer;
layer.type = type;
layer.manifest_path = layer_path;
layer.file_format_version = file_format_version;
layer.descriptor = descriptor;

// Convert the text to a JSON document & validate it.
// It does need to be a valid json formatted file.
QJsonParseError json_parse_error;
const QJsonDocument &json_document = QJsonDocument::fromJson(json_text.toUtf8(), &json_parse_error);
if (json_parse_error.error != QJsonParseError::NoError) {
return LAYER_LOAD_INVALID;
LayerLoadStatus status = layer.Load(json_layer_object);
if (status == LAYER_LOAD_INVALID || status == LAYER_LOAD_IGNORED) {
return status;
}

// Make sure it's not empty
if (json_document.isNull() || json_document.isEmpty()) {
return LAYER_LOAD_INVALID;
Layer *duplicated_layer = this->Find(layer.GetId(), false);
if (duplicated_layer != nullptr) {
if (duplicated_layer->descriptor.removed) {
duplicated_layer->descriptor.removed = false;
duplicated_layer->descriptor.enabled = true;
} else if (duplicated_layer->descriptor.last_modified != layer.descriptor.last_modified) {
// Reload when the manifest was updated
LayerLoadStatus reloaded_status = duplicated_layer->Load(json_layer_object);
status = reloaded_status == LAYER_LOAD_ADDED ? LAYER_LOAD_RELOADED : reloaded_status;
} else {
status = LAYER_LOAD_UNMODIFIED;
}
} else {
this->available_layers.push_back(layer);
}

Version file_format_version;
const QJsonObject &json_root_object = json_document.object();
if (json_root_object.value("file_format_version") != QJsonValue::Undefined) {
file_format_version = Version(json_root_object.value("file_format_version").toString().toStdString().c_str());
}
return status;
}

LayerLoadStatus LayerManager::ApplyParsedLayer(const LayerParseResult &parsed, ConfiguratorMode configurator_mode) {
LayerLoadStatus status = LAYER_LOAD_ADDED;
const std::string &last_modified = layer_path.LastModified();

if (json_root_object.value("layers") != QJsonValue::Undefined) {
const QJsonArray &json_layers_array = json_root_object.value("layers").toArray();
if (parsed.has_layers_array) {
const QJsonArray &json_layers_array = parsed.json_root_object.value("layers").toArray();

for (int i = 0, n = json_layers_array.size(); i < n; ++i) {
const QJsonObject &json_layer_object = json_layers_array[i].toObject();

std::string key = ReadStringValue(json_layer_object, "name");

if (key == "VK_LAYER_LUNARG_override" || !(key.rfind("VK_", 0) == 0)) {
status = LAYER_LOAD_IGNORED;
continue;
}

LayerDescriptor descriptor = this->GetDescriptor(layer_path, key);
LayerDescriptor descriptor = this->GetDescriptor(parsed.layer_path, key);

if (this->validate_manifests && (descriptor.last_modified != last_modified) || !descriptor.validated) {
descriptor.validated = this->Validate(layer_path, json_text, configurator_mode);
descriptor.last_modified = last_modified;
if (this->validate_manifests && (descriptor.last_modified != parsed.last_modified) || !descriptor.validated) {
descriptor.validated = this->Validate(parsed.layer_path, parsed.json_text, configurator_mode);
descriptor.last_modified = parsed.last_modified;
}

status = this->LoadLayer(json_layer_object, layer_path, type, file_format_version, descriptor);
status = this->LoadLayer(json_layer_object, parsed.layer_path, parsed.type, parsed.file_format_version, descriptor);
}
} else if (json_root_object.value("layer") != QJsonValue::Undefined) {
const QJsonObject &json_layer_object = json_root_object.value("layer").toObject();

} else if (parsed.json_root_object.value("layer") != QJsonValue::Undefined) {
const QJsonObject &json_layer_object = parsed.json_root_object.value("layer").toObject();
std::string layer_key = ReadStringValue(json_layer_object, "name");

if (layer_key == "VK_LAYER_LUNARG_override" || !(layer_key.rfind("VK_", 0) == 0)) {
return LAYER_LOAD_IGNORED;
}

LayerDescriptor descriptor = this->GetDescriptor(layer_path, layer_key);
LayerDescriptor descriptor = this->GetDescriptor(parsed.layer_path, layer_key);

if (this->validate_manifests &&
(descriptor.last_modified != last_modified || descriptor.validated == LAYER_VALIDATE_NONE)) {
descriptor.validated = this->Validate(layer_path, json_text, configurator_mode);
descriptor.last_modified = last_modified;
(descriptor.last_modified != parsed.last_modified || descriptor.validated == LAYER_VALIDATE_NONE)) {
descriptor.validated = this->Validate(parsed.layer_path, parsed.json_text, configurator_mode);
descriptor.last_modified = parsed.last_modified;
}

status = this->LoadLayer(json_layer_object, layer_path, type, file_format_version, descriptor);
status = this->LoadLayer(json_layer_object, parsed.layer_path, parsed.type, parsed.file_format_version, descriptor);
} else {
assert(0);
}

return status;
}

LayerLoadStatus LayerManager::LoadLayer(const QJsonObject &json_layer_object, const Path &layer_path, LayerType type,
Version file_format_version, LayerDescriptor descriptor) {
Layer layer;
layer.type = type;
layer.manifest_path = layer_path;
layer.file_format_version = file_format_version;
layer.descriptor = descriptor;
LayerParseResult LayerManager::ParseLayerFile(const Path &layer_path, LayerType type) {
LayerParseResult result;
result.layer_path = layer_path;
result.type = type;

LayerLoadStatus status = layer.Load(json_layer_object);
if (status == LAYER_LOAD_INVALID || status == LAYER_LOAD_IGNORED) {
return status;
QFile file(layer_path.AbsolutePath().c_str());
if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) {
result.status = LAYER_LOAD_INVALID;
return result;
}

Layer *duplicated_layer = this->Find(layer.GetId(), false);
if (duplicated_layer != nullptr) {
if (duplicated_layer->descriptor.removed) {
duplicated_layer->descriptor.removed = false;
duplicated_layer->descriptor.enabled = true;
} else if (duplicated_layer->descriptor.last_modified != layer.descriptor.last_modified) {
// Reload when the manifest was updated
LayerLoadStatus reloaded_status = duplicated_layer->Load(json_layer_object);
status = reloaded_status == LAYER_LOAD_ADDED ? LAYER_LOAD_RELOADED : reloaded_status;
} else {
status = LAYER_LOAD_UNMODIFIED;
}
} else {
this->available_layers.push_back(layer);
QString json_text = file.readAll();
file.close();

QJsonParseError json_parse_error;
const QJsonDocument &json_document = QJsonDocument::fromJson(json_text.toUtf8(), &json_parse_error);
if (json_parse_error.error != QJsonParseError::NoError || json_document.isNull() || json_document.isEmpty()) {
result.status = LAYER_LOAD_INVALID;
return result;
}

return status;
result.json_text = json_text;
result.json_root_object = json_document.object();
result.last_modified = layer_path.LastModified();

if (result.json_root_object.value("file_format_version") != QJsonValue::Undefined) {
result.file_format_version = Version(result.json_root_object.value("file_format_version").toString().toStdString().c_str());
}

result.has_layers_array = result.json_root_object.value("layers") != QJsonValue::Undefined;
result.status = LAYER_LOAD_ADDED;
return result;
}

void LayerManager::RemoveLayer(LayerId id) {
Expand Down
18 changes: 16 additions & 2 deletions vkconfig_core/layer_manager.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,17 @@ struct LayerDisplay {
LayerDescriptor descriptor;
};

struct LayerParseResult {
Path layer_path;
LayerType type;
LayerLoadStatus status = LAYER_LOAD_INVALID;
QJsonObject json_root_object;
QString json_text;
Version file_format_version;
std::string last_modified;
bool has_layers_array = false;
};

bool operator<(const LayerDisplay& a, const LayerDisplay& b);

class LayerManager : public Serialize {
Expand All @@ -61,7 +72,6 @@ class LayerManager : public Serialize {
Layer* FindFromManifest(const Path& manifest_path, bool find_disabled_layers = false);

void LoadAllInstalledLayers(ConfiguratorMode configurator_mode);
LayerLoadStatus LoadLayers(const Path& layer_path, LayerType type, ConfiguratorMode configurator_mode);

void RemoveLayer(LayerId id);
void EnableLayer(LayerId id, bool enable);
Expand All @@ -87,7 +97,11 @@ class LayerManager : public Serialize {
LayerValidated Validate(const Path& layer_path, QString json_text, ConfiguratorMode configurator_mode) const;

LayerLoadStatus LoadLayer(const QJsonObject& json_layer_object, const Path& layer_path, LayerType type,
Version file_format_version, LayerDescriptor descriptor);
Version file_format_version, LayerDescriptor descriptor);

LayerLoadStatus ApplyParsedLayer(const LayerParseResult& parsed, ConfiguratorMode configurator_mode);

static LayerParseResult ParseLayerFile(const Path& layer_path, LayerType type);

void ApplyLayerDescriptor();

Expand Down
Loading