From b3ad6b4c3efe01980b3f695fc081fd01f3cec96e Mon Sep 17 00:00:00 2001 From: Lukas Grossberger Date: Tue, 13 Oct 2026 14:08:24 +0200 Subject: [PATCH] update to APSI 1.1 --- CMakeLists.txt | 27 ++-- README.md | 41 ++++- apsi/servers.py | 4 +- apsi/utils.py | 21 ++- docs/strategy-4-hybrid-wheels-sdist.md | 6 +- docs/strategy-5-conda-forge.md | 12 +- pyproject.toml | 2 +- setup.py | 98 +++++++++--- src/main.cpp | 208 ++++++++++++++++++------- tests/conftest.py | 47 +++--- tests/integration_test.py | 36 ++++- tests/utils_test.py | 17 ++ 12 files changed, 373 insertions(+), 146 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index afa5c8b..054297a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.13.4) +cmake_minimum_required(VERSION 3.25) project(_pyapsi) @@ -10,6 +10,13 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) # See docs/README.md for details. set(APSI_USE_ASM OFF CACHE BOOL "Use ASM implementation" FORCE) +# PyAPSI exposes only APSI's "advanced" API on top of network::StreamChannel, so the +# ZeroMQ based channel and the command line interface are not needed. Dropping them also +# drops the ZeroMQ/cppzmq dependency from the build (see VCPKG_DEPENDENCIES in setup.py). +set(APSI_USE_ZMQ OFF CACHE BOOL "Use ZeroMQ for networking" FORCE) +set(APSI_BUILD_CLI OFF CACHE BOOL "Build example command line interface applications" FORCE) +set(APSI_BUILD_TESTS OFF CACHE BOOL "Build unit and integration tests for APSI" FORCE) + include(FetchContent) FetchContent_Declare( @@ -19,19 +26,13 @@ FetchContent_Declare( ) FetchContent_MakeAvailable(pybind11) -file(READ external/apsi/CMakeLists.txt APSI_CMAKE_CONTENT) -string(REPLACE "find_package(SEAL 4.1 QUIET REQUIRED)" "find_package(SEAL QUIET REQUIRED)" APSI_CMAKE_CONTENT "${APSI_CMAKE_CONTENT}") -file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/apsi_patched/CMakeLists.txt" "${APSI_CMAKE_CONTENT}") - -file(GLOB APSI_SUBDIRS LIST_DIRECTORIES true external/apsi/*) -foreach(SUBDIR ${APSI_SUBDIRS}) - if(IS_DIRECTORY ${SUBDIR}) - get_filename_component(DIRNAME ${SUBDIR} NAME) - file(COPY ${SUBDIR}/ DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/apsi_patched/${DIRNAME}/") - endif() -endforeach() +add_subdirectory(external/apsi) -add_subdirectory("${CMAKE_CURRENT_BINARY_DIR}/apsi_patched/") +# APSI's vendored FourQ headers are configured into ${CMAKE_BINARY_DIR}/common/apsi/fourq/, +# which equals its own include path only in a standalone APSI build. Here APSI is a +# subdirectory, so ${CMAKE_BINARY_DIR} is PyAPSI's build root and that path has to be +# added for APSI's own sources to find "apsi/fourq/...". +target_include_directories(apsi PUBLIC $) pybind11_add_module(_pyapsi src/main.cpp) diff --git a/README.md b/README.md index 2e53a06..5f52f1c 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,27 @@ Python wrapper for labeled and unlabeled asymmetric private set intersection ([APSI](https://github.com/microsoft/apsi)). +Built against APSI 1.1.0. + +## Upgrading to PyAPSI 0.3 + +PyAPSI 0.3 moves from APSI 0.12 to APSI 1.1, which brings a few user visible changes: + +- The log levels are named `"TRACE"`, `"DEBUG"`, `"INFO"`, `"WARNING"`, `"ERROR"` and + `"SUPPRESS"`. The previous `"ALL"` and `"OFF"` are still accepted as `"TRACE"` and + `"SUPPRESS"` respectively. +- APSI retuned its shipped parameter sets, because most of them were above the documented + 2^-40 false-positive probability per query. A server and a client have to agree on their + parameters, so both sides of a deployment should move to the new values together; the + example below is APSI's `parameters/1M-256.json`. +- APSI's serialization version is unchanged and the schemas stayed wire compatible, so a + database saved by an earlier PyAPSI release should still load, but that is untested + here. A cuckoo filter written before APSI 1.0 carries no assertion that nothing was + dropped from it, so it is loaded as one that may have dropped items; that costs some + query time on the affected bin bundles, not correctness. +- Each query a client sends now carries key material of its own, so that a server can no + longer tell from the public keys that two queries came from the same client. + ## Installation ### From PyPI (builds from source) @@ -21,8 +42,9 @@ This downloads the source distribution and builds APSI + all dependencies locall The build automatically selects the best CPU optimizations for your machine. **Requirements:** -- C++ compiler (GCC >= 9, Clang >= 10, or MSVC >= 2019) -- CMake >= 3.13.4 +- C++ compiler with full C++17 support (GCC, Clang, or Visual Studio 2022/2026, as + required by APSI 1.1) +- CMake >= 3.25 - Python 3.11–3.14 - Internet access (vcpkg downloads dependencies during build) @@ -55,18 +77,21 @@ apsi_params = """ { "table_params": { "hash_func_count": 3, - "table_size": 512, - "max_items_per_bin": 92 + "table_size": 585, + "max_items_per_bin": 180 }, - "item_params": {"felts_per_item": 8}, + "item_params": {"felts_per_item": 7}, "query_params": { "ps_low_degree": 0, - "query_powers": [1, 3, 4, 5, 8, 14, 20, 26, 32, 38, 41, 42, 43, 45, 46] + "query_powers": [ + 1, 3, 4, 6, 10, 13, 15, 21, 29, 37, 45, + 53, 61, 69, 77, 81, 83, 86, 87, 90, 92, 96 + ] }, "seal_params": { - "plain_modulus": 40961, + "plain_modulus": 65537, "poly_modulus_degree": 4096, - "coeff_modulus_bits": [40, 32, 32] + "coeff_modulus_bits": [40, 40, 24] } } """ diff --git a/apsi/servers.py b/apsi/servers.py index 5899a33..2aeb1f1 100644 --- a/apsi/servers.py +++ b/apsi/servers.py @@ -59,8 +59,8 @@ class LabeledServer(_BaseServer): @property def max_label_length(self) -> int: - # Refers to what is initialized via max_label_length via the underlying C++ - # implementation, assuring availability also for loaded databases + # Read from the underlying APSI database itself, so that it also reflects what a + # database loaded via `load_db` was initialized with return self._db_label_byte_count def __init__(self): diff --git a/apsi/utils.py b/apsi/utils.py index ff10603..a31ad0e 100644 --- a/apsi/utils.py +++ b/apsi/utils.py @@ -9,7 +9,11 @@ ) -VALID_LOG_LEVELS = ["ALL", "DEBUG", "INFO", "WARNING", "ERROR", "OFF"] +VALID_LOG_LEVELS = ["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "SUPPRESS"] + +# APSI 0.13.0 renamed the most verbose log level from "all" to "trace" and the quietest +# from "off" to "suppress". The previous names remain accepted. +DEPRECATED_LOG_LEVELS = {"ALL": "TRACE", "OFF": "SUPPRESS"} def set_thread_count(thread_count: int) -> None: @@ -34,11 +38,13 @@ def set_log_level(level: str) -> None: """Set APSI log level. Args: - level: One of "ALL", "DEBUG", "INFO", "WARNING", "ERROR", "OFF" + level: One of "TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "SUPPRESS"; + the former names "ALL" and "OFF" are accepted as "TRACE" and "SUPPRESS" Raises: ValueError: In case an invalid log level is provided """ + level = DEPRECATED_LOG_LEVELS.get(level, level) if level not in VALID_LOG_LEVELS: raise ValueError(f"Invalid log level {level}, choose one of {VALID_LOG_LEVELS}") _set_log_level(level) @@ -50,10 +56,17 @@ def enable_console_log() -> None: def disable_console_log() -> None: - """Disable APSI logging to standard output.""" + """Disable APSI logging to standard output. + + If a log file was set via `set_log_file`, output keeps going there. + """ _set_console_log_disabled(True) def set_log_file(file_path: str) -> None: - """Set file path to which APSI log output should be written.""" + """Set file path to which APSI log output should be appended. + + Raises: + ValueError: In case the file cannot be opened for writing + """ _set_log_file(file_path) diff --git a/docs/strategy-4-hybrid-wheels-sdist.md b/docs/strategy-4-hybrid-wheels-sdist.md index daff123..0a15f2c 100644 --- a/docs/strategy-4-hybrid-wheels-sdist.md +++ b/docs/strategy-4-hybrid-wheels-sdist.md @@ -54,7 +54,7 @@ before-all = """ yum install -y git zip && \ git clone https://github.com/microsoft/vcpkg /opt/vcpkg && \ /opt/vcpkg/bootstrap-vcpkg.sh && \ - /opt/vcpkg/vcpkg install seal[no-throw-tran]:x64-linux kuku:x64-linux log4cplus:x64-linux cppzmq:x64-linux flatbuffers:x64-linux jsoncpp:x64-linux + /opt/vcpkg/vcpkg install seal[core,zstd,no-throw-tran]:x64-linux ms-gsl:x64-linux kuku:x64-linux flatbuffers:x64-linux jsoncpp:x64-linux """ environment = "VCPKG_ROOT_DIR=/opt/vcpkg" @@ -64,7 +64,7 @@ before-all = """ brew install git && \ git clone https://github.com/microsoft/vcpkg /tmp/vcpkg && \ /tmp/vcpkg/bootstrap-vcpkg.sh && \ - /tmp/vcpkg/vcpkg install seal[no-throw-tran] kuku log4cplus cppzmq flatbuffers jsoncpp + /tmp/vcpkg/vcpkg install seal[core,zstd,no-throw-tran] ms-gsl kuku flatbuffers jsoncpp """ environment = "VCPKG_ROOT_DIR=/tmp/vcpkg" @@ -73,7 +73,7 @@ archs = "AMD64" before-all = """ git clone https://github.com/microsoft/vcpkg C:\\vcpkg && \ C:\\vcpkg\\bootstrap-vcpkg.bat && \ - C:\\vcpkg\\vcpkg install seal[no-throw-tran]:x64-windows-static-md kuku:x64-windows-static-md log4cplus:x64-windows-static-md cppzmq:x64-windows-static-md flatbuffers:x64-windows-static-md jsoncpp:x64-windows-static-md + C:\\vcpkg\\vcpkg install seal[core,zstd,no-throw-tran]:x64-windows-static-md ms-gsl:x64-windows-static-md kuku:x64-windows-static-md flatbuffers:x64-windows-static-md jsoncpp:x64-windows-static-md """ environment = "VCPKG_ROOT_DIR=C:\\vcpkg" ``` diff --git a/docs/strategy-5-conda-forge.md b/docs/strategy-5-conda-forge.md index def46ca..53af8d4 100644 --- a/docs/strategy-5-conda-forge.md +++ b/docs/strategy-5-conda-forge.md @@ -16,15 +16,17 @@ Packaging PyAPSI for conda-forge would provide an alternative distribution chann | Dependency | Package name | Status | |------------|-------------|--------| -| SEAL | `libseal` or `seal` | Needs verification | +| SEAL (>= 4.4.5, built with `SEAL_THROW_ON_TRANSPARENT_CIPHERTEXT=OFF`) | `libseal` or `seal` | Needs verification | | Kuku | `kuku` | Needs verification | -| log4cplus | `log4cplus` | Likely available | -| cppzmq | `cppzmq` | Available | -| zeromq | `zeromq` | Available | | flatbuffers | `flatbuffers` | Available | -| jsoncpp | `jsoncpp` | Available | +| jsoncpp (>= 1.9.5) | `jsoncpp` | Available | | Microsoft GSL | `ms-gsl` | Available | +APSI 1.0 requires a SEAL built with `SEAL_THROW_ON_TRANSPARENT_CIPHERTEXT=OFF`, which a +conda-forge `seal` package is unlikely to provide, so that would have to be verified +first. ZeroMQ/cppzmq and log4cplus are not needed: PyAPSI builds APSI with +`APSI_USE_ZMQ=OFF`, and APSI 0.13.0 replaced log4cplus with its built-in logger. + If SEAL and/or Kuku are not on conda-forge, feedstocks for those packages would need to be created first. ## Approach diff --git a/pyproject.toml b/pyproject.toml index ac26b70..d34dca6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pyapsi" -version = "0.2.3" +version = "0.3.0" description = "Python wrapper for APSI" authors = ["Lukas Grossberger "] diff --git a/setup.py b/setup.py index 13f2c74..97fb59c 100644 --- a/setup.py +++ b/setup.py @@ -53,11 +53,26 @@ from setuptools import Extension, find_packages, setup from setuptools.command.build_ext import build_ext -__version__ = "0.2.3" - -APSI_VERSION = "0.12.0" -APSI_COMMIT = "b967a126b4e1c682b039afc2d76a98ea2c993230" -VCPKG_COMMIT = "4cf671c303551b27c6360c7c6e1ced5ec46d7258" +__version__ = "0.3.0" + +APSI_VERSION = "1.1.0" +APSI_COMMIT = "c8020026972258eabad054ad1225aea41f8d1d7d" +# Matches the "builtin-baseline" of APSI 1.1.0's vcpkg.json, so that the dependency +# versions are the ones APSI is tested against (Microsoft SEAL 4.4.5, jsoncpp 1.9.6). +VCPKG_COMMIT = "2c88ca199b27256e4c7d9f796216f82f2b976b0e" + +# APSI 1.1.0 dependencies, mirroring its vcpkg.json. Microsoft SEAL has to be built with +# SEAL_THROW_ON_TRANSPARENT_CIPHERTEXT=OFF ("no-throw-tran"), otherwise APSI refuses to +# compile. ZeroMQ/cppzmq is not needed because PyAPSI only uses the stream based channel +# (see APSI_USE_ZMQ in CMakeLists.txt), and log4cplus was dropped by APSI 0.13.0 in favor +# of its built-in logger. +VCPKG_DEPENDENCIES = [ + "seal[core,zstd,no-throw-tran]", + "ms-gsl", + "kuku", + "flatbuffers", + "jsoncpp", +] PLAT_TO_CMAKE = { "win32": "Win32", @@ -101,12 +116,35 @@ def _get_vcpkg_triplet(): return "x64-linux" -def _vcpkg_deps_installed(vcpkg_dir, triplet): - """Check if vcpkg dependencies are installed for the given triplet.""" +def _vcpkg_deps_stamp_path(vcpkg_src_dir, triplet): + """Return the path of the file recording which dependencies were installed.""" + return os.path.join(vcpkg_src_dir, f"pyapsi-deps-{triplet}.stamp") + + +def _vcpkg_deps_stamp(triplet): + """Return a marker identifying the dependency set that is required.""" + return "\n".join([VCPKG_COMMIT, triplet] + sorted(VCPKG_DEPENDENCIES)) + + +def _vcpkg_deps_installed(vcpkg_src_dir, triplet): + """Check if the required vcpkg dependencies are installed for the given triplet. + + The stamp records the vcpkg commit and the dependency list, so that an existing + `_vcpkg` directory left over from an earlier PyAPSI release (which pinned different + dependency versions) is detected and rebuilt instead of failing the CMake configure. + """ seal_config = os.path.join( - vcpkg_dir, "installed", triplet, "share", "seal", "SEALConfig.cmake" + vcpkg_src_dir, "installed", triplet, "share", "seal", "SEALConfig.cmake" ) - return os.path.isfile(seal_config) + if not os.path.isfile(seal_config): + return False + + stamp_path = _vcpkg_deps_stamp_path(vcpkg_src_dir, triplet) + if not os.path.isfile(stamp_path): + return False + + with open(stamp_path, "r") as fh: + return fh.read() == _vcpkg_deps_stamp(triplet) def _bootstrap_vcpkg(vcpkg_dir): @@ -163,18 +201,17 @@ def _bootstrap_vcpkg(vcpkg_dir): def _install_vcpkg_deps(vcpkg_src_dir, triplet): """Install APSI dependencies via vcpkg for the given triplet.""" print(f"Installing APSI dependencies via vcpkg (triplet: {triplet})...") - deps = [ - "seal[no-throw-tran]", - "kuku", - "log4cplus", - "cppzmq", - "flatbuffers", - "jsoncpp", - ] vcpkg_exec = os.path.join(vcpkg_src_dir, "vcpkg") if sys.platform.startswith("win"): vcpkg_exec = os.path.join(vcpkg_src_dir, "vcpkg.exe") - subprocess.check_call([vcpkg_exec, "install", "--triplet", triplet] + deps, cwd=vcpkg_src_dir) + subprocess.check_call( + [vcpkg_exec, "install", "--triplet", triplet] + VCPKG_DEPENDENCIES, + cwd=vcpkg_src_dir, + ) + + with open(_vcpkg_deps_stamp_path(vcpkg_src_dir, triplet), "w") as fh: + fh.write(_vcpkg_deps_stamp(triplet)) + print("vcpkg dependency installation complete.") @@ -184,14 +221,24 @@ def _get_vcpkg_toolchain(vcpkg_src_dir): def _ensure_apsi_source(): - """Download APSI source if not present (for sdist installs).""" + """Download the pinned APSI source if not already present. + + A stamp file records the commit, so that an `external/apsi` checkout of a different + APSI version (for example from a previous PyAPSI release) is replaced rather than + silently built against. + """ apsi_dir = os.path.join( os.path.dirname(os.path.abspath(__file__)), "external", "apsi" ) - if os.path.isdir(os.path.join(apsi_dir, "CMakeLists.txt")): - return - - print("APSI source not found, downloading...") + stamp_path = os.path.join(apsi_dir, ".pyapsi-apsi-commit") + if os.path.isfile(os.path.join(apsi_dir, "CMakeLists.txt")) and os.path.isfile( + stamp_path + ): + with open(stamp_path, "r") as fh: + if fh.read().strip() == APSI_COMMIT: + return + + print(f"APSI {APSI_VERSION} source not found, downloading...") os.makedirs(os.path.dirname(apsi_dir), exist_ok=True) tar_url = f"https://github.com/microsoft/APSI/archive/{APSI_COMMIT}.tar.gz" @@ -211,6 +258,9 @@ def _ensure_apsi_source(): shutil.rmtree(apsi_dir) shutil.move(extracted, apsi_dir) + with open(stamp_path, "w") as fh: + fh.write(APSI_COMMIT) + print("APSI source ready.") @@ -253,7 +303,7 @@ def build_extension(self, ext): else: vcpkg_src_dir = _bootstrap_vcpkg(vcpkg_dir) - if not _vcpkg_deps_installed(vcpkg_dir, triplet): + if not _vcpkg_deps_installed(vcpkg_src_dir, triplet): _install_vcpkg_deps(vcpkg_src_dir, triplet) toolchain = _get_vcpkg_toolchain(vcpkg_src_dir) diff --git a/src/main.cpp b/src/main.cpp index e68f60b..adaeffe 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -26,6 +26,8 @@ */ // STD +#include +#include #include #include #include @@ -33,6 +35,7 @@ #include #include #include +#include // pybind11 #include @@ -62,40 +65,65 @@ void sigint_handler(int param [[maybe_unused]]) exit(0); } -void set_log_level(const string &level) -{ - Log::Level ll; +/* +APSI 0.13.0 replaced the apsi::Log class with free functions and a pluggable apsi::Logger, +which means Log::SetLogFile and Log::SetConsoleDisabled are gone. PyAPSI keeps offering the +log file and the console as two knobs that can be set independently, so both settings are +remembered here and the logger matching their combination is installed on every change. +*/ +string _log_file_path; +bool _console_log_enabled = true; - if (level == "all" || level == "ALL") - { - ll = Log::Level::all; - } - else if (level == "debug" || level == "DEBUG") - { - ll = Log::Level::debug; - } - else if (level == "info" || level == "INFO") +void refresh_logger() +{ + if (!_log_file_path.empty()) { - ll = Log::Level::info; + SetLogger(NewFileLogger(_log_file_path, _console_log_enabled)); } - else if (level == "warning" || level == "WARNING") + else if (_console_log_enabled) { - ll = Log::Level::warning; + ResetDefaultLogger(); } - else if (level == "error" || level == "ERROR") + else { - ll = Log::Level::error; + // A logger without any handler discards every message. The level threshold is + // deliberately left alone, so that it still applies once a sink is installed again. + SetLogger(Logger::Create({}, {}, {})); } - else if (level == "off" || level == "OFF") +} + +void set_log_level(const string &level) +{ + string normalized_level(level.size(), '\0'); + transform(level.begin(), level.end(), normalized_level.begin(), [](unsigned char c) { + return static_cast(tolower(c)); + }); + + // APSI 0.13.0 renamed the most verbose level from "all" to "trace" and the quietest + // from "off" to "suppress". Both of the previous names remain accepted here. + if (normalized_level == "all") { - ll = Log::Level::off; + normalized_level = "trace"; } - else + else if (normalized_level == "off") { - throw invalid_argument("unknown log level"); + normalized_level = "suppress"; } - Log::SetLogLevel(ll); + // Throws std::invalid_argument for anything that is not a level name + SetLogLevel(normalized_level); +} + +void set_console_log_disabled(bool disabled) +{ + _console_log_enabled = !disabled; + refresh_logger(); +} + +void set_log_file(const string &log_file_path) +{ + _log_file_path = log_file_path; + refresh_logger(); } /* @@ -140,6 +168,13 @@ class APSIClient receiver_items.push_back(item.cast()); } + if (receiver_items.size() > oprf::oprf_query_count_max) + { + throw invalid_argument( + "a single query cannot hold more than " + to_string(oprf::oprf_query_count_max) + + " items"); + } + _oprf_receiver = Receiver::CreateOPRFReceiver(receiver_items); Request request = Receiver::CreateOPRFRequest(_oprf_receiver); @@ -151,10 +186,33 @@ class APSIClient { _channel.set_in_buffer(oprf_response_string); OPRFResponse oprf_response = to_oprf_response(_channel.receive_response()); - tie(_hashed_recv_items, _label_keys) = Receiver::ExtractHashes(oprf_response, _oprf_receiver); + if (!oprf_response) + { + throw runtime_error("Failed reading the OPRF response"); + } + auto hashes_and_label_keys = Receiver::ExtractHashes(oprf_response, _oprf_receiver); + _hashed_recv_items = move(hashes_and_label_keys.first); + _label_keys = move(hashes_and_label_keys.second); + if (_hashed_recv_items.size() != _oprf_receiver.item_count()) + { + // ExtractHashes reports a response that does not match the request by returning + // nothing, rather than by throwing + throw runtime_error("Failed extracting the OPRF hashes from the OPRF response"); + } + + // Since APSI 1.0 every query is expected to carry key material of its own, so that a + // server cannot tell from the public keys that two queries came from the same client. + // Receiver::create_query does not rotate the keys itself, because the result of an + // earlier query is encrypted under the key a rotation discards; a caller sending more + // than one query has to rotate them while none is in flight, which is here. + if (_query_created) + { + _receiver.reset_keys(); + } // Create query and send pair recv_query = _receiver.create_query(_hashed_recv_items); + _query_created = true; _itt = make_shared(move(recv_query.second)); _channel.send(move(recv_query.first)); @@ -163,54 +221,60 @@ class APSIClient py::list extract_unlabeled_result_from_query_response(const string &query_response_string) { - signal(SIGINT, sigint_handler); - - _channel.set_in_buffer(query_response_string); - QueryResponse query_response = to_query_response(_channel.receive_response()); - uint32_t package_count = query_response->package_count; - - vector rps; - while (package_count--) - { - rps.push_back(_channel.receive_result(_receiver.get_seal_context())); - } - - vector query_result = _receiver.process_result(_label_keys, *_itt, rps); - py::list matches; - for (auto const &qr : query_result) + for (auto const &qr : process_query_response(query_response_string)) matches.append(qr.found); return matches; } py::list extract_labeled_result_from_query_response(const string &query_response_string) + { + py::list labels; + for (auto const &qr : process_query_response(query_response_string)) + labels.append(qr.label.to_string()); + return labels; + } + +private: + vector process_query_response(const string &query_response_string) { signal(SIGINT, sigint_handler); + if (!_itt) + { + throw runtime_error("You need to build a query first."); + } + _channel.set_in_buffer(query_response_string); QueryResponse query_response = to_query_response(_channel.receive_response()); + if (!query_response) + { + throw runtime_error("Failed reading the query response"); + } uint32_t package_count = query_response->package_count; vector rps; while (package_count--) { - rps.push_back(_channel.receive_result(_receiver.get_seal_context())); + // Since APSI 1.0 a channel reports a result part it could not read by returning + // nullptr, which process_result would then skip silently + ResultPart rp = _channel.receive_result(_receiver.get_seal_context()); + if (!rp) + { + throw runtime_error("Failed reading all result parts of the query response"); + } + rps.push_back(move(rp)); } - vector query_result = _receiver.process_result(_label_keys, *_itt, rps); - - py::list labels; - for (auto const &qr : query_result) - labels.append(qr.label.to_string()); - return labels; + return _receiver.process_result(_label_keys, *_itt, rps); } -private: shared_ptr _itt; Receiver _receiver; oprf::OPRFReceiver _oprf_receiver = oprf::OPRFReceiver(vector()); vector _hashed_recv_items; - vector _label_keys; + LabelKeyVector _label_keys; + bool _query_created = false; StringStreamChannel _channel; }; @@ -223,7 +287,6 @@ class APSIServer string ¶ms_json, size_t label_byte_count, size_t nonce_byte_count, bool compressed) { - db_label_byte_count = label_byte_count; auto params = PSIParams::Load(params_json); _db = make_shared( params, label_byte_count, nonce_byte_count, compressed); @@ -235,12 +298,16 @@ class APSIServer { ofstream ofs; ofs.open(db_file_path, ios::binary); + if (!ofs.is_open()) + { + throw runtime_error("cannot open " + db_file_path + " for writing"); + } _db->save(ofs); ofs.close(); } catch (const exception &e) { - throw runtime_error("Failed saving database"); + throw runtime_error(string("Failed saving database: ") + e.what()); } } @@ -250,16 +317,29 @@ class APSIServer { ifstream ifs; ifs.open(db_file_path, ios::binary); + if (!ifs.is_open()) + { + throw runtime_error("cannot open " + db_file_path + " for reading"); + } auto [data, size] = SenderDB::Load(ifs); _db = make_shared(move(data)); ifs.close(); } catch (const exception &e) { - throw runtime_error("Failed loading database"); + // APSI 1.0 rejects a serialized SenderDB that omits a required field, so the + // reason for a failure is worth passing on + throw runtime_error(string("Failed loading database: ") + e.what()); } } + size_t get_label_byte_count() const + { + // Reads the value from the database itself, so that it is also correct for a + // database that was loaded rather than initialized here + return _db ? _db->get_label_byte_count() : 0; + } + void add_item(const string &input_item, const string &input_label) { Item item(input_item); @@ -288,10 +368,14 @@ class APSIServer { _channel.set_in_buffer(oprf_request_string); - OPRFRequest oprf_request2 = to_oprf_request(_channel.receive_operation( + OPRFRequest oprf_request = to_oprf_request(_channel.receive_operation( nullptr, network::SenderOperationType::sop_oprf)); - Sender::RunOPRF(oprf_request2, _db->get_oprf_key(), _channel); + if (!oprf_request) + { + throw runtime_error("Failed reading the OPRF request"); + } + Sender::RunOPRF(oprf_request, _db->get_oprf_key(), _channel); return py::bytes(_channel.extract_out_buffer()); } @@ -302,15 +386,22 @@ class APSIServer QueryRequest sender_query = to_query_request(_channel.receive_operation( _db->get_seal_context(), network::SenderOperationType::sop_query)); + if (!sender_query) + { + throw runtime_error("Failed reading the query request"); + } Query query(move(sender_query), _db); + if (!query) + { + // Since APSI 1.0 a query that the SenderDB parameters no longer describe is + // rejected rather than answered + throw runtime_error("The query does not match the parameters of this database"); + } Sender::RunQuery(query, _channel); return py::bytes(_channel.extract_out_buffer()); } -public: - size_t db_label_byte_count; - private: shared_ptr _db; StringStreamChannel _channel; @@ -321,9 +412,9 @@ PYBIND11_MODULE(_pyapsi, m) py::module utils = m.def_submodule("utils", "APSI related utilities."); utils.def("_set_log_level", &set_log_level, "Set APSI log level."); - utils.def("_set_console_log_disabled", &Log::SetConsoleDisabled, + utils.def("_set_console_log_disabled", &set_console_log_disabled, "Enable or disable standard out console logging."); - utils.def("_set_log_file", &Log::SetLogFile, + utils.def("_set_log_file", &set_log_file, "Set file for APSI log output."); utils.def("_set_thread_count", &ThreadPoolMgr::SetThreadCount, "Set thread count for parallelization."); @@ -339,8 +430,7 @@ PYBIND11_MODULE(_pyapsi, m) .def("_add_unlabeled_items", &APSIServer::add_unlabeled_items) .def("_handle_oprf_request", &APSIServer::handle_oprf_request) .def("_handle_query", &APSIServer::handle_query) - // TODO: use def_property_readonly instead - .def_readwrite("_db_label_byte_count", &APSIServer::db_label_byte_count); + .def_property_readonly("_db_label_byte_count", &APSIServer::get_label_byte_count); py::class_(m, "APSIClient") .def(py::init()) .def("_oprf_request", &APSIClient::oprf_request) diff --git a/tests/conftest.py b/tests/conftest.py index d4f1b66..00fd786 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,26 +1,31 @@ -import json - import pytest +# APSI's parameters/1M-256.json, as retuned by APSI 1.0 to stay below the documented +# 2^-40 false-positive probability per query +APSI_PARAMS = """ +{ + "table_params": { + "hash_func_count": 3, + "table_size": 585, + "max_items_per_bin": 180 + }, + "item_params": {"felts_per_item": 7}, + "query_params": { + "ps_low_degree": 0, + "query_powers": [ + 1, 3, 4, 6, 10, 13, 15, 21, 29, 37, 45, + 53, 61, 69, 77, 81, 83, 86, 87, 90, 92, 96 + ] + }, + "seal_params": { + "plain_modulus": 65537, + "poly_modulus_degree": 4096, + "coeff_modulus_bits": [40, 40, 24] + } +} +""" + @pytest.fixture def apsi_params() -> str: - return json.dumps( - { - "table_params": { - "hash_func_count": 3, - "table_size": 512, - "max_items_per_bin": 92, - }, - "item_params": {"felts_per_item": 8}, - "query_params": { - "ps_low_degree": 0, - "query_powers": [1, 3, 4, 5, 8, 14, 20, 26, 32, 38, 41, 42, 43, 45, 46], - }, - "seal_params": { - "plain_modulus": 40961, - "poly_modulus_degree": 4096, - "coeff_modulus_bits": [40, 32, 32], - }, - } - ) + return APSI_PARAMS diff --git a/tests/integration_test.py b/tests/integration_test.py index b7201b8..a45ec9e 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -89,6 +89,27 @@ def test_save_and_load_db(apsi_params: str, tmp_path: pathlib.Path): assert _query(client, new_server, ["item", "unknown"]) == ["item"] +def test_save_and_load_labeled_db(apsi_params: str, tmp_path: pathlib.Path): + db_file_path = str(tmp_path / "apsi-labeled.db") + + orig_server = LabeledServer() + orig_server.init_db(apsi_params, max_label_length=10) + orig_server.add_item("item", "1234567890") + orig_server.save_db(db_file_path) + + new_server = LabeledServer() + new_server.load_db(db_file_path) + + # The maximum label length is read back from the loaded database + assert new_server.max_label_length == 10 + with pytest.raises(ValueError, match="length"): + new_server.add_item("item2", "12345678901") + + client = LabeledClient(apsi_params) + + assert _query(client, new_server, ["item", "unknown"]) == {"item": "1234567890"} + + def test_load_non_existent_db_fails(): server = UnlabeledServer() @@ -112,18 +133,21 @@ def test_readme_example(): { "table_params": { "hash_func_count": 3, - "table_size": 512, - "max_items_per_bin": 92 + "table_size": 585, + "max_items_per_bin": 180 }, - "item_params": {"felts_per_item": 8}, + "item_params": {"felts_per_item": 7}, "query_params": { "ps_low_degree": 0, - "query_powers": [1, 3, 4, 5, 8, 14, 20, 26, 32, 38, 41, 42, 43, 45, 46] + "query_powers": [ + 1, 3, 4, 6, 10, 13, 15, 21, 29, 37, 45, + 53, 61, 69, 77, 81, 83, 86, 87, 90, 92, 96 + ] }, "seal_params": { - "plain_modulus": 40961, + "plain_modulus": 65537, "poly_modulus_degree": 4096, - "coeff_modulus_bits": [40, 32, 32] + "coeff_modulus_bits": [40, 40, 24] } } """ diff --git a/tests/utils_test.py b/tests/utils_test.py index 556f7b5..f205ab4 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -5,6 +5,7 @@ import pytest from apsi.servers import UnlabeledServer from apsi.utils import ( + VALID_LOG_LEVELS, disable_console_log, enable_console_log, get_thread_count, @@ -29,6 +30,22 @@ def test_thread_count(thread_count: int): assert get_thread_count() == thread_count +@pytest.mark.parametrize("level", VALID_LOG_LEVELS) +def test_set_log_level(level: str): + set_log_level(level) + + +@pytest.mark.parametrize("level", ["ALL", "OFF"]) +def test_set_deprecated_log_level(level: str): + """The log level names APSI used before 0.13.0 are still accepted.""" + set_log_level(level) + + +def test_set_invalid_log_level_raises_error(): + with pytest.raises(ValueError, match="Invalid log level"): + set_log_level("VERBOSE") + + def test_log_file_output(tmp_path: pathlib.Path, apsi_params: str): set_log_level("ALL") log_file_path = tmp_path / "test.log"