update to APSI 1.1

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.13.4)
+cmake_minimum_required(VERSION 3.25)
 
 project(_pyapsi)
 
@@ -10,6 +10,13 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 # See docs/README.md for details.
 set(APSI_USE_ASM OFF CACHE BOOL "Use ASM implementation" FORCE)
 
+# PyAPSI exposes only APSI's "advanced" API on top of network::StreamChannel, so the
+# ZeroMQ based channel and the command line interface are not needed. Dropping them also
+# drops the ZeroMQ/cppzmq dependency from the build (see VCPKG_DEPENDENCIES in setup.py).
+set(APSI_USE_ZMQ OFF CACHE BOOL "Use ZeroMQ for networking" FORCE)
+set(APSI_BUILD_CLI OFF CACHE BOOL "Build example command line interface applications" FORCE)
+set(APSI_BUILD_TESTS OFF CACHE BOOL "Build unit and integration tests for APSI" FORCE)
+
 include(FetchContent)
 
 FetchContent_Declare(
@@ -19,19 +26,13 @@ FetchContent_Declare(
 )
 FetchContent_MakeAvailable(pybind11)
 
-file(READ external/apsi/CMakeLists.txt APSI_CMAKE_CONTENT)
-string(REPLACE "find_package(SEAL 4.1 QUIET REQUIRED)" "find_package(SEAL QUIET REQUIRED)" APSI_CMAKE_CONTENT "${APSI_CMAKE_CONTENT}")
-file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/apsi_patched/CMakeLists.txt" "${APSI_CMAKE_CONTENT}")
-
-file(GLOB APSI_SUBDIRS LIST_DIRECTORIES true external/apsi/*)
-foreach(SUBDIR ${APSI_SUBDIRS})
-    if(IS_DIRECTORY ${SUBDIR})
-        get_filename_component(DIRNAME ${SUBDIR} NAME)
-        file(COPY ${SUBDIR}/ DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/apsi_patched/${DIRNAME}/")
-    endif()
-endforeach()
+add_subdirectory(external/apsi)
 
-add_subdirectory("${CMAKE_CURRENT_BINARY_DIR}/apsi_patched/")
+# APSI's vendored FourQ headers are configured into ${CMAKE_BINARY_DIR}/common/apsi/fourq/,
+# which equals its own include path only in a standalone APSI build. Here APSI is a
+# subdirectory, so ${CMAKE_BINARY_DIR} is PyAPSI's build root and that path has to be
+# added for APSI's own sources to find "apsi/fourq/...".
+target_include_directories(apsi PUBLIC $<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/common>)
 
 pybind11_add_module(_pyapsi src/main.cpp)
 

README.md

@@ -9,6 +9,27 @@
 Python wrapper for labeled and unlabeled asymmetric private set intersection
 ([APSI](https://github.com/microsoft/apsi)).
 
+Built against APSI 1.1.0.
+
+## Upgrading to PyAPSI 0.3
+
+PyAPSI 0.3 moves from APSI 0.12 to APSI 1.1, which brings a few user visible changes:
+
+- The log levels are named `"TRACE"`, `"DEBUG"`, `"INFO"`, `"WARNING"`, `"ERROR"` and
+  `"SUPPRESS"`. The previous `"ALL"` and `"OFF"` are still accepted as `"TRACE"` and
+  `"SUPPRESS"` respectively.
+- APSI retuned its shipped parameter sets, because most of them were above the documented
+  2^-40 false-positive probability per query. A server and a client have to agree on their
+  parameters, so both sides of a deployment should move to the new values together; the
+  example below is APSI's `parameters/1M-256.json`.
+- APSI's serialization version is unchanged and the schemas stayed wire compatible, so a
+  database saved by an earlier PyAPSI release should still load, but that is untested
+  here. A cuckoo filter written before APSI 1.0 carries no assertion that nothing was
+  dropped from it, so it is loaded as one that may have dropped items; that costs some
+  query time on the affected bin bundles, not correctness.
+- Each query a client sends now carries key material of its own, so that a server can no
+  longer tell from the public keys that two queries came from the same client.
+
 ## Installation
 
 ### From PyPI (builds from source)
@@ -21,8 +42,9 @@ This downloads the source distribution and builds APSI + all dependencies locall
 The build automatically selects the best CPU optimizations for your machine.
 
 **Requirements:**
-- C++ compiler (GCC >= 9, Clang >= 10, or MSVC >= 2019)
-- CMake >= 3.13.4
+- C++ compiler with full C++17 support (GCC, Clang, or Visual Studio 2022/2026, as
+  required by APSI 1.1)
+- CMake >= 3.25
 - Python 3.11–3.14
 - Internet access (vcpkg downloads dependencies during build)
 
@@ -55,18 +77,21 @@ apsi_params = """
 {
     "table_params": {
         "hash_func_count": 3,
-        "table_size": 512,
-        "max_items_per_bin": 92
+        "table_size": 585,
+        "max_items_per_bin": 180
     },
-    "item_params": {"felts_per_item": 8},
+    "item_params": {"felts_per_item": 7},
     "query_params": {
         "ps_low_degree": 0,
-        "query_powers": [1, 3, 4, 5, 8, 14, 20, 26, 32, 38, 41, 42, 43, 45, 46]
+        "query_powers": [
+            1, 3, 4, 6, 10, 13, 15, 21, 29, 37, 45,
+            53, 61, 69, 77, 81, 83, 86, 87, 90, 92, 96
+        ]
     },
     "seal_params": {
-        "plain_modulus": 40961,
+        "plain_modulus": 65537,
         "poly_modulus_degree": 4096,
-        "coeff_modulus_bits": [40, 32, 32]
+        "coeff_modulus_bits": [40, 40, 24]
     }
 }
 """

apsi/servers.py

@@ -59,8 +59,8 @@ class LabeledServer(_BaseServer):
 
     @property
     def max_label_length(self) -> int:
-        # Refers to what is initialized via max_label_length via the underlying C++
-        # implementation, assuring availability also for loaded databases
+        # Read from the underlying APSI database itself, so that it also reflects what a
+        # database loaded via `load_db` was initialized with
         return self._db_label_byte_count
 
     def __init__(self):

apsi/utils.py

@@ -9,7 +9,11 @@
 )
 
 
-VALID_LOG_LEVELS = ["ALL", "DEBUG", "INFO", "WARNING", "ERROR", "OFF"]
+VALID_LOG_LEVELS = ["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "SUPPRESS"]
+
+# APSI 0.13.0 renamed the most verbose log level from "all" to "trace" and the quietest
+# from "off" to "suppress". The previous names remain accepted.
+DEPRECATED_LOG_LEVELS = {"ALL": "TRACE", "OFF": "SUPPRESS"}
 
 
 def set_thread_count(thread_count: int) -> None:
@@ -34,11 +38,13 @@ def set_log_level(level: str) -> None:
     """Set APSI log level.
 
     Args:
-        level: One of "ALL", "DEBUG", "INFO", "WARNING", "ERROR", "OFF"
+        level: One of "TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "SUPPRESS";
+            the former names "ALL" and "OFF" are accepted as "TRACE" and "SUPPRESS"
 
     Raises:
         ValueError: In case an invalid log level is provided
     """
+    level = DEPRECATED_LOG_LEVELS.get(level, level)
     if level not in VALID_LOG_LEVELS:
         raise ValueError(f"Invalid log level {level}, choose one of {VALID_LOG_LEVELS}")
     _set_log_level(level)
@@ -50,10 +56,17 @@ def enable_console_log() -> None:
 
 
 def disable_console_log() -> None:
-    """Disable APSI logging to standard output."""
+    """Disable APSI logging to standard output.
+
+    If a log file was set via `set_log_file`, output keeps going there.
+    """
     _set_console_log_disabled(True)
 
 
 def set_log_file(file_path: str) -> None:
-    """Set file path to which APSI log output should be written."""
+    """Set file path to which APSI log output should be appended.
+
+    Raises:
+        ValueError: In case the file cannot be opened for writing
+    """
     _set_log_file(file_path)

docs/strategy-4-hybrid-wheels-sdist.md

@@ -54,7 +54,7 @@ before-all = """
     yum install -y git zip && \
     git clone https://github.com/microsoft/vcpkg /opt/vcpkg && \
     /opt/vcpkg/bootstrap-vcpkg.sh && \
-    /opt/vcpkg/vcpkg install seal[no-throw-tran]:x64-linux kuku:x64-linux log4cplus:x64-linux cppzmq:x64-linux flatbuffers:x64-linux jsoncpp:x64-linux
+    /opt/vcpkg/vcpkg install seal[core,zstd,no-throw-tran]:x64-linux ms-gsl:x64-linux kuku:x64-linux flatbuffers:x64-linux jsoncpp:x64-linux
 """
 environment = "VCPKG_ROOT_DIR=/opt/vcpkg"
 
@@ -64,7 +64,7 @@ before-all = """
     brew install git && \
     git clone https://github.com/microsoft/vcpkg /tmp/vcpkg && \
     /tmp/vcpkg/bootstrap-vcpkg.sh && \
-    /tmp/vcpkg/vcpkg install seal[no-throw-tran] kuku log4cplus cppzmq flatbuffers jsoncpp
+    /tmp/vcpkg/vcpkg install seal[core,zstd,no-throw-tran] ms-gsl kuku flatbuffers jsoncpp
 """
 environment = "VCPKG_ROOT_DIR=/tmp/vcpkg"
 
@@ -73,7 +73,7 @@ archs = "AMD64"
 before-all = """
     git clone https://github.com/microsoft/vcpkg C:\\vcpkg && \
     C:\\vcpkg\\bootstrap-vcpkg.bat && \
-    C:\\vcpkg\\vcpkg install seal[no-throw-tran]:x64-windows-static-md kuku:x64-windows-static-md log4cplus:x64-windows-static-md cppzmq:x64-windows-static-md flatbuffers:x64-windows-static-md jsoncpp:x64-windows-static-md
+    C:\\vcpkg\\vcpkg install seal[core,zstd,no-throw-tran]:x64-windows-static-md ms-gsl:x64-windows-static-md kuku:x64-windows-static-md flatbuffers:x64-windows-static-md jsoncpp:x64-windows-static-md
 """
 environment = "VCPKG_ROOT_DIR=C:\\vcpkg"
 ```

docs/strategy-5-conda-forge.md

@@ -16,15 +16,17 @@ Packaging PyAPSI for conda-forge would provide an alternative distribution chann
 
 | Dependency | Package name | Status |
 |------------|-------------|--------|
-| SEAL | `libseal` or `seal` | Needs verification |
+| SEAL (>= 4.4.5, built with `SEAL_THROW_ON_TRANSPARENT_CIPHERTEXT=OFF`) | `libseal` or `seal` | Needs verification |
 | Kuku | `kuku` | Needs verification |
-| log4cplus | `log4cplus` | Likely available |
-| cppzmq | `cppzmq` | Available |
-| zeromq | `zeromq` | Available |
 | flatbuffers | `flatbuffers` | Available |
-| jsoncpp | `jsoncpp` | Available |
+| jsoncpp (>= 1.9.5) | `jsoncpp` | Available |
 | Microsoft GSL | `ms-gsl` | Available |
 
+APSI 1.0 requires a SEAL built with `SEAL_THROW_ON_TRANSPARENT_CIPHERTEXT=OFF`, which a
+conda-forge `seal` package is unlikely to provide, so that would have to be verified
+first. ZeroMQ/cppzmq and log4cplus are not needed: PyAPSI builds APSI with
+`APSI_USE_ZMQ=OFF`, and APSI 0.13.0 replaced log4cplus with its built-in logger.
+
 If SEAL and/or Kuku are not on conda-forge, feedstocks for those packages would need to be created first.
 
 ## Approach

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pyapsi"
-version = "0.2.3"
+version = "0.3.0"
 description = "Python wrapper for APSI"
 authors = ["Lukas Grossberger <code@grossberger.xyz>"]
 

setup.py

@@ -53,11 +53,26 @@
 from setuptools import Extension, find_packages, setup
 from setuptools.command.build_ext import build_ext
 
-__version__ = "0.2.3"
-
-APSI_VERSION = "0.12.0"
-APSI_COMMIT = "b967a126b4e1c682b039afc2d76a98ea2c993230"
-VCPKG_COMMIT = "4cf671c303551b27c6360c7c6e1ced5ec46d7258"
+__version__ = "0.3.0"
+
+APSI_VERSION = "1.1.0"
+APSI_COMMIT = "c8020026972258eabad054ad1225aea41f8d1d7d"
+# Matches the "builtin-baseline" of APSI 1.1.0's vcpkg.json, so that the dependency
+# versions are the ones APSI is tested against (Microsoft SEAL 4.4.5, jsoncpp 1.9.6).
+VCPKG_COMMIT = "2c88ca199b27256e4c7d9f796216f82f2b976b0e"
+
+# APSI 1.1.0 dependencies, mirroring its vcpkg.json. Microsoft SEAL has to be built with
+# SEAL_THROW_ON_TRANSPARENT_CIPHERTEXT=OFF ("no-throw-tran"), otherwise APSI refuses to
+# compile. ZeroMQ/cppzmq is not needed because PyAPSI only uses the stream based channel
+# (see APSI_USE_ZMQ in CMakeLists.txt), and log4cplus was dropped by APSI 0.13.0 in favor
+# of its built-in logger.
+VCPKG_DEPENDENCIES = [
+    "seal[core,zstd,no-throw-tran]",
+    "ms-gsl",
+    "kuku",
+    "flatbuffers",
+    "jsoncpp",
+]
 
 PLAT_TO_CMAKE = {
     "win32": "Win32",
@@ -101,12 +116,35 @@ def _get_vcpkg_triplet():
     return "x64-linux"
 
 
-def _vcpkg_deps_installed(vcpkg_dir, triplet):
-    """Check if vcpkg dependencies are installed for the given triplet."""
+def _vcpkg_deps_stamp_path(vcpkg_src_dir, triplet):
+    """Return the path of the file recording which dependencies were installed."""
+    return os.path.join(vcpkg_src_dir, f"pyapsi-deps-{triplet}.stamp")
+
+
+def _vcpkg_deps_stamp(triplet):
+    """Return a marker identifying the dependency set that is required."""
+    return "\n".join([VCPKG_COMMIT, triplet] + sorted(VCPKG_DEPENDENCIES))
+
+
+def _vcpkg_deps_installed(vcpkg_src_dir, triplet):
+    """Check if the required vcpkg dependencies are installed for the given triplet.
+
+    The stamp records the vcpkg commit and the dependency list, so that an existing
+    `_vcpkg` directory left over from an earlier PyAPSI release (which pinned different
+    dependency versions) is detected and rebuilt instead of failing the CMake configure.
+    """
     seal_config = os.path.join(
-        vcpkg_dir, "installed", triplet, "share", "seal", "SEALConfig.cmake"
+        vcpkg_src_dir, "installed", triplet, "share", "seal", "SEALConfig.cmake"
     )
-    return os.path.isfile(seal_config)
+    if not os.path.isfile(seal_config):
+        return False
+
+    stamp_path = _vcpkg_deps_stamp_path(vcpkg_src_dir, triplet)
+    if not os.path.isfile(stamp_path):
+        return False
+
+    with open(stamp_path, "r") as fh:
+        return fh.read() == _vcpkg_deps_stamp(triplet)
 
 
 def _bootstrap_vcpkg(vcpkg_dir):
@@ -163,18 +201,17 @@ def _bootstrap_vcpkg(vcpkg_dir):
 def _install_vcpkg_deps(vcpkg_src_dir, triplet):
     """Install APSI dependencies via vcpkg for the given triplet."""
     print(f"Installing APSI dependencies via vcpkg (triplet: {triplet})...")
-    deps = [
-        "seal[no-throw-tran]",
-        "kuku",
-        "log4cplus",
-        "cppzmq",
-        "flatbuffers",
-        "jsoncpp",
-    ]
     vcpkg_exec = os.path.join(vcpkg_src_dir, "vcpkg")
     if sys.platform.startswith("win"):
         vcpkg_exec = os.path.join(vcpkg_src_dir, "vcpkg.exe")
-    subprocess.check_call([vcpkg_exec, "install", "--triplet", triplet] + deps, cwd=vcpkg_src_dir)
+    subprocess.check_call(
+        [vcpkg_exec, "install", "--triplet", triplet] + VCPKG_DEPENDENCIES,
+        cwd=vcpkg_src_dir,
+    )
+
+    with open(_vcpkg_deps_stamp_path(vcpkg_src_dir, triplet), "w") as fh:
+        fh.write(_vcpkg_deps_stamp(triplet))
+
     print("vcpkg dependency installation complete.")
 
 
@@ -184,14 +221,24 @@ def _get_vcpkg_toolchain(vcpkg_src_dir):
 
 
 def _ensure_apsi_source():
-    """Download APSI source if not present (for sdist installs)."""
+    """Download the pinned APSI source if not already present.
+
+    A stamp file records the commit, so that an `external/apsi` checkout of a different
+    APSI version (for example from a previous PyAPSI release) is replaced rather than
+    silently built against.
+    """
     apsi_dir = os.path.join(
         os.path.dirname(os.path.abspath(__file__)), "external", "apsi"
     )
-    if os.path.isdir(os.path.join(apsi_dir, "CMakeLists.txt")):
-        return
-
-    print("APSI source not found, downloading...")
+    stamp_path = os.path.join(apsi_dir, ".pyapsi-apsi-commit")
+    if os.path.isfile(os.path.join(apsi_dir, "CMakeLists.txt")) and os.path.isfile(
+        stamp_path
+    ):
+        with open(stamp_path, "r") as fh:
+            if fh.read().strip() == APSI_COMMIT:
+                return
+
+    print(f"APSI {APSI_VERSION} source not found, downloading...")
     os.makedirs(os.path.dirname(apsi_dir), exist_ok=True)
 
     tar_url = f"https://github.com/microsoft/APSI/archive/{APSI_COMMIT}.tar.gz"
@@ -211,6 +258,9 @@ def _ensure_apsi_source():
             shutil.rmtree(apsi_dir)
         shutil.move(extracted, apsi_dir)
 
+    with open(stamp_path, "w") as fh:
+        fh.write(APSI_COMMIT)
+
     print("APSI source ready.")
 
 
@@ -253,7 +303,7 @@ def build_extension(self, ext):
         else:
             vcpkg_src_dir = _bootstrap_vcpkg(vcpkg_dir)
 
-        if not _vcpkg_deps_installed(vcpkg_dir, triplet):
+        if not _vcpkg_deps_installed(vcpkg_src_dir, triplet):
             _install_vcpkg_deps(vcpkg_src_dir, triplet)
 
         toolchain = _get_vcpkg_toolchain(vcpkg_src_dir)